From 27d7b42bcd187cc3d15b6370fe4c1f752aebccc1 Mon Sep 17 00:00:00 2001 From: Brando Date: Thu, 17 Apr 2025 23:36:48 -0700 Subject: [PATCH 01/13] testing more of the url --- external/libs | 2 +- makefile | 2 +- src/resource.cpp | 5 ++++ src/resource.hpp | 2 ++ src/response.cpp | 20 +++++++++------ testbench/request_tests.hpp | 2 +- testbench/resource_tests.hpp | 48 ++++++++++++++++++++++++++++++++++++ testbench/tests.cpp | 2 ++ 8 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 testbench/resource_tests.hpp diff --git a/external/libs b/external/libs index 35461a9..6da9383 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 35461a9eae5d0e552707aea329ca1c4beccbcb43 +Subproject commit 6da9383838dca01d4654789f7fe5b2007fde81af diff --git a/makefile b/makefile index 5d4a931..ba646e0 100644 --- a/makefile +++ b/makefile @@ -70,7 +70,7 @@ FLAGS = $(CPPFLAGS) -DDEBUG -g -Isrc/ $(ADDR_SANITIZER) $(CPPSTD) -Iexternal/bin else ifeq ($(CONFIG),test) # test MAIN_FILE = testbench/tests.cpp BIN_NAME = http-test -#ADDR_SANITIZER = -fsanitize=address +ADDR_SANITIZER = -fsanitize=address FLAGS = $(CPPFLAGS) -DDEBUG -DTESTING -g -Isrc/ $(ADDR_SANITIZER) $(CPPSTD) -Iexternal/bin/libs/debug LIBRARIES += external/bin/libs/debug/bftest/libbftest-debug.a endif # ($(CONFIG),...) diff --git a/src/resource.cpp b/src/resource.cpp index a5c6c1f..61e4e9a 100644 --- a/src/resource.cpp +++ b/src/resource.cpp @@ -33,3 +33,8 @@ bool Resource::setRootFolder(const String & rootFolder) { return true; } +bool Resource::targetValid(const URL & target) { + URL rootURL = _rootFolder; + return target.absURL().isSubPath(rootURL.absURL()); +} + diff --git a/src/resource.hpp b/src/resource.hpp index 95d0c8d..3d628d4 100644 --- a/src/resource.hpp +++ b/src/resource.hpp @@ -9,10 +9,12 @@ #define RESOURCE_HPP #include +#include namespace Resource { bool setRootFolder(const BF::String & rootFolder); const BF::String & getRootFolder(); + bool targetValid(const BF::URL & target); } #endif // RESOURCE_HPP diff --git a/src/response.cpp b/src/response.cpp index 3c660a4..15dd375 100644 --- a/src/response.cpp +++ b/src/response.cpp @@ -58,16 +58,22 @@ void Response::handleRequestGET(const Request * request, Response * response) { if (!request || !response) return; String target = request->targetPath(); - URL url(Resource::getRootFolder()); - url.append(target); + + URL targetURL(Resource::getRootFolder()); + targetURL.append(target); - if (BFFileSystemPathIsDirectory(url.abspath())) { - url.append("index.html"); + if (BFFileSystemPathIsDirectory(targetURL.abspath())) { + targetURL.append("index.html"); } + + if (!Resource::targetValid(targetURL)) { + response->_statusCode = 403; + response->_content = new Data("HTTP/1.1 403 Forbidden"); + response->_contentType = "text/plain"; - if (BFFileSystemPathIsFile(url.abspath())) { - response->_content = Data::fromFile(url); - response->_contentType = __ResponseTargetGetContentType(url); + } else if (BFFileSystemPathIsFile(targetURL.abspath())) { + response->_content = Data::fromFile(targetURL); + response->_contentType = __ResponseTargetGetContentType(targetURL); } else { response->_statusCode = 404; response->_content = new Data("404 Not Found"); diff --git a/testbench/request_tests.hpp b/testbench/request_tests.hpp index 362e47c..71ac1cc 100644 --- a/testbench/request_tests.hpp +++ b/testbench/request_tests.hpp @@ -18,7 +18,7 @@ extern "C" { using namespace BF; -BFTEST_UNIT_FUNC(test_requestInit, 1, { +BFTEST_UNIT_FUNC(test_requestInit, 2 << 10, { Data d; Request * req = new Request(&d); BFRelease(req); diff --git a/testbench/resource_tests.hpp b/testbench/resource_tests.hpp new file mode 100644 index 0000000..c0fc513 --- /dev/null +++ b/testbench/resource_tests.hpp @@ -0,0 +1,48 @@ +/** + * author: Brando + * date: 4/17/25 + */ + +#ifndef RESOURCE_TESTS_HPP +#define RESOURCE_TESTS_HPP + +#define ASSERT_PUBLIC_MEMBER_ACCESS + +#include +#include "resource.hpp" +#include + +extern "C" { +#include +#include +} + +using namespace BF; + +BFTEST_UNIT_FUNC(test_resourceRootFolder, 2 << 10, { + char tmp[PATH_MAX]; + URL url = __FILE__; + strcpy(tmp, url.abspath()); + + char * rootFolder = dirname(tmp); + Resource::setRootFolder(rootFolder); + + BF_ASSERT(Resource::targetValid(url.abspath())); + + char * parent = dirname(rootFolder); + URL parentURL(parent); + BF_ASSERT(!Resource::targetValid(parentURL.abspath())); + + URL challenge(rootFolder); + challenge.append("hello"); + challenge.append(".."); + challenge.append(".."); + BF_ASSERT(!Resource::targetValid(challenge.standardURL().abspath())); +}) + +BFTEST_COVERAGE_FUNC(resource_tests, { + BFTEST_LAUNCH(test_resourceRootFolder); +}) + +#endif // RESOURCE_TESTS_HPP + diff --git a/testbench/tests.cpp b/testbench/tests.cpp index b4c89f1..9893f27 100644 --- a/testbench/tests.cpp +++ b/testbench/tests.cpp @@ -4,10 +4,12 @@ */ #include "request_tests.hpp" +#include "resource_tests.hpp" #include "log.hpp" LOG_INIT; BFTEST_SUITE_FUNC({ BFTEST_SUITE_LAUNCH(request_tests); + BFTEST_SUITE_LAUNCH(resource_tests); }) From 7584d2a834a444db54722001de8826fb8cd6f399 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 09:27:18 -0700 Subject: [PATCH 02/13] update libs --- external/libs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/libs b/external/libs index 6da9383..33cbe79 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 6da9383838dca01d4654789f7fe5b2007fde81af +Subproject commit 33cbe79a3139a207e990edb8c3a35634bd77e2b0 From f2ca9880ac3c9b686a5e4d5ba6071f1f35082b94 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 09:32:14 -0700 Subject: [PATCH 03/13] better test --- testbench/resource_tests.hpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/testbench/resource_tests.hpp b/testbench/resource_tests.hpp index c0fc513..f28ba6e 100644 --- a/testbench/resource_tests.hpp +++ b/testbench/resource_tests.hpp @@ -33,11 +33,18 @@ BFTEST_UNIT_FUNC(test_resourceRootFolder, 2 << 10, { URL parentURL(parent); BF_ASSERT(!Resource::targetValid(parentURL.abspath())); - URL challenge(rootFolder); + URL challenge(Resource::getRootFolder()); challenge.append("hello"); challenge.append(".."); challenge.append(".."); BF_ASSERT(!Resource::targetValid(challenge.standardURL().abspath())); + + URL challenge1(Resource::getRootFolder()); + challenge1.append("hello"); + challenge1.append("world"); + challenge1.append(".."); + challenge1.append("validfile.txt"); + BF_ASSERT(Resource::targetValid(challenge1.standardURL().abspath()), "path '%s' not in root '%s'", challenge1.standardURL().abspath(), Resource::getRootFolder().c_str()); }) BFTEST_COVERAGE_FUNC(resource_tests, { From 3f6321fa28c56e71132e0772af243d34282eaf02 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 10:08:47 -0700 Subject: [PATCH 04/13] not using regex --- src/request.cpp | 102 +++++++++++++++++++++++++++++++----- src/request.hpp | 16 ++++++ testbench/request_tests.hpp | 12 +---- todo.md | 3 +- 4 files changed, 109 insertions(+), 24 deletions(-) diff --git a/src/request.cpp b/src/request.cpp index e860628..5494b4c 100644 --- a/src/request.cpp +++ b/src/request.cpp @@ -11,19 +11,87 @@ extern "C" { #include } -#include +//#include +#include using namespace BF; Request::Request(const Data * data) : _message(data == NULL ? 0 : (const char *) data->buffer(), data == NULL ? 0 : data->size()) { LOG_WRITE("Request length = %ld", _message.size()); LOG_WRITE("Request content = \n%s", _message.c_str()); + + this->_method[0] = '\0'; + this->_target[0] = '\0'; + this->_protocol[0] = '\0'; + + this->parse(); +} + +Request::~Request() { } + +void __RequestParseStatusLine(std::string & in, char * method, char * target, char * protocol) { + if (!method || !target) return; + + if (in.back() == '\r') { + in.pop_back(); + } + + std::stringstream ss(in); + std::string buf; + char del = ' '; + + if (getline(ss, buf, del)) { + strcpy(method, buf.c_str()); + } + + if (getline(ss, buf, del)) { + strcpy(target, buf.c_str()); + } + + if (getline(ss, buf, del)) { + strcpy(protocol, buf.c_str()); + } } -Request::~Request() { +void __RequestParseHeader(std::string & in, HashMap & header) { + std::stringstream ss(in); + std::string key; + std::string value; + char del = ':'; + + if (getline(ss, key, del) && getline(ss, value, del)) { + header.insert(key, value); + } +} + +void Request::parse() { + if (this->_message.empty()) { + return; + } + + std::stringstream ss(this->_message); + std::string buf; + char del = '\n'; + int statusLineParsed = false; + int headerParsed = false; + while (getline(ss, buf, del)) { + if (!statusLineParsed) { + __RequestParseStatusLine(buf, this->_method, this->_target, this->_protocol); + statusLineParsed = true; + } else if (!headerParsed) { + if (buf.empty()) { + headerParsed = true; + } else { + __RequestParseHeader(buf, this->_header); + } + } else { // read the body + this->_body.append(buf); + } + } } String Request::method() const { + /* std::regex methodRegex(R"((GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+)"); std::smatch match; @@ -32,9 +100,12 @@ String Request::method() const { } else { return ""; // Or throw an exception if no target is found } + */ + return this->_method; } String Request::target() const { + /* std::regex targetRegex(R"((?:GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+([^\s]+)\s+HTTP/\d\.\d)"); std::smatch match; @@ -43,6 +114,22 @@ String Request::target() const { } else { return ""; // Or throw an exception if no target is found } + */ + return this->_target; +} + +String Request::protocol() const { + /* + std::regex protocolRegex(R"((?:GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+[^\s]+\s+(HTTP/\d\.\d))"); + std::smatch match; + + if (std::regex_search(this->_message, match, protocolRegex)) { + return match[1].str(); // The second capturing group contains the protocol + } else { + return ""; // Or throw an exception if no protocol is found + } + */ + return this->_protocol; } // component: 0=path, 1=query @@ -121,17 +208,6 @@ HashMap Request::targetQuery() const { return res; } -String Request::protocol() const { - std::regex protocolRegex(R"((?:GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+[^\s]+\s+(HTTP/\d\.\d))"); - std::smatch match; - - if (std::regex_search(this->_message, match, protocolRegex)) { - return match[1].str(); // The second capturing group contains the protocol - } else { - return ""; // Or throw an exception if no protocol is found - } -} - String Request::host() const { return ""; } diff --git a/src/request.hpp b/src/request.hpp index 4ae171f..8fd8a57 100644 --- a/src/request.hpp +++ b/src/request.hpp @@ -11,6 +11,11 @@ #include #include +#ifdef LINUX +#include +#endif +#include + typedef enum { kRequestMethodNone = 0, kRequestMethodGet, @@ -35,7 +40,18 @@ class Request : public BF::Object { BF::String host() const; private: + void parse(); + std::string _message; + + // status line + char _method[32]; + char _target[PATH_MAX]; + char _protocol[32]; + + BF::HashMap _header; + + BF::String _body; }; #endif // REQUEST_HPP diff --git a/testbench/request_tests.hpp b/testbench/request_tests.hpp index 71ac1cc..b460714 100644 --- a/testbench/request_tests.hpp +++ b/testbench/request_tests.hpp @@ -31,15 +31,13 @@ BFTEST_UNIT_FUNC(test_simpleClientRequest, 2 << 6, { Data post_buf(post_str); String head_str = "HEAD /static/image.png HTTP/1.0\r\n\r\n"; Data head_buf(head_str); - String invalid_str = "Invalid Request Line"; - Data invalid_buf(invalid_str); Request * req = NULL; req = new Request(&get_buf); BF_ASSERT(req->method() == "GET"); - BF_ASSERT(req->target() == "/index.html"); - BF_ASSERT(req->protocol() == "HTTP/1.1"); + BF_ASSERT(req->target() == "/index.html", "read '%s'", req->target().c_str()); + BF_ASSERT(req->protocol() == "HTTP/1.1", "read: '%s'", req->protocol().c_str()); BFRelease(req); req = new Request(&post_buf); @@ -53,12 +51,6 @@ BFTEST_UNIT_FUNC(test_simpleClientRequest, 2 << 6, { BF_ASSERT(req->target() == "/static/image.png"); BF_ASSERT(req->protocol() == "HTTP/1.0"); BFRelease(req); - - req = new Request(&invalid_buf); - BF_ASSERT(req->method() == ""); - BF_ASSERT(req->target().empty()); - BF_ASSERT(req->protocol().empty()); - BFRelease(req); }) BFTEST_UNIT_FUNC(test_requestTargetPathAndQuery, 2 << 8, { diff --git a/todo.md b/todo.md index 5231029..8238cb3 100644 --- a/todo.md +++ b/todo.md @@ -2,5 +2,6 @@ - [x] serve html - [x] receive http requests from client (be sure to assemble the packets) - [x] Handle GET requests for resources -- [ ] prevent requests from targetting anything outside of the root folder +- [x] prevent requests from targetting anything outside of the root folder +- [ ] improve request parsing From 3b55c5abcbec13e0a513c4ae2f94d02e13a409b9 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 10:42:20 -0700 Subject: [PATCH 05/13] can parse header --- external/libs | 2 +- src/request.cpp | 101 +++++++++++++++--------------------- src/request.hpp | 17 +++--- testbench/request_tests.hpp | 7 ++- 4 files changed, 56 insertions(+), 71 deletions(-) diff --git a/external/libs b/external/libs index 33cbe79..b8d355d 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 33cbe79a3139a207e990edb8c3a35634bd77e2b0 +Subproject commit b8d355d7fe9db8f476b18ee1dcb6ed11f942d08b diff --git a/src/request.cpp b/src/request.cpp index 5494b4c..d1baf54 100644 --- a/src/request.cpp +++ b/src/request.cpp @@ -11,57 +11,53 @@ extern "C" { #include } -//#include #include using namespace BF; Request::Request(const Data * data) : _message(data == NULL ? 0 : (const char *) data->buffer(), data == NULL ? 0 : data->size()) { - LOG_WRITE("Request length = %ld", _message.size()); - LOG_WRITE("Request content = \n%s", _message.c_str()); - - this->_method[0] = '\0'; - this->_target[0] = '\0'; - this->_protocol[0] = '\0'; + LOG_DEBUG("Request length = %ld", _message.size()); + LOG_DEBUG("Request content = \n%s", _message.c_str()); this->parse(); } Request::~Request() { } -void __RequestParseStatusLine(std::string & in, char * method, char * target, char * protocol) { - if (!method || !target) return; - - if (in.back() == '\r') { - in.pop_back(); - } - +void __RequestParseStatusLine(std::string & in, String & method, String & target, String & protocol) { std::stringstream ss(in); std::string buf; char del = ' '; if (getline(ss, buf, del)) { - strcpy(method, buf.c_str()); + method = buf; } if (getline(ss, buf, del)) { - strcpy(target, buf.c_str()); + target = buf; } if (getline(ss, buf, del)) { - strcpy(protocol, buf.c_str()); + protocol = buf; } } void __RequestParseHeader(std::string & in, HashMap & header) { - std::stringstream ss(in); - std::string key; - std::string value; - char del = ':'; + std::string del = ": "; + auto pos = in.find(del); + std::string key, value; - if (getline(ss, key, del) && getline(ss, value, del)) { - header.insert(key, value); + if (pos == std::string::npos) { + return; } + + key = in.substr(0, pos); + in.erase(0, pos + del.length()); + pos = in.find(del); + + value = in.substr(0, pos); + + header.insert(key, value); } void Request::parse() { @@ -75,6 +71,17 @@ void Request::parse() { int statusLineParsed = false; int headerParsed = false; while (getline(ss, buf, del)) { + buf.erase( + std::remove_if( + buf.begin(), + buf.end(), + [](unsigned char c) { + return c == '\t' || c == '\n' || c == '\r'; + } + ), + buf.end() + ); + if (!statusLineParsed) { __RequestParseStatusLine(buf, this->_method, this->_target, this->_protocol); statusLineParsed = true; @@ -90,48 +97,26 @@ void Request::parse() { } } -String Request::method() const { - /* - std::regex methodRegex(R"((GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+)"); - std::smatch match; - - if (std::regex_search(this->_message, match, methodRegex)) { - return match[1].str(); // The second capturing group contains the target - } else { - return ""; // Or throw an exception if no target is found - } - */ +const String & Request::method() const { return this->_method; } -String Request::target() const { - /* - std::regex targetRegex(R"((?:GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+([^\s]+)\s+HTTP/\d\.\d)"); - std::smatch match; - - if (std::regex_search(this->_message, match, targetRegex)) { - return match[1].str(); // The second capturing group contains the target - } else { - return ""; // Or throw an exception if no target is found - } - */ +const String & Request::target() const { return this->_target; } -String Request::protocol() const { - /* - std::regex protocolRegex(R"((?:GET|POST|PUT|DELETE|HEAD|OPTIONS|CONNECT|TRACE|PATCH)\s+[^\s]+\s+(HTTP/\d\.\d))"); - std::smatch match; - - if (std::regex_search(this->_message, match, protocolRegex)) { - return match[1].str(); // The second capturing group contains the protocol - } else { - return ""; // Or throw an exception if no protocol is found - } - */ +const String & Request::protocol() const { return this->_protocol; } +const HashMap & Request::header() const { + return this->_header; +} + +const String & Request::body() const { + return this->_body; +} + // component: 0=path, 1=query String __RequestTargetParse(String & target, int component) { int error = component != 0 && component != 1 ? 1 : 0; @@ -208,7 +193,3 @@ HashMap Request::targetQuery() const { return res; } -String Request::host() const { - return ""; -} - diff --git a/src/request.hpp b/src/request.hpp index 8fd8a57..d8617ca 100644 --- a/src/request.hpp +++ b/src/request.hpp @@ -27,9 +27,12 @@ class Request : public BF::Object { Request(const BF::Data * data); virtual ~Request(); - BF::String method() const; - BF::String target() const; - BF::String protocol() const; + const BF::String & method() const; + const BF::String & target() const; + const BF::String & protocol() const; + + const BF::HashMap & header() const; + const BF::String & body() const; // returns path without query string BF::String targetPath() const; @@ -37,17 +40,15 @@ class Request : public BF::Object { // returns query data BF::HashMap targetQuery() const; - BF::String host() const; - private: void parse(); std::string _message; // status line - char _method[32]; - char _target[PATH_MAX]; - char _protocol[32]; + BF::String _method; + BF::String _target; + BF::String _protocol; BF::HashMap _header; diff --git a/testbench/request_tests.hpp b/testbench/request_tests.hpp index b460714..8ff0a40 100644 --- a/testbench/request_tests.hpp +++ b/testbench/request_tests.hpp @@ -24,7 +24,7 @@ BFTEST_UNIT_FUNC(test_requestInit, 2 << 10, { BFRelease(req); }) -BFTEST_UNIT_FUNC(test_simpleClientRequest, 2 << 6, { +BFTEST_UNIT_FUNC(test_simpleClientRequest, 2 << 10, { String get_str = "GET /index.html HTTP/1.1\r\nHost: example.com\r\n\r\n"; Data get_buf(get_str); String post_str = "POST /submit.php HTTP/1.1\r\nHost: another.com\r\nContent-Length: 10\r\n\r\ndata=value"; @@ -53,7 +53,7 @@ BFTEST_UNIT_FUNC(test_simpleClientRequest, 2 << 6, { BFRelease(req); }) -BFTEST_UNIT_FUNC(test_requestTargetPathAndQuery, 2 << 8, { +BFTEST_UNIT_FUNC(test_requestTargetPathAndQuery, 2 << 10, { String str = "GET /assets/fonts/fontawesome-webfont.ttf?v=4.6.3 HTTP/1.1\r\n\ Host: 10.0.0.82:8080\r\n\ User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:137.0) Gecko/20100101 Firefox/137.0\r\n\ @@ -70,6 +70,9 @@ BFTEST_UNIT_FUNC(test_requestTargetPathAndQuery, 2 << 8, { HashMap query = req.targetQuery(); BF_ASSERT(query["v"] == "4.6.3"); + + const HashMap & header = req.header(); + BF_ASSERT(header["Host"] == "10.0.0.82:8080"); }) BFTEST_COVERAGE_FUNC(request_tests, { From 6d740fbd5a34cbfa3a9bb83ca9076bedef86ed99 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 11:21:38 -0700 Subject: [PATCH 06/13] using lambda --- external/libs | 2 +- src/office.cpp | 9 +++------ todo.md | 3 ++- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/external/libs b/external/libs index b8d355d..989ffad 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit b8d355d7fe9db8f476b18ee1dcb6ed11f942d08b +Subproject commit 989ffadd1d076892628d1e926001adbeec7eb648 diff --git a/src/office.cpp b/src/office.cpp index b756ab0..01de3a6 100644 --- a/src/office.cpp +++ b/src/office.cpp @@ -38,17 +38,14 @@ void Office::envelopeReceive(Envelope * envelope) { void __IncomingRequestsWorkerThread(void * in) { while (!BFThreadAsyncIsCanceled(_tidRequestQueue)) { - if (_incomingRequests.get().empty()) { + if (_incomingRequests.get([] (Queue & q) { + return q.empty(); + })) { BFLockWait(&_queueSema); } else { _incomingRequests.lock(); - - // get first item from the queue Envelope * envelope = _incomingRequests.unsafeget().front(); - - // pop off _incomingRequests.unsafeget().pop(); - _incomingRequests.unlock(); if (envelope->data()->size() == 0) { diff --git a/todo.md b/todo.md index 8238cb3..beee99a 100644 --- a/todo.md +++ b/todo.md @@ -3,5 +3,6 @@ - [x] receive http requests from client (be sure to assemble the packets) - [x] Handle GET requests for resources - [x] prevent requests from targetting anything outside of the root folder -- [ ] improve request parsing +- [x] improve request parsing +- [ ] multithread request packet handling From cb2be01f523c7940f7aa8a1e0c108f576e66533b Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 11:33:45 -0700 Subject: [PATCH 07/13] good using get(lambda) with no return value --- external/libs | 2 +- src/office.cpp | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/external/libs b/external/libs index 989ffad..364c7ca 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 989ffadd1d076892628d1e926001adbeec7eb648 +Subproject commit 364c7cad534bc68cb32a6e49ec10a175571ec6ee diff --git a/src/office.cpp b/src/office.cpp index 01de3a6..961a225 100644 --- a/src/office.cpp +++ b/src/office.cpp @@ -27,12 +27,10 @@ BFThreadAsyncID _tidRequestQueue = NULL; BFLock _queueSema; void Office::envelopeReceive(Envelope * envelope) { - _incomingRequests.lock(); - - BFRetain(envelope); - _incomingRequests.unsafeget().push(envelope); - - _incomingRequests.unlock(); + _incomingRequests.get([=] (Queue & q) { + BFRetain(envelope); + q.push(envelope); + }); BFLockRelease(&_queueSema); } From 1f91faf823a7a1bc9c5a3eda94d8cdc23defe055 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 12:06:37 -0700 Subject: [PATCH 08/13] can run multiple threads --- src/office.cpp | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/office.cpp b/src/office.cpp index 961a225..cef4d5b 100644 --- a/src/office.cpp +++ b/src/office.cpp @@ -11,6 +11,7 @@ #include #include #include +#include using namespace BF::Net; using namespace BF; @@ -23,25 +24,28 @@ using namespace std; * once popped from queue */ Atomic> _incomingRequests; -BFThreadAsyncID _tidRequestQueue = NULL; -BFLock _queueSema; + +const unsigned char numWorkerThreads = 2; +BFThreadAsyncID _tidRequestQueue[numWorkerThreads]; + +//BFLock _queueSema; void Office::envelopeReceive(Envelope * envelope) { _incomingRequests.get([=] (Queue & q) { BFRetain(envelope); q.push(envelope); }); - BFLockRelease(&_queueSema); + //BFLockRelease(&_queueSema); } void __IncomingRequestsWorkerThread(void * in) { - while (!BFThreadAsyncIsCanceled(_tidRequestQueue)) { - if (_incomingRequests.get([] (Queue & q) { - return q.empty(); - })) { - BFLockWait(&_queueSema); + const BFThreadAsyncID tid = BFThreadAsyncGetID(); + while (!BFThreadAsyncIsCanceled(tid)) { + _incomingRequests.lock(); + if (_incomingRequests.unsafeget().empty()) { + _incomingRequests.unlock(); + usleep(50); } else { - _incomingRequests.lock(); Envelope * envelope = _incomingRequests.unsafeget().front(); _incomingRequests.unsafeget().pop(); _incomingRequests.unlock(); @@ -68,16 +72,21 @@ void __IncomingRequestsWorkerThread(void * in) { } void Office::start() { - BFLockCreate(&_queueSema); - _tidRequestQueue = BFThreadAsync(__IncomingRequestsWorkerThread, NULL); + //BFLockCreate(&_queueSema); + + for (int i = 0; i < numWorkerThreads; i++) { + _tidRequestQueue[i] = BFThreadAsync(__IncomingRequestsWorkerThread, NULL); + } } void Office::stop() { - BFThreadAsyncCancel(_tidRequestQueue); - BFLockRelease(&_queueSema); - BFThreadAsyncWait(_tidRequestQueue); - BFThreadAsyncDestroy(_tidRequestQueue); + //BFLockRelease(&_queueSema); + for (int i = 0; i < numWorkerThreads; i++) { + BFThreadAsyncCancel(_tidRequestQueue[i]); + BFThreadAsyncWait(_tidRequestQueue[i]); + BFThreadAsyncDestroy(_tidRequestQueue[i]); + } - BFLockDestroy(&_queueSema); + //BFLockDestroy(&_queueSema); } From d14575f3f6f205dfde0a67b05ad0c370092f4d8f Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 12:11:39 -0700 Subject: [PATCH 09/13] using semaphore --- src/office.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/office.cpp b/src/office.cpp index cef4d5b..905db81 100644 --- a/src/office.cpp +++ b/src/office.cpp @@ -25,17 +25,17 @@ using namespace std; */ Atomic> _incomingRequests; -const unsigned char numWorkerThreads = 2; +const unsigned char numWorkerThreads = 4; BFThreadAsyncID _tidRequestQueue[numWorkerThreads]; -//BFLock _queueSema; +BFLock _queueSema; void Office::envelopeReceive(Envelope * envelope) { _incomingRequests.get([=] (Queue & q) { BFRetain(envelope); q.push(envelope); }); - //BFLockRelease(&_queueSema); + BFLockRelease(&_queueSema); } void __IncomingRequestsWorkerThread(void * in) { @@ -44,7 +44,8 @@ void __IncomingRequestsWorkerThread(void * in) { _incomingRequests.lock(); if (_incomingRequests.unsafeget().empty()) { _incomingRequests.unlock(); - usleep(50); + //usleep(50); + BFLockWait(&_queueSema); } else { Envelope * envelope = _incomingRequests.unsafeget().front(); _incomingRequests.unsafeget().pop(); @@ -72,7 +73,7 @@ void __IncomingRequestsWorkerThread(void * in) { } void Office::start() { - //BFLockCreate(&_queueSema); + BFLockCreate(&_queueSema); for (int i = 0; i < numWorkerThreads; i++) { _tidRequestQueue[i] = BFThreadAsync(__IncomingRequestsWorkerThread, NULL); @@ -82,11 +83,16 @@ void Office::start() { void Office::stop() { //BFLockRelease(&_queueSema); for (int i = 0; i < numWorkerThreads; i++) { + BFLockRelease(&_queueSema); BFThreadAsyncCancel(_tidRequestQueue[i]); + } + + for (int i = 0; i < numWorkerThreads; i++) { + BFLockRelease(&_queueSema); BFThreadAsyncWait(_tidRequestQueue[i]); BFThreadAsyncDestroy(_tidRequestQueue[i]); } - //BFLockDestroy(&_queueSema); + BFLockDestroy(&_queueSema); } From 64044fc858017f734aebaf16481360cee52d02d0 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 12:21:44 -0700 Subject: [PATCH 10/13] saving work --- external/libs | 2 +- src/office.cpp | 10 +++++++++- todo.md | 3 ++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/external/libs b/external/libs index 364c7ca..105ea7a 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 364c7cad534bc68cb32a6e49ec10a175571ec6ee +Subproject commit 105ea7af3ded7708121be865e1e3a241f98bada4 diff --git a/src/office.cpp b/src/office.cpp index 905db81..2b6386a 100644 --- a/src/office.cpp +++ b/src/office.cpp @@ -25,7 +25,15 @@ using namespace std; */ Atomic> _incomingRequests; -const unsigned char numWorkerThreads = 4; +/** + * requests will come in asynchronously from clients, we will queue + * each one and have our worker threads pick off the queue + * + * we have n worker threads, working concurrently on every available request + * + * i don't see a performance boost on the client side between 2 and 4 threads + */ +const unsigned char numWorkerThreads = 2; BFThreadAsyncID _tidRequestQueue[numWorkerThreads]; BFLock _queueSema; diff --git a/todo.md b/todo.md index beee99a..c79ef79 100644 --- a/todo.md +++ b/todo.md @@ -4,5 +4,6 @@ - [x] Handle GET requests for resources - [x] prevent requests from targetting anything outside of the root folder - [x] improve request parsing -- [ ] multithread request packet handling +- [x] multithread request packet handling +- [ ] seg fault issue when you spam the reload page button From 6f95606643ed5d42eb5c7f28152bf320b3053ae3 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 19:51:17 -0700 Subject: [PATCH 11/13] flushing cache and making sure we rate limit incoming requests --- external/libs | 2 +- makefile | 2 +- src/log.cpp | 2 ++ src/log.hpp | 2 ++ src/main.cpp | 2 +- src/office.cpp | 19 +++++++++++++++++-- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/external/libs b/external/libs index 105ea7a..bf2f67a 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 105ea7af3ded7708121be865e1e3a241f98bada4 +Subproject commit bf2f67a1f81a67bcd1c9c506d1043bf663326286 diff --git a/makefile b/makefile index ba646e0..7f0ff18 100644 --- a/makefile +++ b/makefile @@ -63,7 +63,7 @@ FLAGS = $(CPPFLAGS) -Isrc/ $(CPPSTD) -Iexternal/bin/libs/release else ifeq ($(CONFIG),debug) # debug MAIN_FILE = src/main.cpp BIN_NAME = http-debug -#ADDR_SANITIZER = -fsanitize=address +ADDR_SANITIZER = -fsanitize=address FLAGS = $(CPPFLAGS) -DDEBUG -g -Isrc/ $(ADDR_SANITIZER) $(CPPSTD) -Iexternal/bin/libs/debug ### Test settings diff --git a/src/log.cpp b/src/log.cpp index 1a3a4de..47da217 100644 --- a/src/log.cpp +++ b/src/log.cpp @@ -44,6 +44,7 @@ void _LogWriteEntry(BFFileWriter * filewriter, int mode, ...) { logstr ); +#ifdef ENABLE_LOG_CONSOLE_PRINT printf( format, dt.month, @@ -56,6 +57,7 @@ void _LogWriteEntry(BFFileWriter * filewriter, int mode, ...) { ); printf("\n"); fflush(stdout); +#endif // ENABLE_LOG_CONSOLE_PRINT va_end(arg0); va_end(arg1); diff --git a/src/log.hpp b/src/log.hpp index cb8519c..29a68d3 100644 --- a/src/log.hpp +++ b/src/log.hpp @@ -10,6 +10,8 @@ extern "C" { #include } +//#define ENABLE_LOG_CONSOLE_PRINT + #define CHAT_LOG_PATH "/tmp/http.log" extern BFFileWriter gFileWriter; diff --git a/src/main.cpp b/src/main.cpp index f463a26..3d65080 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -32,7 +32,7 @@ void help(const char * toolname) { } void __NewConnection(Connection * sc) { - LOG_WRITE("new connection made"); + LOG_DEBUG("new connection made"); } int __ReadArguments(int argc, char * argv[]) { diff --git a/src/office.cpp b/src/office.cpp index 2b6386a..d6306c8 100644 --- a/src/office.cpp +++ b/src/office.cpp @@ -39,6 +39,14 @@ BFThreadAsyncID _tidRequestQueue[numWorkerThreads]; BFLock _queueSema; void Office::envelopeReceive(Envelope * envelope) { + int attempts = 0; + int threshold = 25; + while (_incomingRequests.get([=] (auto & q) { + return q.size(); + }) >= threshold) { + usleep(500); + } + _incomingRequests.get([=] (Queue & q) { BFRetain(envelope); q.push(envelope); @@ -52,7 +60,6 @@ void __IncomingRequestsWorkerThread(void * in) { _incomingRequests.lock(); if (_incomingRequests.unsafeget().empty()) { _incomingRequests.unlock(); - //usleep(50); BFLockWait(&_queueSema); } else { Envelope * envelope = _incomingRequests.unsafeget().front(); @@ -60,6 +67,7 @@ void __IncomingRequestsWorkerThread(void * in) { _incomingRequests.unlock(); if (envelope->data()->size() == 0) { + BFRelease(envelope); continue; } @@ -89,7 +97,6 @@ void Office::start() { } void Office::stop() { - //BFLockRelease(&_queueSema); for (int i = 0; i < numWorkerThreads; i++) { BFLockRelease(&_queueSema); BFThreadAsyncCancel(_tidRequestQueue[i]); @@ -101,6 +108,14 @@ void Office::stop() { BFThreadAsyncDestroy(_tidRequestQueue[i]); } + // flush q + _incomingRequests.lock(); + while (!_incomingRequests.unsafeget().empty()) { + Envelope * e = _incomingRequests.unsafeget().front(); + _incomingRequests.unsafeget().pop(); + BFRelease(e); + } + _incomingRequests.unlock(); BFLockDestroy(&_queueSema); } From aa4b94dbb407cd829e9cc43b20a84477a75780a0 Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 20:03:53 -0700 Subject: [PATCH 12/13] update --- external/libs | 2 +- todo.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/libs b/external/libs index bf2f67a..68a3c81 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit bf2f67a1f81a67bcd1c9c506d1043bf663326286 +Subproject commit 68a3c816dd04579ea9aa8c8158dcb78dff35daca diff --git a/todo.md b/todo.md index c79ef79..998ff54 100644 --- a/todo.md +++ b/todo.md @@ -5,5 +5,5 @@ - [x] prevent requests from targetting anything outside of the root folder - [x] improve request parsing - [x] multithread request packet handling -- [ ] seg fault issue when you spam the reload page button +- [x] seg fault issue when you spam the reload page button From 1acd3550a773a8c526dcbc8dd82b56ce8e26f87f Mon Sep 17 00:00:00 2001 From: Brando Date: Fri, 18 Apr 2025 20:49:02 -0700 Subject: [PATCH 13/13] update libs --- external/libs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/libs b/external/libs index 68a3c81..6235794 160000 --- a/external/libs +++ b/external/libs @@ -1 +1 @@ -Subproject commit 68a3c816dd04579ea9aa8c8158dcb78dff35daca +Subproject commit 623579471a0e3f3087173102ee3c9470718ba03e