From 9884b3f789dba7731f645b82d73c054258113a49 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 18 Dec 2025 11:48:33 +0100 Subject: [PATCH 1/6] security correction --- library.json | 2 +- library.properties | 2 +- src/AsyncHttpClient.cpp | 43 ++++++++++++++++++++++++++++++++++++----- src/HttpCommon.h | 29 ++++++++++++++++++++++++++- src/HttpRequest.cpp | 9 ++++++--- 5 files changed, 74 insertions(+), 11 deletions(-) diff --git a/library.json b/library.json index 3ad5bcd..56bedf3 100644 --- a/library.json +++ b/library.json @@ -1,6 +1,6 @@ { "name": "ESPAsyncWebClient", - "version": "1.1.3", + "version": "1.1.4", "description": "Asynchronous HTTP client library for ESP32 ", "keywords": [ "http", diff --git a/library.properties b/library.properties index 0a2252d..cb2f219 100644 --- a/library.properties +++ b/library.properties @@ -1,5 +1,5 @@ name=ESPAsyncWebClient -version=1.1.3 +version=1.1.4 author=playmiel maintainer=playmiel sentence=Asynchronous HTTP client library for ESP32 microcontrollers diff --git a/src/AsyncHttpClient.cpp b/src/AsyncHttpClient.cpp index 4fecfaf..21543f7 100644 --- a/src/AsyncHttpClient.cpp +++ b/src/AsyncHttpClient.cpp @@ -225,7 +225,12 @@ uint32_t AsyncHttpClient::patch(const char* url, const char* data, SuccessCallba } void AsyncHttpClient::setHeader(const char* name, const char* value) { - String nameStr(name), valueStr(value); + if (!name) + return; + String nameStr(name); + String valueStr(value ? value : ""); + if (!isValidHttpHeaderName(nameStr) || !isValidHttpHeaderValue(valueStr)) + return; lock(); for (auto& h : _defaultHeaders) { if (h.name.equalsIgnoreCase(nameStr)) { @@ -264,7 +269,7 @@ void AsyncHttpClient::setTimeout(uint32_t timeout) { } void AsyncHttpClient::setUserAgent(const char* userAgent) { lock(); - _defaultUserAgent = String(userAgent); + _defaultUserAgent = userAgent ? String(userAgent) : String(); unlock(); } @@ -362,6 +367,12 @@ void AsyncHttpClient::setCookie(const char* name, const char* value, const char* bool secure) { if (!name || strlen(name) == 0) return; + if (!isValidHttpHeaderValue(String(name))) + return; + if (strchr(name, '=') || strchr(name, ';')) + return; + if (value && !isValidHttpHeaderValue(String(value))) + return; int64_t now = currentTimeSeconds(); StoredCookie cookie; cookie.name = String(name); @@ -396,6 +407,11 @@ void AsyncHttpClient::setCookie(const char* name, const char* value, const char* uint32_t AsyncHttpClient::makeRequest(HttpMethod method, const char* url, const char* data, SuccessCallback onSuccess, ErrorCallback onError) { + if (!url || strlen(url) == 0) { + if (onError) + onError(CONNECTION_FAILED, "URL is empty"); + return 0; + } // Snapshot global defaults under lock to avoid concurrent modification issues std::vector headersCopy; String uaCopy; @@ -425,6 +441,17 @@ uint32_t AsyncHttpClient::makeRequest(HttpMethod method, const char* url, const } uint32_t AsyncHttpClient::request(AsyncHttpRequest* request, SuccessCallback onSuccess, ErrorCallback onError) { + if (!request) { + if (onError) + onError(CONNECTION_FAILED, "Request is null"); + return 0; + } + if (request->getHost().length() == 0 || request->getPath().length() == 0) { + if (onError) + onError(CONNECTION_FAILED, "Invalid URL"); + delete request; + return 0; + } RequestContext* ctx = new RequestContext(); ctx->request = request; ctx->response = new AsyncHttpResponse(); @@ -606,6 +633,10 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); return; } + if (storeBody && context->expectedContentLength > 0 && !context->chunked && + (!enforceLimit || context->expectedContentLength <= _maxBodySize)) { + context->response->reserveBody(context->expectedContentLength); + } context->responseBuffer.remove(0, headerEnd + 4); if (handleRedirect(context)) return; @@ -846,6 +877,7 @@ bool AsyncHttpClient::parseResponseHeaders(RequestContext* context, const String if (colonPos != -1) { String name = line.substring(0, colonPos); String value = line.substring(colonPos + 1); + name.trim(); value.trim(); context->response->setHeader(name, value); if (name.equalsIgnoreCase("Content-Length")) { @@ -854,9 +886,6 @@ bool AsyncHttpClient::parseResponseHeaders(RequestContext* context, const String parsed = 0; context->expectedContentLength = (size_t)parsed; context->response->setContentLength(context->expectedContentLength); - bool storeBody = !context->request->getNoStoreBody(); - if (storeBody) - context->response->reserveBody(context->expectedContentLength); } else if (name.equalsIgnoreCase("Transfer-Encoding") && value.equalsIgnoreCase("chunked")) { context->chunked = true; } else if (name.equalsIgnoreCase("Connection")) { @@ -1198,6 +1227,10 @@ void AsyncHttpClient::sendStreamData(RequestContext* context) { triggerError(context, BODY_STREAM_READ_FAILED, "Body stream read failed"); return; } + if (written > (int)sizeof(temp)) { + triggerError(context, BODY_STREAM_READ_FAILED, "Body stream provider overrun"); + return; + } if (written > 0) context->transport->write((const char*)temp, written); if (final) diff --git a/src/HttpCommon.h b/src/HttpCommon.h index fa08873..b9a94e5 100644 --- a/src/HttpCommon.h +++ b/src/HttpCommon.h @@ -11,7 +11,7 @@ // Library version (single source of truth inside code). Keep in sync with library.json and library.properties. #ifndef ESP_ASYNC_WEB_CLIENT_VERSION -#define ESP_ASYNC_WEB_CLIENT_VERSION "1.1.3" +#define ESP_ASYNC_WEB_CLIENT_VERSION "1.1.4" #endif struct HttpHeader { @@ -92,4 +92,31 @@ inline const char* httpClientErrorToString(HttpClientError error) { } } +// Basic validation to prevent request-line / header injection when user-provided strings are used. +// This is intentionally strict: CR/LF and ASCII control characters are rejected. +inline bool isValidHttpHeaderName(const String& name) { + if (name.length() == 0) + return false; + for (size_t i = 0; i < name.length(); ++i) { + unsigned char c = static_cast(name.charAt(i)); + bool ok = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '!' || + c == '#' || c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' || c == '+' || c == '-' || + c == '.' || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; + if (!ok) + return false; + } + return true; +} + +inline bool isValidHttpHeaderValue(const String& value) { + for (size_t i = 0; i < value.length(); ++i) { + unsigned char c = static_cast(value.charAt(i)); + if (c == '\r' || c == '\n' || c == 0x00) + return false; + if ((c < 0x20 && c != '\t') || c == 0x7F) + return false; + } + return true; +} + #endif // HTTP_COMMON_H diff --git a/src/HttpRequest.cpp b/src/HttpRequest.cpp index 5f5a7e7..0055214 100644 --- a/src/HttpRequest.cpp +++ b/src/HttpRequest.cpp @@ -16,6 +16,8 @@ AsyncHttpRequest::AsyncHttpRequest(HttpMethod method, const String& url) AsyncHttpRequest::~AsyncHttpRequest() {} void AsyncHttpRequest::setHeader(const String& name, const String& value) { + if (!isValidHttpHeaderName(name) || !isValidHttpHeaderValue(value)) + return; // Check if header already exists and update it for (auto& header : _headers) { if (header.name.equalsIgnoreCase(name)) { @@ -137,14 +139,15 @@ void AsyncHttpRequest::addQueryParam(const String& key, const String& value) { String out; const char* hex = "0123456789ABCDEF"; for (size_t i = 0; i < in.length(); ++i) { - char c = in[i]; + uint8_t uc = static_cast(in[i]); + char c = static_cast(uc); if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.' || c == '~') { out += c; } else { out += '%'; - out += hex[(c >> 4) & 0xF]; - out += hex[c & 0xF]; + out += hex[(uc >> 4) & 0xF]; + out += hex[uc & 0xF]; } } return out; From b365c88cf521349966a7996099329d66381febf3 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 18 Dec 2025 12:11:12 +0100 Subject: [PATCH 2/6] add documentation , correction insecure mode , modif test cookies , correction security for cookies --- README.md | 25 +++- platformio.ini | 3 +- src/AsyncHttpClient.cpp | 226 +++++++++++++++++++++++++----- src/AsyncHttpClient.h | 32 ++++- src/HttpCommon.h | 12 +- test/test_cookies/test_main.cpp | 50 +++++++ test/test_redirects/test_main.cpp | 49 +++++++ 7 files changed, 352 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index f54fa77..cf95b75 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ An asynchronous HTTP client library for ESP32 microcontrollers, built on top of AsyncTCP. This library provides a simple and efficient way to make HTTP requests without blocking your main program execution. -> 🔐 **HTTPS Ready**: TLS/HTTPS is available via AsyncTCP + mbedTLS. Load a CA certificate or fingerprint before talking to real servers, or call `client.setTlsInsecure(true)` only for testing. See the *HTTPS / TLS configuration* section below. +> 🔐 **HTTPS Ready**: TLS/HTTPS is available via AsyncTCP + mbedTLS. Load a CA certificate or fingerprint before talking to real servers. `client.setTlsInsecure(true)` is intended for debug/pinning scenarios; fully insecure TLS requires an explicit build-time opt-in (see the *HTTPS / TLS configuration* section below). ## Features @@ -167,12 +167,26 @@ void setKeepAlive(bool enable, uint16_t idleMs = 5000); // Cookie jar helpers void clearCookies(); +void setAllowCookieDomainAttribute(bool enable); +void addAllowedCookieDomain(const char* domain); +void clearAllowedCookieDomains(); void setCookie(const char* name, const char* value, const char* path = "/", const char* domain = nullptr, bool secure = false); + +// Redirect header policy (when followRedirects is enabled) +void setRedirectHeaderPolicy(RedirectHeaderPolicy policy); +void addRedirectSafeHeader(const char* name); +void clearRedirectSafeHeaders(); ``` Cookies are captured automatically from `Set-Cookie` responses and replayed on matching hosts/paths; call -`clearCookies()` to wipe the jar or `setCookie()` to pre-seed entries manually. Keep-alive pooling is off by default; +`clearCookies()` to wipe the jar or `setCookie()` to pre-seed entries manually. + +By default, cookies set without a `Domain=` attribute are treated as **host-only** (sent only to the exact host that +set them). `Domain=` attributes that would widen scope are ignored unless explicitly allowlisted via +`setAllowCookieDomainAttribute(true)` + `addAllowedCookieDomain("example.com")`. + +Keep-alive pooling is off by default; enable it with `setKeepAlive(true, idleMs)` to reuse TCP/TLS connections for the same host/port (respecting server `Connection: close` requests). @@ -304,7 +318,8 @@ client.post("http://example.com/login", "user=demo", [](AsyncHttpResponse* respo - 301/302/303 responses switch to `GET` automatically (body dropped). - 307/308 keep the original method and body (stream bodies cannot be replayed automatically). -- Sensitive headers (`Authorization`, `Proxy-Authorization`) are stripped when the redirect crosses hosts. +- Cross-origin redirects default to forwarding only a small safe set of headers (e.g. `User-Agent`, `Accept`, etc.). + Use `setRedirectHeaderPolicy(...)` and `addRedirectSafeHeader(...)` if you need to forward additional headers. - Redirects are triggered as soon as the headers arrive; the client skips downloading any subsequent 3xx body data. See `examples/arduino/NoStoreToSD/NoStoreToSD.ino` for a full download example using `setNoStoreBody(true)` and a global `onBodyChunk` handler that streams chunked and non-chunked responses to an SD card. @@ -409,9 +424,11 @@ Highlights / limitations: `https://` URLs now use the built-in AsyncTCP + mbedTLS transport. Supply trust material before making real requests: -- `client.setTlsCACert(caPem)` — load a PEM CA chain (null-terminated). Mandatory unless using fingerprint pinning or `setTlsInsecure(true)`. +- `client.setTlsCACert(caPem)` — load a PEM CA chain (null-terminated). Mandatory unless using fingerprint pinning (see `setTlsFingerprint(...)`). - `client.setTlsClientCert(certPem, keyPem)` — optional mutual-TLS credentials (PEM). - `client.setTlsFingerprint("AA:BB:...")` — 32-byte SHA-256 fingerprint pinning. Validated after the handshake in addition to CA checks. +- `client.setTlsInsecure(true)` — skips CA validation. By default this is only effective when a fingerprint is configured (pinning). + To allow fully insecure TLS (MITM-unsafe) for local debugging, build with `-DASYNC_HTTP_ALLOW_INSECURE_TLS=1`. - `client.setTlsInsecure(true)` — disable CA validation (development only; do not ship with this enabled). - `client.setTlsHandshakeTimeout(ms)` — default is 12s; tune for slow networks. diff --git a/platformio.ini b/platformio.ini index 04457c2..a2d8541 100644 --- a/platformio.ini +++ b/platformio.ini @@ -18,7 +18,7 @@ framework = arduino platform_packages = framework-arduinoespressif32@^3 ; Run only Arduino-suitable tests on the device build -test_filter = test_parse_url, test_chunk_parse, test_keep_alive, test_cookies +test_filter = test_parse_url, test_chunk_parse, test_keep_alive, test_cookies, test_redirects test_ignore = test_urlparser_native lib_deps = https://github.com/ESP32Async/AsyncTCP.git @@ -77,4 +77,3 @@ test_ignore = test_parse_url, test_chunk_parse, test_redirects, test_cookies, te build_src_filter = -<*> + build_flags = -I test/test_urlparser_native - diff --git a/src/AsyncHttpClient.cpp b/src/AsyncHttpClient.cpp index 21543f7..addd269 100644 --- a/src/AsyncHttpClient.cpp +++ b/src/AsyncHttpClient.cpp @@ -40,6 +40,15 @@ static const char* kPublicSuffixes[] = {"com", "firebaseapp.com", "cloudfront.net"}; +static String normalizeDomainForStorage(const String& domain) { + String cleaned = domain; + cleaned.trim(); + if (cleaned.startsWith(".")) + cleaned.remove(0, 1); + cleaned.toLowerCase(); + return cleaned; +} + static bool equalsIgnoreCase(const String& a, const char* b) { size_t lenA = a.length(); size_t lenB = strlen(b); @@ -180,17 +189,17 @@ AsyncHttpClient::~AsyncHttpClient() { } #if defined(ARDUINO_ARCH_ESP32) && defined(ASYNC_HTTP_ENABLE_AUTOLOOP) -void AsyncHttpClient::lock() { +void AsyncHttpClient::lock() const { if (_reqMutex) xSemaphoreTakeRecursive(_reqMutex, portMAX_DELAY); } -void AsyncHttpClient::unlock() { +void AsyncHttpClient::unlock() const { if (_reqMutex) xSemaphoreGiveRecursive(_reqMutex); } #else -void AsyncHttpClient::lock() {} -void AsyncHttpClient::unlock() {} +void AsyncHttpClient::lock() const {} +void AsyncHttpClient::unlock() const {} #endif #if !ASYNC_TCP_HAS_TIMEOUT && defined(ARDUINO_ARCH_ESP32) && defined(ASYNC_HTTP_ENABLE_AUTOLOOP) @@ -282,6 +291,37 @@ void AsyncHttpClient::setFollowRedirects(bool enable, uint8_t maxHops) { unlock(); } +void AsyncHttpClient::setRedirectHeaderPolicy(RedirectHeaderPolicy policy) { + lock(); + _redirectHeaderPolicy = policy; + unlock(); +} + +void AsyncHttpClient::addRedirectSafeHeader(const char* name) { + if (!name || strlen(name) == 0) + return; + String headerName(name); + headerName.trim(); + if (headerName.length() == 0) + return; + headerName.toLowerCase(); + lock(); + for (const auto& existing : _redirectSafeHeaders) { + if (existing.equalsIgnoreCase(headerName)) { + unlock(); + return; + } + } + _redirectSafeHeaders.push_back(headerName); + unlock(); +} + +void AsyncHttpClient::clearRedirectSafeHeaders() { + lock(); + _redirectSafeHeaders.clear(); + unlock(); +} + void AsyncHttpClient::setMaxHeaderBytes(size_t maxBytes) { lock(); _maxHeaderBytes = maxBytes; @@ -363,6 +403,37 @@ void AsyncHttpClient::clearCookies() { unlock(); } +void AsyncHttpClient::setAllowCookieDomainAttribute(bool enable) { + lock(); + _allowCookieDomainAttribute = enable; + unlock(); +} + +void AsyncHttpClient::addAllowedCookieDomain(const char* domain) { + if (!domain || strlen(domain) == 0) + return; + String normalized = normalizeDomainForStorage(String(domain)); + if (normalized.length() == 0) + return; + if (normalized.indexOf('.') == -1) + return; + lock(); + for (const auto& existing : _allowedCookieDomains) { + if (existing.equalsIgnoreCase(normalized)) { + unlock(); + return; + } + } + _allowedCookieDomains.push_back(normalized); + unlock(); +} + +void AsyncHttpClient::clearAllowedCookieDomains() { + lock(); + _allowedCookieDomains.clear(); + unlock(); +} + void AsyncHttpClient::setCookie(const char* name, const char* value, const char* path, const char* domain, bool secure) { if (!name || strlen(name) == 0) @@ -379,6 +450,7 @@ void AsyncHttpClient::setCookie(const char* name, const char* value, const char* cookie.value = value ? String(value) : String(); cookie.path = (path && strlen(path) > 0) ? String(path) : String("/"); cookie.domain = domain ? String(domain) : String(); + cookie.hostOnly = false; // Manual cookies are treated as domain cookies (domain=="" means "any host"). cookie.secure = secure; cookie.createdAt = now; cookie.lastAccessAt = now; @@ -1061,6 +1133,13 @@ bool AsyncHttpClient::buildRedirectRequest(RequestContext* context, AsyncHttpReq newRequest->setNoStoreBody(context->request->getNoStoreBody()); bool sameOrigin = isSameOrigin(context->request, newRequest); + RedirectHeaderPolicy headerPolicy; + std::vector redirectSafeHeaders; + lock(); + headerPolicy = _redirectHeaderPolicy; + redirectSafeHeaders = _redirectSafeHeaders; + unlock(); + auto isCrossOriginSensitiveHeader = [](const String& name) { String lower = name; lower.toLowerCase(); @@ -1068,14 +1147,49 @@ bool AsyncHttpClient::buildRedirectRequest(RequestContext* context, AsyncHttpReq lower.equals("cookie2") || lower.startsWith("x-api-key") || lower.startsWith("x-auth-token") || lower.startsWith("x-access-token"); }; + auto isDefaultCrossOriginSafeHeader = [dropBody](const String& name) { + if (equalsIgnoreCase(name, "User-Agent")) + return true; + if (equalsIgnoreCase(name, "Accept")) + return true; + if (equalsIgnoreCase(name, "Accept-Encoding")) + return true; + if (equalsIgnoreCase(name, "Accept-Language")) + return true; + if (!dropBody && equalsIgnoreCase(name, "Content-Type")) + return true; + return false; + }; + auto isAllowlistedForCrossOrigin = [&redirectSafeHeaders](const String& name) { + String lower = name; + lower.toLowerCase(); + for (const auto& allowed : redirectSafeHeaders) { + if (allowed.equalsIgnoreCase(lower)) + return true; + } + return false; + }; const auto& headers = context->request->getHeaders(); for (const auto& hdr : headers) { if (hdr.name.equalsIgnoreCase("Content-Length")) continue; - if (dropBody && hdr.name.equalsIgnoreCase("Content-Type")) + // Always rebuild cookies for the redirected request from the cookie jar (avoids duplicates and leaks). + if (hdr.name.equalsIgnoreCase("Cookie") || hdr.name.equalsIgnoreCase("Cookie2")) + continue; + // Prevent callers from pinning an old Host header across redirects. + if (hdr.name.equalsIgnoreCase("Host")) continue; - if (!sameOrigin && isCrossOriginSensitiveHeader(hdr.name)) + if (dropBody && hdr.name.equalsIgnoreCase("Content-Type")) continue; + if (!sameOrigin) { + if (headerPolicy == RedirectHeaderPolicy::kLegacyDropSensitiveOnly) { + if (isCrossOriginSensitiveHeader(hdr.name)) + continue; + } else if (headerPolicy == RedirectHeaderPolicy::kDropAllCrossOrigin) { + if (!isDefaultCrossOriginSafeHeader(hdr.name) && !isAllowlistedForCrossOrigin(hdr.name)) + continue; + } + } newRequest->setHeader(hdr.name, hdr.value); } @@ -1248,12 +1362,32 @@ bool AsyncHttpClient::shouldEnforceBodyLimit(RequestContext* context) { } AsyncHttpTLSConfig AsyncHttpClient::resolveTlsConfig(const AsyncHttpRequest* request) const { - AsyncHttpTLSConfig cfg = _defaultTlsConfig; - if (!request || !request->hasTlsConfig()) + AsyncHttpTLSConfig cfg; + lock(); + cfg = _defaultTlsConfig; + unlock(); + + auto sanitize = [](AsyncHttpTLSConfig* c) { + if (!c) + return; + if (c->handshakeTimeoutMs == 0) + c->handshakeTimeoutMs = 12000; +#if !ASYNC_HTTP_ALLOW_INSECURE_TLS + // Allow skipping CA validation only when pinning is configured. + if (c->insecure && c->fingerprint.length() == 0) + c->insecure = false; +#endif + }; + + if (!request || !request->hasTlsConfig()) { + sanitize(&cfg); return cfg; + } const AsyncHttpTLSConfig* overrideCfg = request->getTlsConfig(); - if (!overrideCfg) + if (!overrideCfg) { + sanitize(&cfg); return cfg; + } if (overrideCfg->caCert.length() > 0) cfg.caCert = overrideCfg->caCert; if (overrideCfg->clientCert.length() > 0) @@ -1265,8 +1399,7 @@ AsyncHttpTLSConfig AsyncHttpClient::resolveTlsConfig(const AsyncHttpRequest* req cfg.insecure = overrideCfg->insecure; if (overrideCfg->handshakeTimeoutMs > 0) cfg.handshakeTimeoutMs = overrideCfg->handshakeTimeoutMs; - if (cfg.handshakeTimeoutMs == 0) - cfg.handshakeTimeoutMs = _defaultTlsConfig.handshakeTimeoutMs; + sanitize(&cfg); return cfg; } @@ -1464,34 +1597,58 @@ static bool isPublicSuffix(const String& domain) { return false; } -bool AsyncHttpClient::normalizeCookieDomain(String& domain, const String& host, bool domainAttributeProvided) const { - String cleaned = domain; - cleaned.trim(); - if (cleaned.startsWith(".")) - cleaned.remove(0, 1); +bool AsyncHttpClient::normalizeCookieDomain(String& domain, const String& host, bool domainAttributeProvided, + bool* outHostOnly) const { + if (outHostOnly) + *outHostOnly = true; + String hostLower = host; + hostLower.toLowerCase(); + String cleaned = normalizeDomainForStorage(domain); + // No Domain= attribute (or empty) => host-only cookie. if (!domainAttributeProvided || cleaned.length() == 0) { - domain = host; + domain = hostLower; + if (outHostOnly) + *outHostOnly = true; return true; } - String hostLower = host; - hostLower.toLowerCase(); - cleaned.toLowerCase(); - + // Reject Domain= on IP literals and unrelated domains. if (isIpLiteral(hostLower)) return false; if (!domainMatches(cleaned, hostLower)) return false; - // Heuristic public-suffix guard: require both host and domain to have at least one dot - if (hostLower.indexOf('.') == -1) - return false; - if (cleaned.indexOf('.') == -1) - return false; - if (isPublicSuffix(cleaned)) - return false; - domain = cleaned; + // Public suffix and "TLD-like" Domain= attributes are ignored (stored as host-only instead). + // This avoids broad cookie scope even when the embedded public-suffix list is incomplete. + if (hostLower.indexOf('.') == -1 || cleaned.indexOf('.') == -1 || isPublicSuffix(cleaned)) { + domain = hostLower; + if (outHostOnly) + *outHostOnly = true; + return true; + } + + bool allowDomainCookie = false; + lock(); + if (_allowCookieDomainAttribute) { + for (const auto& allowedDomain : _allowedCookieDomains) { + if (allowedDomain.equalsIgnoreCase(cleaned)) { + allowDomainCookie = true; + break; + } + } + } + unlock(); + + if (allowDomainCookie) { + domain = cleaned; + if (outHostOnly) + *outHostOnly = false; + } else { + domain = hostLower; + if (outHostOnly) + *outHostOnly = true; + } return true; } @@ -1535,8 +1692,13 @@ bool AsyncHttpClient::cookieMatchesRequest(const StoredCookie& cookie, const Asy return false; if (cookie.secure && !request->isSecure()) return false; - if (!domainMatches(cookie.domain, request->getHost())) - return false; + if (cookie.hostOnly) { + if (!request->getHost().equalsIgnoreCase(cookie.domain)) + return false; + } else { + if (!domainMatches(cookie.domain, request->getHost())) + return false; + } if (!pathMatches(cookie.path, request->getPath())) return false; return !cookie.value.isEmpty(); @@ -1719,7 +1881,7 @@ void AsyncHttpClient::storeResponseCookie(const AsyncHttpRequest* request, const pos = next; } - if (!normalizeCookieDomain(cookie.domain, request->getHost(), domainAttributeProvided)) + if (!normalizeCookieDomain(cookie.domain, request->getHost(), domainAttributeProvided, &cookie.hostOnly)) return; if (!cookie.path.startsWith("/")) cookie.path = "/" + cookie.path; diff --git a/src/AsyncHttpClient.h b/src/AsyncHttpClient.h index 84a73e0..6edec2a 100644 --- a/src/AsyncHttpClient.h +++ b/src/AsyncHttpClient.h @@ -25,6 +25,15 @@ class AsyncHttpClient { typedef std::function BodyChunkCallback; // global // (Per-request chunk callback removed for API simplification) + enum class RedirectHeaderPolicy { + // Cross-origin redirects forward only a small safe set (+ any added via allowlist). + kDropAllCrossOrigin, + // Legacy heuristic-based filtering on cross-origin redirects. + kLegacyDropSensitiveOnly, + // Preserve all request headers across redirects (unsafe). + kPreserveAll + }; + AsyncHttpClient(); ~AsyncHttpClient(); @@ -82,8 +91,17 @@ class AsyncHttpClient { return _defaultTlsConfig; } void clearCookies(); + // By default, Domain= attributes are rejected unless they exactly match the request host. + // To allow a server to set cookies for a parent domain (e.g., Domain=example.com from api.example.com), + // enable this and add allowed parent domains explicitly. + void setAllowCookieDomainAttribute(bool enable); + void addAllowedCookieDomain(const char* domain); + void clearAllowedCookieDomains(); void setCookie(const char* name, const char* value, const char* path = "/", const char* domain = nullptr, bool secure = false); + void setRedirectHeaderPolicy(RedirectHeaderPolicy policy); + void addRedirectSafeHeader(const char* name); + void clearRedirectSafeHeaders(); // Advanced request method uint32_t request(AsyncHttpRequest* request, SuccessCallback onSuccess, ErrorCallback onError = nullptr); @@ -109,8 +127,8 @@ class AsyncHttpClient { private: // Lightweight locking helpers (no-op unless ESP32 auto-loop task is enabled) - void lock(); - void unlock(); + void lock() const; + void unlock() const; // Internal auto-loop task for fallback timeout mode #if !ASYNC_TCP_HAS_TIMEOUT && defined(ARDUINO_ARCH_ESP32) && defined(ASYNC_HTTP_ENABLE_AUTOLOOP) @@ -180,6 +198,10 @@ class AsyncHttpClient { AsyncHttpTLSConfig _defaultTlsConfig; bool _keepAliveEnabled = false; uint32_t _keepAliveIdleMs = 5000; + bool _allowCookieDomainAttribute = false; + std::vector _allowedCookieDomains; + RedirectHeaderPolicy _redirectHeaderPolicy = RedirectHeaderPolicy::kDropAllCrossOrigin; + std::vector _redirectSafeHeaders; struct PooledConnection { AsyncTransport* transport = nullptr; @@ -192,7 +214,7 @@ class AsyncHttpClient { std::vector _idleConnections; #if defined(ARDUINO_ARCH_ESP32) && defined(ASYNC_HTTP_ENABLE_AUTOLOOP) - SemaphoreHandle_t _reqMutex = nullptr; // recursive mutex + mutable SemaphoreHandle_t _reqMutex = nullptr; // recursive mutex #endif // Internal methods @@ -231,6 +253,7 @@ class AsyncHttpClient { String value; String domain; String path; + bool hostOnly = true; bool secure = false; int64_t expiresAt = -1; // -1 means no expiration set int64_t createdAt = 0; @@ -245,7 +268,8 @@ class AsyncHttpClient { void evictOneCookieLocked(); bool domainMatches(const String& cookieDomain, const String& host) const; bool pathMatches(const String& cookiePath, const String& requestPath) const; - bool normalizeCookieDomain(String& domain, const String& host, bool domainAttributeProvided) const; + bool normalizeCookieDomain(String& domain, const String& host, bool domainAttributeProvided, + bool* outHostOnly) const; bool isIpLiteral(const String& host) const; public: diff --git a/src/HttpCommon.h b/src/HttpCommon.h index b9a94e5..bc46ae6 100644 --- a/src/HttpCommon.h +++ b/src/HttpCommon.h @@ -9,6 +9,12 @@ 0 // 0 = only set Accept-Encoding header (no inflation). 1 = future: enable minimal gzip inflate. #endif +// Security: allow enabling insecure TLS (skips CA validation) only when explicitly opted in at build time. +// Recommended alternatives: `setTlsCACert(...)` or `setTlsFingerprint(...)` (pinning). +#ifndef ASYNC_HTTP_ALLOW_INSECURE_TLS +#define ASYNC_HTTP_ALLOW_INSECURE_TLS 0 +#endif + // Library version (single source of truth inside code). Keep in sync with library.json and library.properties. #ifndef ESP_ASYNC_WEB_CLIENT_VERSION #define ESP_ASYNC_WEB_CLIENT_VERSION "1.1.4" @@ -99,9 +105,9 @@ inline bool isValidHttpHeaderName(const String& name) { return false; for (size_t i = 0; i < name.length(); ++i) { unsigned char c = static_cast(name.charAt(i)); - bool ok = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '!' || - c == '#' || c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' || c == '+' || c == '-' || - c == '.' || c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; + bool ok = (c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '!' || c == '#' || + c == '$' || c == '%' || c == '&' || c == '\'' || c == '*' || c == '+' || c == '-' || c == '.' || + c == '^' || c == '_' || c == '`' || c == '|' || c == '~'; if (!ok) return false; } diff --git a/test/test_cookies/test_main.cpp b/test/test_cookies/test_main.cpp index db6cef8..2db2c5b 100644 --- a/test/test_cookies/test_main.cpp +++ b/test/test_cookies/test_main.cpp @@ -71,6 +71,53 @@ static void test_rejects_mismatched_domain_attribute() { TEST_ASSERT_EQUAL(0, (int)client._cookies.size()); } +static void test_rejects_parent_domain_attribute_by_default() { + AsyncHttpClient client; + AsyncHttpRequest req(HTTP_METHOD_GET, "http://api.example.com/"); + + client.storeResponseCookie(&req, "s=1; Domain=example.com; Path=/"); + AsyncHttpRequest sameHost(HTTP_METHOD_GET, "http://api.example.com/"); + client.applyCookies(&sameHost); + TEST_ASSERT_EQUAL_STRING("s=1", sameHost.getHeader("Cookie").c_str()); + + AsyncHttpRequest otherSubdomain(HTTP_METHOD_GET, "http://foo.example.com/"); + client.applyCookies(&otherSubdomain); + TEST_ASSERT_TRUE(otherSubdomain.getHeader("Cookie").isEmpty()); + TEST_ASSERT_EQUAL(1, (int)client._cookies.size()); +} + +static void test_allows_allowlisted_parent_domain_attribute() { + AsyncHttpClient client; + client.setAllowCookieDomainAttribute(true); + client.addAllowedCookieDomain("example.com"); + + AsyncHttpRequest req(HTTP_METHOD_GET, "http://api.example.com/"); + client.storeResponseCookie(&req, "s=1; Domain=example.com; Path=/"); + + AsyncHttpRequest follow(HTTP_METHOD_GET, "http://foo.example.com/"); + client.applyCookies(&follow); + TEST_ASSERT_EQUAL_STRING("s=1", follow.getHeader("Cookie").c_str()); + TEST_ASSERT_EQUAL(1, (int)client._cookies.size()); +} + +static void test_rejects_public_suffix_even_if_allowlisted() { + AsyncHttpClient client; + client.setAllowCookieDomainAttribute(true); + client.addAllowedCookieDomain("co.uk"); + + AsyncHttpRequest req(HTTP_METHOD_GET, "http://a.co.uk/"); + client.storeResponseCookie(&req, "s=1; Domain=co.uk; Path=/"); + + AsyncHttpRequest sameHost(HTTP_METHOD_GET, "http://a.co.uk/"); + client.applyCookies(&sameHost); + TEST_ASSERT_EQUAL_STRING("s=1", sameHost.getHeader("Cookie").c_str()); + + AsyncHttpRequest otherSubdomain(HTTP_METHOD_GET, "http://b.co.uk/"); + client.applyCookies(&otherSubdomain); + TEST_ASSERT_TRUE(otherSubdomain.getHeader("Cookie").isEmpty()); + TEST_ASSERT_EQUAL(1, (int)client._cookies.size()); +} + static void test_cookie_path_matching_rfc6265_rule() { AsyncHttpClient client; AsyncHttpRequest req(HTTP_METHOD_GET, "http://example.com/administrator"); @@ -154,6 +201,9 @@ int runUnityTests() { RUN_TEST(test_max_age_removes_cookie); RUN_TEST(test_clear_and_public_set_cookie_api); RUN_TEST(test_rejects_mismatched_domain_attribute); + RUN_TEST(test_rejects_parent_domain_attribute_by_default); + RUN_TEST(test_allows_allowlisted_parent_domain_attribute); + RUN_TEST(test_rejects_public_suffix_even_if_allowlisted); RUN_TEST(test_cookie_path_matching_rfc6265_rule); RUN_TEST(test_expires_and_max_age_enforcement); RUN_TEST(test_cookie_jar_eviction_is_lru_session_then_scope); diff --git a/test/test_redirects/test_main.cpp b/test/test_redirects/test_main.cpp index c41fcee..af9eca8 100644 --- a/test/test_redirects/test_main.cpp +++ b/test/test_redirects/test_main.cpp @@ -85,6 +85,53 @@ static void test_redirect_cross_host_preserve_method_strip_auth() { cleanupContext(ctx); } +static void test_redirect_cross_host_drops_unknown_headers_by_default() { + AsyncHttpClient client; + client.setFollowRedirects(true, 3); + auto ctx = makeRedirectContext(HTTP_METHOD_GET, "http://example.com/a"); + ctx->request->setHeader("X-Custom-Token", "secret"); + ctx->request->setHeader("Accept", "application/json"); + + ctx->response->setStatusCode(302); + ctx->response->setHeader("Location", "http://other.example.com/b"); + + AsyncHttpRequest* newReq = nullptr; + HttpClientError err = CONNECTION_FAILED; + String message; + bool decision = client.buildRedirectRequest(ctx, &newReq, &err, &message); + + TEST_ASSERT_TRUE(decision); + TEST_ASSERT_NOT_NULL(newReq); + TEST_ASSERT_TRUE(newReq->getHeader("X-Custom-Token").isEmpty()); + TEST_ASSERT_EQUAL_STRING("application/json", newReq->getHeader("Accept").c_str()); + + delete newReq; + cleanupContext(ctx); +} + +static void test_redirect_cross_host_can_allowlist_header() { + AsyncHttpClient client; + client.setFollowRedirects(true, 3); + client.addRedirectSafeHeader("X-Custom-Token"); + auto ctx = makeRedirectContext(HTTP_METHOD_GET, "http://example.com/a"); + ctx->request->setHeader("X-Custom-Token", "secret"); + + ctx->response->setStatusCode(302); + ctx->response->setHeader("Location", "http://other.example.com/b"); + + AsyncHttpRequest* newReq = nullptr; + HttpClientError err = CONNECTION_FAILED; + String message; + bool decision = client.buildRedirectRequest(ctx, &newReq, &err, &message); + + TEST_ASSERT_TRUE(decision); + TEST_ASSERT_NOT_NULL(newReq); + TEST_ASSERT_EQUAL_STRING("secret", newReq->getHeader("X-Custom-Token").c_str()); + + delete newReq; + cleanupContext(ctx); +} + static void test_redirect_too_many_hops() { AsyncHttpClient client; client.setFollowRedirects(true, 2); @@ -231,6 +278,8 @@ void setup() { UNITY_BEGIN(); RUN_TEST(test_redirect_same_host_get); RUN_TEST(test_redirect_cross_host_preserve_method_strip_auth); + RUN_TEST(test_redirect_cross_host_drops_unknown_headers_by_default); + RUN_TEST(test_redirect_cross_host_can_allowlist_header); RUN_TEST(test_redirect_too_many_hops); RUN_TEST(test_redirect_to_https_supported); RUN_TEST(test_header_limit_triggers_error); From 5f19e37b3311c80839af8bdd41dae60674f3da30 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 18 Dec 2025 12:27:03 +0100 Subject: [PATCH 3/6] delete public suffix --- src/AsyncHttpClient.cpp | 35 +-------------------------------- test/test_cookies/test_main.cpp | 19 ------------------ 2 files changed, 1 insertion(+), 53 deletions(-) diff --git a/src/AsyncHttpClient.cpp b/src/AsyncHttpClient.cpp index addd269..344ef4c 100644 --- a/src/AsyncHttpClient.cpp +++ b/src/AsyncHttpClient.cpp @@ -18,27 +18,6 @@ static constexpr size_t kDefaultMaxHeaderBytes = 2800; // ~2.8 KiB static constexpr size_t kDefaultMaxBodyBytes = 8192; // 8 KiB static constexpr size_t kMaxCookieCount = 16; static constexpr size_t kMaxCookieBytes = 4096; -static const char* kPublicSuffixes[] = {"com", - "net", - "org", - "gov", - "edu", - "mil", - "int", - "co.uk", - "ac.uk", - "gov.uk", - "uk", - "io", - "co", - "app", - "dev", - "github.io", - "web.app", - "pages.dev", - "vercel.app", - "firebaseapp.com", - "cloudfront.net"}; static String normalizeDomainForStorage(const String& domain) { String cleaned = domain; @@ -1585,18 +1564,6 @@ bool AsyncHttpClient::isIpLiteral(const String& host) const { return hasColon || hasDot; } -static bool isPublicSuffix(const String& domain) { - if (domain.length() == 0) - return false; - String lower = domain; - lower.toLowerCase(); - for (auto suffix : kPublicSuffixes) { - if (lower.equals(suffix)) - return true; - } - return false; -} - bool AsyncHttpClient::normalizeCookieDomain(String& domain, const String& host, bool domainAttributeProvided, bool* outHostOnly) const { if (outHostOnly) @@ -1621,7 +1588,7 @@ bool AsyncHttpClient::normalizeCookieDomain(String& domain, const String& host, // Public suffix and "TLD-like" Domain= attributes are ignored (stored as host-only instead). // This avoids broad cookie scope even when the embedded public-suffix list is incomplete. - if (hostLower.indexOf('.') == -1 || cleaned.indexOf('.') == -1 || isPublicSuffix(cleaned)) { + if (hostLower.indexOf('.') == -1 || cleaned.indexOf('.') == -1) { domain = hostLower; if (outHostOnly) *outHostOnly = true; diff --git a/test/test_cookies/test_main.cpp b/test/test_cookies/test_main.cpp index 2db2c5b..a7c736d 100644 --- a/test/test_cookies/test_main.cpp +++ b/test/test_cookies/test_main.cpp @@ -100,24 +100,6 @@ static void test_allows_allowlisted_parent_domain_attribute() { TEST_ASSERT_EQUAL(1, (int)client._cookies.size()); } -static void test_rejects_public_suffix_even_if_allowlisted() { - AsyncHttpClient client; - client.setAllowCookieDomainAttribute(true); - client.addAllowedCookieDomain("co.uk"); - - AsyncHttpRequest req(HTTP_METHOD_GET, "http://a.co.uk/"); - client.storeResponseCookie(&req, "s=1; Domain=co.uk; Path=/"); - - AsyncHttpRequest sameHost(HTTP_METHOD_GET, "http://a.co.uk/"); - client.applyCookies(&sameHost); - TEST_ASSERT_EQUAL_STRING("s=1", sameHost.getHeader("Cookie").c_str()); - - AsyncHttpRequest otherSubdomain(HTTP_METHOD_GET, "http://b.co.uk/"); - client.applyCookies(&otherSubdomain); - TEST_ASSERT_TRUE(otherSubdomain.getHeader("Cookie").isEmpty()); - TEST_ASSERT_EQUAL(1, (int)client._cookies.size()); -} - static void test_cookie_path_matching_rfc6265_rule() { AsyncHttpClient client; AsyncHttpRequest req(HTTP_METHOD_GET, "http://example.com/administrator"); @@ -203,7 +185,6 @@ int runUnityTests() { RUN_TEST(test_rejects_mismatched_domain_attribute); RUN_TEST(test_rejects_parent_domain_attribute_by_default); RUN_TEST(test_allows_allowlisted_parent_domain_attribute); - RUN_TEST(test_rejects_public_suffix_even_if_allowlisted); RUN_TEST(test_cookie_path_matching_rfc6265_rule); RUN_TEST(test_expires_and_max_age_enforcement); RUN_TEST(test_cookie_jar_eviction_is_lru_session_then_scope); From e469f4b5047257d43261c17ce530928d2d797ba4 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 18 Dec 2025 12:49:36 +0100 Subject: [PATCH 4/6] add gzip --- README.md | 18 +- platformio.ini | 17 +- src/AsyncHttpClient.cpp | 206 +++++- src/AsyncHttpClient.h | 16 +- src/GzipDecoder.cpp | 354 ++++++++++ src/GzipDecoder.h | 81 +++ src/HttpCommon.h | 7 +- src/third_party/miniz/LICENSE | 22 + src/third_party/miniz/miniz.h | 21 + src/third_party/miniz/miniz_common.h | 97 +++ src/third_party/miniz/miniz_export.h | 10 + src/third_party/miniz/miniz_tinfl.c | 770 +++++++++++++++++++++ src/third_party/miniz/miniz_tinfl.h | 150 ++++ test/test_gzip_decode_native/test_main.cpp | 122 ++++ 14 files changed, 1847 insertions(+), 44 deletions(-) create mode 100644 src/GzipDecoder.cpp create mode 100644 src/GzipDecoder.h create mode 100644 src/third_party/miniz/LICENSE create mode 100644 src/third_party/miniz/miniz.h create mode 100644 src/third_party/miniz/miniz_common.h create mode 100644 src/third_party/miniz/miniz_export.h create mode 100644 src/third_party/miniz/miniz_tinfl.c create mode 100644 src/third_party/miniz/miniz_tinfl.h create mode 100644 test/test_gzip_decode_native/test_main.cpp diff --git a/README.md b/README.md index cf95b75..4edf2e0 100644 --- a/README.md +++ b/README.md @@ -468,7 +468,7 @@ Common HTTPS errors: - Redirects disabled by default; opt-in via `client.setFollowRedirects(...)` - No long-lived keep-alive: default header `Connection: close`; no connection reuse currently. - Manual timeout loop required if AsyncTCP version lacks `setTimeout` (call `client.loop()` in `loop()`). -- No specific content-encoding handling (gzip/deflate ignored if sent). +- No general content-encoding handling (br/deflate not supported); optional `gzip` decode is available via `ASYNC_HTTP_ENABLE_GZIP_DECODE`. ## Object lifecycle / Ownership @@ -505,6 +505,7 @@ Single authoritative list (kept in sync with `HttpCommon.h`): | -15 | TLS_CERT_INVALID | TLS certificate validation failed | | -16 | TLS_FINGERPRINT_MISMATCH | TLS fingerprint pinning rejected the peer certificate | | -17 | TLS_HANDSHAKE_TIMEOUT | TLS handshake exceeded the configured timeout | +| -18 | GZIP_DECODE_FAILED | Failed to decode gzip body (`Content-Encoding: gzip`) | | >0 | (AsyncTCP) | Not used: transport errors are mapped to CONNECTION_FAILED | Example mapping in a callback: @@ -577,7 +578,7 @@ Contributions are welcome! Please feel free to submit a Pull Request. - Global body chunk callback (per-request callback removed for API simplicity) - Basic Auth helper (request->setBasicAuth) - Query param builder (addQueryParam/finalizeQueryParams) -- Optional Accept-Encoding: gzip (no automatic decompression yet) +- Optional Accept-Encoding: gzip (+ optional transparent decode via `ASYNC_HTTP_ENABLE_GZIP_DECODE`) - Separate connect timeout and total timeout - Optional request queue limiting parallel connections (setMaxParallel) - Soft response buffering guard (`setMaxBodySize`) to fail fast on oversized payloads @@ -586,11 +587,16 @@ Contributions are welcome! Please feel free to submit a Pull Request. ### Gzip / Compression -Current: only the `Accept-Encoding: gzip` header can be added via `enableGzipAcceptEncoding(true)`. -The library DOES NOT yet decompress gzip payloads. If you don't want compressed responses, simply don't enable the header. +Default: only the `Accept-Encoding: gzip` header can be added via `enableGzipAcceptEncoding(true)`. -Important: calling `enableGzipAcceptEncoding(false)` does not remove the header if it was already added earlier on the same request instance. Create a new request without enabling it to avoid sending the header. -A future optional flag (`ASYNC_HTTP_ENABLE_GZIP_DECODE`) may add a tiny inflater (miniz/zlib) after flash/RAM impact is evaluated. +Optional decode: build with `-DASYNC_HTTP_ENABLE_GZIP_DECODE=1` to transparently inflate `Content-Encoding: gzip` responses (both in-memory body and `client.onBodyChunk(...)` stream). + +Notes: + +- If you don't want compressed responses, simply don't enable the header. +- `enableGzipAcceptEncoding(false)` removes `Accept-Encoding` from the request's header list (or call `request.removeHeader("Accept-Encoding")`). +- `Content-Length` (when present) refers to the *compressed* payload size; completion detection still follows the wire length. +- RAM impact: enabling gzip decode allocates an internal 32KB sliding window per active gzip-decoded response (plus small state). ### HTTPS quick reference diff --git a/platformio.ini b/platformio.ini index a2d8541..28cf349 100644 --- a/platformio.ini +++ b/platformio.ini @@ -38,6 +38,20 @@ build_src_filter = -<*> +<../src/> +<../test/compile_test_internal/compile_test. lib_deps = https://github.com/ESP32Async/AsyncTCP.git +[env:compile_test_gzip] +platform = espressif32 +board = esp32dev +framework = arduino +platform_packages = + framework-arduinoespressif32@^3 +build_flags = + ${env.build_flags} + -DCOMPILE_TEST_ONLY + -DASYNC_HTTP_ENABLE_GZIP_DECODE=1 +build_src_filter = -<*> +<../src/> +<../test/compile_test_internal/compile_test.cpp> +lib_deps = + https://github.com/ESP32Async/AsyncTCP.git + # Environment for testing with different AsyncTCP versions [env:esp32dev_asynctcp_dev] @@ -74,6 +88,7 @@ platform = native ; Only build the standalone URL parser for host tests to avoid Arduino deps ; Do not compile Arduino-based tests in native test_ignore = test_parse_url, test_chunk_parse, test_redirects, test_cookies, test_keep_alive -build_src_filter = -<*> + +build_src_filter = -<*> + + + build_flags = -I test/test_urlparser_native + -I src diff --git a/src/AsyncHttpClient.cpp b/src/AsyncHttpClient.cpp index 344ef4c..f59f885 100644 --- a/src/AsyncHttpClient.cpp +++ b/src/AsyncHttpClient.cpp @@ -651,20 +651,99 @@ void AsyncHttpClient::handleConnect(RequestContext* context) { } void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len) { - bool storeBody = context && context->request && !context->request->getNoStoreBody(); - bool bufferThisChunk = context && (!context->headersComplete || context->chunked); + if (!context) + return; + bool storeBody = context->request && !context->request->getNoStoreBody(); + bool bufferThisChunk = (!context->headersComplete || context->chunked); if (bufferThisChunk) context->responseBuffer.concat(data, len); bool enforceLimit = shouldEnforceBodyLimit(context); - auto wouldExceedLimit = [&](size_t incoming) -> bool { + auto wouldExceedBodyLimit = [&](size_t incoming) -> bool { if (!enforceLimit) return false; - size_t current = context->receivedContentLength; + size_t current = context->receivedBodyLength; if (current >= _maxBodySize) return true; return incoming > (_maxBodySize - current); }; + auto emitBodyBytes = [&](const char* out, size_t outLen) -> bool { + if (!out || outLen == 0) + return true; + if (wouldExceedBodyLimit(outLen)) { + triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); + return false; + } + if (storeBody) { + context->response->appendBody(out, outLen); + } + context->receivedBodyLength += outLen; + auto cb = _bodyChunkCallback; + if (cb) + cb(out, outLen, false); + return true; + }; + + auto deliverWireBytes = [&](const char* wire, size_t wireLen) -> bool { + if (wireLen == 0) + return true; + context->receivedContentLength += wireLen; +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + if (context->gzipDecodeActive) { + size_t offset = 0; + while (offset < wireLen) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + size_t consumed = 0; + GzipDecoder::Result r = context->gzipDecoder.write(reinterpret_cast(wire + offset), + wireLen - offset, &consumed, &outPtr, &outLen, true); + if (outLen > 0) { + if (!emitBodyBytes(reinterpret_cast(outPtr), outLen)) + return false; + } + if (r == GzipDecoder::Result::kError) { + triggerError(context, GZIP_DECODE_FAILED, context->gzipDecoder.lastError()); + return false; + } + offset += consumed; + if (consumed == 0 && outLen == 0) { + triggerError(context, GZIP_DECODE_FAILED, "Gzip decoder stalled"); + return false; + } + if (r == GzipDecoder::Result::kNeedMoreInput && offset >= wireLen) { + break; + } + } + return true; + } +#endif + return emitBodyBytes(wire, wireLen); + }; + + auto finalizeDecoding = [&]() -> bool { +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + if (!context->gzipDecodeActive) + return true; + for (;;) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + GzipDecoder::Result r = context->gzipDecoder.finish(&outPtr, &outLen); + if (outLen > 0) { + if (!emitBodyBytes(reinterpret_cast(outPtr), outLen)) + return false; + } + if (r == GzipDecoder::Result::kDone) + return true; + if (r == GzipDecoder::Result::kOk) + continue; + triggerError(context, GZIP_DECODE_FAILED, context->gzipDecoder.lastError()); + return false; + } +#else + return true; +#endif + }; + if (!context->headersComplete) { int headerEnd = context->responseBuffer.indexOf("\r\n\r\n"); if (_maxHeaderBytes > 0) { @@ -679,12 +758,17 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len String headerData = context->responseBuffer.substring(0, headerEnd); if (parseResponseHeaders(context, headerData)) { context->headersComplete = true; - if (enforceLimit && context->expectedContentLength > 0 && +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + bool gzipActive = context->gzipDecodeActive; +#else + bool gzipActive = false; +#endif + if (enforceLimit && !gzipActive && context->expectedContentLength > 0 && context->expectedContentLength > _maxBodySize) { triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); return; } - if (storeBody && context->expectedContentLength > 0 && !context->chunked && + if (storeBody && !gzipActive && context->expectedContentLength > 0 && !context->chunked && (!enforceLimit || context->expectedContentLength <= _maxBodySize)) { context->response->reserveBody(context->expectedContentLength); } @@ -693,17 +777,12 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len return; if (!context->chunked && context->responseBuffer.length() > 0) { size_t incomingLen = context->responseBuffer.length(); - if (wouldExceedLimit(incomingLen)) { + if (!gzipActive && wouldExceedBodyLimit(incomingLen)) { triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); return; } - if (storeBody) { - context->response->appendBody(context->responseBuffer.c_str(), incomingLen); - } - context->receivedContentLength += incomingLen; - auto cb = _bodyChunkCallback; - if (cb) - cb(context->responseBuffer.c_str(), incomingLen, false); + if (!deliverWireBytes(context->responseBuffer.c_str(), incomingLen)) + return; context->responseBuffer = ""; } } else { @@ -712,17 +791,17 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len } } } else if (!context->chunked) { - if (wouldExceedLimit(len)) { +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + bool gzipActive = context->gzipDecodeActive; +#else + bool gzipActive = false; +#endif + if (!gzipActive && wouldExceedBodyLimit(len)) { triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); return; } - if (storeBody) { - context->response->appendBody(data, len); - } - context->receivedContentLength += len; - auto cb2 = _bodyChunkCallback; - if (cb2) - cb2(data, len, false); + if (!deliverWireBytes(data, len)) + return; } while (context->headersComplete && context->chunked && !context->chunkedComplete) { @@ -796,7 +875,12 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len triggerError(context, CHUNKED_DECODE_FAILED, "Chunk size parse error"); return; } - if (chunkSize > 0 && wouldExceedLimit(chunkSize)) { +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + bool gzipActive = context->gzipDecodeActive; +#else + bool gzipActive = false; +#endif + if (!gzipActive && chunkSize > 0 && wouldExceedBodyLimit(chunkSize)) { triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); return; } @@ -817,18 +901,9 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len return; } size_t chunkLen = context->currentChunkRemaining; - if (wouldExceedLimit(chunkLen)) { - triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); - return; - } const char* chunkPtr = context->responseBuffer.c_str(); - if (storeBody) { - context->response->appendBody(chunkPtr, chunkLen); - } - context->receivedContentLength += chunkLen; - auto cb3 = _bodyChunkCallback; - if (cb3) - cb3(chunkPtr, chunkLen, false); + if (!deliverWireBytes(chunkPtr, chunkLen)) + return; context->responseBuffer.remove(0, needed); context->currentChunkRemaining = 0; } @@ -841,6 +916,8 @@ void AsyncHttpClient::handleData(RequestContext* context, char* data, size_t len context->receivedContentLength >= context->expectedContentLength) complete = true; if (complete) { + if (!finalizeDecoding()) + return; processResponse(context); } } @@ -893,6 +970,51 @@ void AsyncHttpClient::handleDisconnect(RequestContext* context) { triggerError(context, CONNECTION_CLOSED_MID_BODY, "Truncated response"); return; } +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + if (context->gzipDecodeActive) { + bool storeBody = context->request && !context->request->getNoStoreBody(); + bool enforceLimit = shouldEnforceBodyLimit(context); + auto wouldExceedBodyLimit = [&](size_t incoming) -> bool { + if (!enforceLimit) + return false; + size_t current = context->receivedBodyLength; + if (current >= _maxBodySize) + return true; + return incoming > (_maxBodySize - current); + }; + auto emitBodyBytes = [&](const char* out, size_t outLen) -> bool { + if (!out || outLen == 0) + return true; + if (wouldExceedBodyLimit(outLen)) { + triggerError(context, MAX_BODY_SIZE_EXCEEDED, "Body exceeds configured maximum"); + return false; + } + if (storeBody) { + context->response->appendBody(out, outLen); + } + context->receivedBodyLength += outLen; + auto cb = _bodyChunkCallback; + if (cb) + cb(out, outLen, false); + return true; + }; + for (;;) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + GzipDecoder::Result r = context->gzipDecoder.finish(&outPtr, &outLen); + if (outLen > 0) { + if (!emitBodyBytes(reinterpret_cast(outPtr), outLen)) + return; + } + if (r == GzipDecoder::Result::kDone) + break; + if (r == GzipDecoder::Result::kOk) + continue; + triggerError(context, GZIP_DECODE_FAILED, context->gzipDecoder.lastError()); + return; + } + } +#endif // Otherwise success: either Content-Length reached, or no Content-Length and closure marks the end processResponse(context); } @@ -939,6 +1061,16 @@ bool AsyncHttpClient::parseResponseHeaders(RequestContext* context, const String context->response->setContentLength(context->expectedContentLength); } else if (name.equalsIgnoreCase("Transfer-Encoding") && value.equalsIgnoreCase("chunked")) { context->chunked = true; + } else if (name.equalsIgnoreCase("Content-Encoding")) { +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + String lower = value; + lower.toLowerCase(); + if (lower.indexOf("gzip") != -1) { + context->gzipEncoded = true; + context->gzipDecodeActive = true; + context->gzipDecoder.begin(); + } +#endif } else if (name.equalsIgnoreCase("Connection")) { String lower = value; lower.toLowerCase(); @@ -1212,6 +1344,7 @@ void AsyncHttpClient::resetContextForRedirect(RequestContext* context, AsyncHttp context->responseProcessed = false; context->expectedContentLength = 0; context->receivedContentLength = 0; + context->receivedBodyLength = 0; context->chunked = false; context->chunkedComplete = false; context->currentChunkRemaining = 0; @@ -1224,6 +1357,11 @@ void AsyncHttpClient::resetContextForRedirect(RequestContext* context, AsyncHttp context->serverRequestedClose = false; context->usingPooledConnection = false; context->resolvedTlsConfig = AsyncHttpTLSConfig(); +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + context->gzipEncoded = false; + context->gzipDecodeActive = false; + context->gzipDecoder.reset(); +#endif #if !ASYNC_TCP_HAS_TIMEOUT context->timeoutTimer = millis(); #endif diff --git a/src/AsyncHttpClient.h b/src/AsyncHttpClient.h index 6edec2a..309f4d0 100644 --- a/src/AsyncHttpClient.h +++ b/src/AsyncHttpClient.h @@ -11,6 +11,9 @@ #include "HttpResponse.h" #include "HttpCommon.h" #include "AsyncTransport.h" +#if ASYNC_HTTP_ENABLE_GZIP_DECODE +#include "GzipDecoder.h" +#endif #include #if defined(ARDUINO_ARCH_ESP32) && defined(ASYNC_HTTP_ENABLE_AUTOLOOP) #include @@ -147,6 +150,7 @@ class AsyncHttpClient { bool responseProcessed; size_t expectedContentLength; size_t receivedContentLength; + size_t receivedBodyLength; bool chunked; bool chunkedComplete; size_t currentChunkRemaining; @@ -164,16 +168,26 @@ class AsyncHttpClient { bool serverRequestedClose; bool usingPooledConnection; AsyncHttpTLSConfig resolvedTlsConfig; +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + bool gzipEncoded; + bool gzipDecodeActive; + GzipDecoder gzipDecoder; +#endif #if !ASYNC_TCP_HAS_TIMEOUT uint32_t timeoutTimer; #endif RequestContext() : request(nullptr), response(nullptr), transport(nullptr), headersComplete(false), responseProcessed(false), - expectedContentLength(0), receivedContentLength(0), chunked(false), chunkedComplete(false), + expectedContentLength(0), receivedContentLength(0), receivedBodyLength(0), chunked(false), + chunkedComplete(false), currentChunkRemaining(0), awaitingFinalChunkTerminator(false), id(0), trailerLineCount(0), redirectCount(0), notifiedEndCallback(false), connectStartMs(0), connectTimeoutMs(0), headersSent(false), streamingBodyInProgress(false), requestKeepAlive(false), serverRequestedClose(false), usingPooledConnection(false) +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + , + gzipEncoded(false), gzipDecodeActive(false) +#endif #if !ASYNC_TCP_HAS_TIMEOUT , timeoutTimer(0) diff --git a/src/GzipDecoder.cpp b/src/GzipDecoder.cpp new file mode 100644 index 0000000..51e41d2 --- /dev/null +++ b/src/GzipDecoder.cpp @@ -0,0 +1,354 @@ +#include "GzipDecoder.h" + +#include +#include + +#include "third_party/miniz/miniz_tinfl.h" + +static constexpr size_t kGzipFixedHeaderSize = 10; +static constexpr size_t kGzipTrailerSize = 8; +static constexpr size_t kTinflDictSize = 32768; + +static constexpr uint8_t kGzipId1 = 0x1f; +static constexpr uint8_t kGzipId2 = 0x8b; +static constexpr uint8_t kGzipCmDeflate = 0x08; + +static constexpr uint8_t kGzipFlagHcrc = 0x02; +static constexpr uint8_t kGzipFlagExtra = 0x04; +static constexpr uint8_t kGzipFlagName = 0x08; +static constexpr uint8_t kGzipFlagComment = 0x10; + +GzipDecoder::GzipDecoder() : _state(State::kHeader), _headerStage(HeaderStage::kFixed10), _error(nullptr), _fixedLen(0), + _flags(0), _extraLenRead(0), _extraRemaining(0), _needName(false), _needComment(false), + _needHcrc(false), _trailerLen(0), _dict(nullptr), _dictOfs(0), _decomp(nullptr) { + memset(_fixed, 0, sizeof(_fixed)); + memset(_extraLenBytes, 0, sizeof(_extraLenBytes)); + memset(_trailer, 0, sizeof(_trailer)); +} + +GzipDecoder::~GzipDecoder() { + reset(); +} + +void GzipDecoder::reset() { + if (_dict) { + free(_dict); + _dict = nullptr; + } + if (_decomp) { + free(_decomp); + _decomp = nullptr; + } + + _state = State::kHeader; + _headerStage = HeaderStage::kFixed10; + _error = nullptr; + + _fixedLen = 0; + _flags = 0; + _extraLenRead = 0; + _extraRemaining = 0; + _needName = false; + _needComment = false; + _needHcrc = false; + + _trailerLen = 0; + _dictOfs = 0; +} + +bool GzipDecoder::begin() { + reset(); + return true; +} + +bool GzipDecoder::isDone() const { + return _state == State::kDone; +} + +const char* GzipDecoder::lastError() const { + return _error ? _error : ""; +} + +void GzipDecoder::setError(const char* msg) { + _error = msg ? msg : "gzip decode error"; + _state = State::kError; +} + +bool GzipDecoder::initInflater() { + if (_state == State::kError) + return false; + if (_decomp) + return true; + + _dict = malloc(kTinflDictSize); + if (!_dict) { + setError("Out of memory (gzip dict)"); + return false; + } + memset(_dict, 0, kTinflDictSize); + + _decomp = malloc(sizeof(tinfl_decompressor)); + if (!_decomp) { + setError("Out of memory (tinfl)"); + return false; + } + tinfl_init(static_cast(_decomp)); + _dictOfs = 0; + return true; +} + +GzipDecoder::Result GzipDecoder::consumeHeader(const uint8_t* in, size_t inLen, size_t* inConsumed) { + if (!inConsumed) + return Result::kError; + *inConsumed = 0; + + while (*inConsumed < inLen && _state == State::kHeader) { + switch (_headerStage) { + case HeaderStage::kFixed10: { + size_t remaining = inLen - *inConsumed; + size_t need = kGzipFixedHeaderSize - _fixedLen; + size_t take = remaining < need ? remaining : need; + if (take > 0) { + memcpy(_fixed + _fixedLen, in + *inConsumed, take); + _fixedLen += take; + *inConsumed += take; + } + if (_fixedLen < kGzipFixedHeaderSize) { + return Result::kNeedMoreInput; + } + + if (_fixed[0] != kGzipId1 || _fixed[1] != kGzipId2) { + setError("Not a gzip stream"); + return Result::kError; + } + if (_fixed[2] != kGzipCmDeflate) { + setError("Unsupported gzip compression method"); + return Result::kError; + } + + _flags = _fixed[3]; + _needName = (_flags & kGzipFlagName) != 0; + _needComment = (_flags & kGzipFlagComment) != 0; + _needHcrc = (_flags & kGzipFlagHcrc) != 0; + _headerStage = (_flags & kGzipFlagExtra) ? HeaderStage::kExtraLen : HeaderStage::kName; + break; + } + case HeaderStage::kExtraLen: { + while (*inConsumed < inLen && _extraLenRead < 2) { + _extraLenBytes[_extraLenRead++] = in[(*inConsumed)++]; + } + if (_extraLenRead < 2) + return Result::kNeedMoreInput; + _extraRemaining = (uint16_t)_extraLenBytes[0] | ((uint16_t)_extraLenBytes[1] << 8); + _headerStage = HeaderStage::kExtraData; + break; + } + case HeaderStage::kExtraData: { + if (_extraRemaining == 0) { + _headerStage = HeaderStage::kName; + break; + } + size_t remaining = inLen - *inConsumed; + size_t take = remaining < (size_t)_extraRemaining ? remaining : (size_t)_extraRemaining; + _extraRemaining -= (uint16_t)take; + *inConsumed += take; + if (_extraRemaining > 0) + return Result::kNeedMoreInput; + _headerStage = HeaderStage::kName; + break; + } + case HeaderStage::kName: { + if (!_needName) { + _headerStage = HeaderStage::kComment; + break; + } + while (*inConsumed < inLen) { + uint8_t c = in[(*inConsumed)++]; + if (c == 0) { + _needName = false; + break; + } + } + if (_needName) + return Result::kNeedMoreInput; + _headerStage = HeaderStage::kComment; + break; + } + case HeaderStage::kComment: { + if (!_needComment) { + _headerStage = HeaderStage::kHcrc; + break; + } + while (*inConsumed < inLen) { + uint8_t c = in[(*inConsumed)++]; + if (c == 0) { + _needComment = false; + break; + } + } + if (_needComment) + return Result::kNeedMoreInput; + _headerStage = HeaderStage::kHcrc; + break; + } + case HeaderStage::kHcrc: { + if (!_needHcrc) { + _headerStage = HeaderStage::kDone; + break; + } + size_t remaining = inLen - *inConsumed; + if (remaining < 2) + return Result::kNeedMoreInput; + *inConsumed += 2; // skip header CRC + _needHcrc = false; + _headerStage = HeaderStage::kDone; + break; + } + case HeaderStage::kDone: + default: + _state = State::kInflate; + if (!initInflater()) + return Result::kError; + return Result::kOk; + } + } + + return (_state == State::kInflate) ? Result::kOk : Result::kNeedMoreInput; +} + +GzipDecoder::Result GzipDecoder::consumeTrailer(const uint8_t* in, size_t inLen, size_t* inConsumed) { + if (!inConsumed) + return Result::kError; + *inConsumed = 0; + + while (*inConsumed < inLen && _state == State::kTrailer) { + size_t remaining = inLen - *inConsumed; + size_t need = kGzipTrailerSize - _trailerLen; + size_t take = remaining < need ? remaining : need; + if (take > 0) { + memcpy(_trailer + _trailerLen, in + *inConsumed, take); + _trailerLen += take; + *inConsumed += take; + } + if (_trailerLen < kGzipTrailerSize) + return Result::kNeedMoreInput; + _state = State::kDone; + return Result::kDone; + } + + return (_state == State::kDone) ? Result::kDone : Result::kNeedMoreInput; +} + +GzipDecoder::Result GzipDecoder::write(const uint8_t* in, size_t inLen, size_t* inConsumed, const uint8_t** outPtr, + size_t* outLen, bool hasMoreInput) { + if (!inConsumed || !outPtr || !outLen) + return Result::kError; + *inConsumed = 0; + *outPtr = nullptr; + *outLen = 0; + + if (_state == State::kError) + return Result::kError; + if (_state == State::kDone) + return Result::kDone; + + size_t totalConsumed = 0; + if (_state == State::kHeader) { + size_t localConsumed = 0; + Result hr = consumeHeader(in, inLen, &localConsumed); + totalConsumed += localConsumed; + *inConsumed = totalConsumed; + if (hr == Result::kError || hr == Result::kDone) + return hr; + if (_state != State::kInflate) + return hr; + if (totalConsumed >= inLen) + return Result::kOk; + // fallthrough to inflate with remaining input + } + + if (_state == State::kTrailer) { + size_t localConsumed = 0; + Result tr = consumeTrailer(in, inLen, &localConsumed); + totalConsumed += localConsumed; + *inConsumed = totalConsumed; + return tr; + } + + if (_state != State::kInflate) { + setError("Invalid gzip state"); + return Result::kError; + } + + if (!_decomp || !_dict) { + setError("Inflater not initialized"); + return Result::kError; + } + + tinfl_decompressor* decomp = static_cast(_decomp); + mz_uint8* dict = static_cast(_dict); + + const mz_uint8* inPtr = reinterpret_cast(in ? (in + totalConsumed) : nullptr); + size_t srcBufSize = (inLen >= totalConsumed) ? (inLen - totalConsumed) : 0; + size_t dstBufSize = kTinflDictSize - _dictOfs; + int flags = hasMoreInput ? TINFL_FLAG_HAS_MORE_INPUT : 0; + + tinfl_status status = tinfl_decompress(decomp, inPtr, &srcBufSize, dict, dict + _dictOfs, &dstBufSize, flags); + + totalConsumed += srcBufSize; + *inConsumed = totalConsumed; + if (dstBufSize > 0) { + *outPtr = reinterpret_cast(dict + _dictOfs); + *outLen = dstBufSize; + _dictOfs = (_dictOfs + dstBufSize) & (kTinflDictSize - 1); + } + + if (status < 0) { + setError("Deflate inflate failed"); + return Result::kError; + } + + if (status == TINFL_STATUS_NEEDS_MORE_INPUT) { + return Result::kNeedMoreInput; + } + if (status == TINFL_STATUS_HAS_MORE_OUTPUT) { + return Result::kOk; + } + if (status == TINFL_STATUS_DONE) { + _state = State::kTrailer; + if (totalConsumed < inLen) { + size_t trailerConsumed = 0; + Result tr = consumeTrailer(in + totalConsumed, inLen - totalConsumed, &trailerConsumed); + totalConsumed += trailerConsumed; + *inConsumed = totalConsumed; + if (tr == Result::kDone) + return Result::kDone; + if (tr == Result::kError) + return Result::kError; + } + return Result::kOk; + } + + setError("Unexpected inflate status"); + return Result::kError; +} + +GzipDecoder::Result GzipDecoder::finish(const uint8_t** outPtr, size_t* outLen) { + size_t consumed = 0; + Result r = write(nullptr, 0, &consumed, outPtr, outLen, false); + + if (r == Result::kNeedMoreInput) { + if (_state == State::kHeader) { + setError("Truncated gzip header"); + return Result::kError; + } + if (_state == State::kTrailer) { + setError("Truncated gzip trailer"); + return Result::kError; + } + setError("Truncated gzip stream"); + return Result::kError; + } + + return r; +} diff --git a/src/GzipDecoder.h b/src/GzipDecoder.h new file mode 100644 index 0000000..edcc246 --- /dev/null +++ b/src/GzipDecoder.h @@ -0,0 +1,81 @@ +#ifndef GZIP_DECODER_H +#define GZIP_DECODER_H + +#include +#include + +#ifndef ASYNC_HTTP_ENABLE_GZIP_DECODE +#define ASYNC_HTTP_ENABLE_GZIP_DECODE 0 +#endif + +class GzipDecoder { + public: + enum class Result { + kOk, + kNeedMoreInput, + kDone, + kError, + }; + + GzipDecoder(); + ~GzipDecoder(); + + void reset(); + bool begin(); + + Result write(const uint8_t* in, size_t inLen, size_t* inConsumed, const uint8_t** outPtr, size_t* outLen, + bool hasMoreInput); + Result finish(const uint8_t** outPtr, size_t* outLen); + + bool isDone() const; + const char* lastError() const; + + private: + enum class State { + kHeader, + kInflate, + kTrailer, + kDone, + kError, + }; + + enum class HeaderStage { + kFixed10, + kExtraLen, + kExtraData, + kName, + kComment, + kHcrc, + kDone, + }; + + bool initInflater(); + Result consumeHeader(const uint8_t* in, size_t inLen, size_t* inConsumed); + Result consumeTrailer(const uint8_t* in, size_t inLen, size_t* inConsumed); + void setError(const char* msg); + + State _state; + HeaderStage _headerStage; + const char* _error; + + uint8_t _fixed[10]; + size_t _fixedLen; + uint8_t _flags; + uint8_t _extraLenBytes[2]; + size_t _extraLenRead; + uint16_t _extraRemaining; + bool _needName; + bool _needComment; + bool _needHcrc; + + uint8_t _trailer[8]; + size_t _trailerLen; + + // Deflate (tinfl) state + void* _dict; + size_t _dictOfs; + void* _decomp; // tinfl_decompressor +}; + +#endif // GZIP_DECODER_H + diff --git a/src/HttpCommon.h b/src/HttpCommon.h index bc46ae6..208227d 100644 --- a/src/HttpCommon.h +++ b/src/HttpCommon.h @@ -6,7 +6,7 @@ // Feature flags (can be overridden before including library headers) #ifndef ASYNC_HTTP_ENABLE_GZIP_DECODE #define ASYNC_HTTP_ENABLE_GZIP_DECODE \ - 0 // 0 = only set Accept-Encoding header (no inflation). 1 = future: enable minimal gzip inflate. + 0 // 0 = no gzip inflation. 1 = enable transparent gzip response decoding (Content-Encoding: gzip). #endif // Security: allow enabling insecure TLS (skips CA validation) only when explicitly opted in at build time. @@ -54,7 +54,8 @@ enum HttpClientError { TLS_HANDSHAKE_FAILED = -14, TLS_CERT_INVALID = -15, TLS_FINGERPRINT_MISMATCH = -16, - TLS_HANDSHAKE_TIMEOUT = -17 + TLS_HANDSHAKE_TIMEOUT = -17, + GZIP_DECODE_FAILED = -18 }; inline const char* httpClientErrorToString(HttpClientError error) { @@ -93,6 +94,8 @@ inline const char* httpClientErrorToString(HttpClientError error) { return "TLS fingerprint mismatch"; case TLS_HANDSHAKE_TIMEOUT: return "TLS handshake timeout"; + case GZIP_DECODE_FAILED: + return "Failed to decode gzip body"; default: return "Network error"; } diff --git a/src/third_party/miniz/LICENSE b/src/third_party/miniz/LICENSE new file mode 100644 index 0000000..5d38110 --- /dev/null +++ b/src/third_party/miniz/LICENSE @@ -0,0 +1,22 @@ +Copyright 2013-2014 RAD Game Tools and Valve Software +Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC +All Rights Reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/src/third_party/miniz/miniz.h b/src/third_party/miniz/miniz.h new file mode 100644 index 0000000..7737ad0 --- /dev/null +++ b/src/third_party/miniz/miniz.h @@ -0,0 +1,21 @@ +#ifndef MINIZ_H +#define MINIZ_H + +// Minimal wrapper header to satisfy miniz_tinfl.c's include. +// Only the low-level inflate API is used by ESPAsyncWebClient. + +#ifndef MINIZ_NO_ARCHIVE_APIS +#define MINIZ_NO_ARCHIVE_APIS +#endif +#ifndef MINIZ_NO_STDIO +#define MINIZ_NO_STDIO +#endif +#ifndef MINIZ_NO_TIME +#define MINIZ_NO_TIME +#endif + +#include "miniz_common.h" +#include "miniz_tinfl.h" + +#endif // MINIZ_H + diff --git a/src/third_party/miniz/miniz_common.h b/src/third_party/miniz/miniz_common.h new file mode 100644 index 0000000..05cafae --- /dev/null +++ b/src/third_party/miniz/miniz_common.h @@ -0,0 +1,97 @@ +#pragma once +#include +#include +#include +#include + +#include "miniz_export.h" + +/* ------------------- Types and macros */ +typedef unsigned char mz_uint8; +typedef int16_t mz_int16; +typedef uint16_t mz_uint16; +typedef uint32_t mz_uint32; +typedef uint32_t mz_uint; +typedef int64_t mz_int64; +typedef uint64_t mz_uint64; +typedef int mz_bool; + +#define MZ_FALSE (0) +#define MZ_TRUE (1) + +/* Works around MSVC's spammy "warning C4127: conditional expression is constant" message. */ +#ifdef _MSC_VER +#define MZ_MACRO_END while (0, 0) +#else +#define MZ_MACRO_END while (0) +#endif + +#ifdef MINIZ_NO_STDIO +#define MZ_FILE void * +#else +#include +#define MZ_FILE FILE +#endif /* #ifdef MINIZ_NO_STDIO */ + +#ifdef MINIZ_NO_TIME +typedef struct mz_dummy_time_t_tag +{ + mz_uint32 m_dummy1; + mz_uint32 m_dummy2; +} mz_dummy_time_t; +#define MZ_TIME_T mz_dummy_time_t +#else +#define MZ_TIME_T time_t +#endif + +#define MZ_ASSERT(x) assert(x) + +#ifdef MINIZ_NO_MALLOC +#define MZ_MALLOC(x) NULL +#define MZ_FREE(x) (void)x, ((void)0) +#define MZ_REALLOC(p, x) NULL +#else +#define MZ_MALLOC(x) malloc(x) +#define MZ_FREE(x) free(x) +#define MZ_REALLOC(p, x) realloc(p, x) +#endif + +#define MZ_MAX(a, b) (((a) > (b)) ? (a) : (b)) +#define MZ_MIN(a, b) (((a) < (b)) ? (a) : (b)) +#define MZ_CLEAR_OBJ(obj) memset(&(obj), 0, sizeof(obj)) +#define MZ_CLEAR_ARR(obj) memset((obj), 0, sizeof(obj)) +#define MZ_CLEAR_PTR(obj) memset((obj), 0, sizeof(*obj)) + +#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES && MINIZ_LITTLE_ENDIAN +#define MZ_READ_LE16(p) *((const mz_uint16 *)(p)) +#define MZ_READ_LE32(p) *((const mz_uint32 *)(p)) +#else +#define MZ_READ_LE16(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U)) +#define MZ_READ_LE32(p) ((mz_uint32)(((const mz_uint8 *)(p))[0]) | ((mz_uint32)(((const mz_uint8 *)(p))[1]) << 8U) | ((mz_uint32)(((const mz_uint8 *)(p))[2]) << 16U) | ((mz_uint32)(((const mz_uint8 *)(p))[3]) << 24U)) +#endif + +#define MZ_READ_LE64(p) (((mz_uint64)MZ_READ_LE32(p)) | (((mz_uint64)MZ_READ_LE32((const mz_uint8 *)(p) + sizeof(mz_uint32))) << 32U)) + +#ifdef _MSC_VER +#define MZ_FORCEINLINE __forceinline +#elif defined(__GNUC__) +#define MZ_FORCEINLINE __inline__ __attribute__((__always_inline__)) +#else +#define MZ_FORCEINLINE inline +#endif + +#ifdef __cplusplus +extern "C" +{ +#endif + + extern MINIZ_EXPORT void *miniz_def_alloc_func(void *opaque, size_t items, size_t size); + extern MINIZ_EXPORT void miniz_def_free_func(void *opaque, void *address); + extern MINIZ_EXPORT void *miniz_def_realloc_func(void *opaque, void *address, size_t items, size_t size); + +#define MZ_UINT16_MAX (0xFFFFU) +#define MZ_UINT32_MAX (0xFFFFFFFFU) + +#ifdef __cplusplus +} +#endif diff --git a/src/third_party/miniz/miniz_export.h b/src/third_party/miniz/miniz_export.h new file mode 100644 index 0000000..a32fa45 --- /dev/null +++ b/src/third_party/miniz/miniz_export.h @@ -0,0 +1,10 @@ +#ifndef MINIZ_EXPORT_H +#define MINIZ_EXPORT_H + +// ESPAsyncWebClient: embedded use does not require symbol export attributes. +#ifndef MINIZ_EXPORT +#define MINIZ_EXPORT +#endif + +#endif // MINIZ_EXPORT_H + diff --git a/src/third_party/miniz/miniz_tinfl.c b/src/third_party/miniz/miniz_tinfl.c new file mode 100644 index 0000000..3de465f --- /dev/null +++ b/src/third_party/miniz/miniz_tinfl.c @@ -0,0 +1,770 @@ +/************************************************************************** + * + * Copyright 2013-2014 RAD Game Tools and Valve Software + * Copyright 2010-2014 Rich Geldreich and Tenacious Software LLC + * All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + **************************************************************************/ + +#include "miniz.h" + +#ifndef MINIZ_NO_INFLATE_APIS + +#ifdef __cplusplus +extern "C" +{ +#endif + + /* ------------------- Low-level Decompression (completely independent from all compression API's) */ + +#define TINFL_MEMCPY(d, s, l) memcpy(d, s, l) +#define TINFL_MEMSET(p, c, l) memset(p, c, l) + +#define TINFL_CR_BEGIN \ + switch (r->m_state) \ + { \ + case 0: +#define TINFL_CR_RETURN(state_index, result) \ + do \ + { \ + status = result; \ + r->m_state = state_index; \ + goto common_exit; \ + case state_index:; \ + } \ + MZ_MACRO_END +#define TINFL_CR_RETURN_FOREVER(state_index, result) \ + do \ + { \ + for (;;) \ + { \ + TINFL_CR_RETURN(state_index, result); \ + } \ + } \ + MZ_MACRO_END +#define TINFL_CR_FINISH } + +#define TINFL_GET_BYTE(state_index, c) \ + do \ + { \ + while (pIn_buf_cur >= pIn_buf_end) \ + { \ + TINFL_CR_RETURN(state_index, (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) ? TINFL_STATUS_NEEDS_MORE_INPUT : TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS); \ + } \ + c = *pIn_buf_cur++; \ + } \ + MZ_MACRO_END + +#define TINFL_NEED_BITS(state_index, n) \ + do \ + { \ + mz_uint c; \ + TINFL_GET_BYTE(state_index, c); \ + bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); \ + num_bits += 8; \ + } while (num_bits < (mz_uint)(n)) +#define TINFL_SKIP_BITS(state_index, n) \ + do \ + { \ + if (num_bits < (mz_uint)(n)) \ + { \ + TINFL_NEED_BITS(state_index, n); \ + } \ + bit_buf >>= (n); \ + num_bits -= (n); \ + } \ + MZ_MACRO_END +#define TINFL_GET_BITS(state_index, b, n) \ + do \ + { \ + if (num_bits < (mz_uint)(n)) \ + { \ + TINFL_NEED_BITS(state_index, n); \ + } \ + b = bit_buf & ((1 << (n)) - 1); \ + bit_buf >>= (n); \ + num_bits -= (n); \ + } \ + MZ_MACRO_END + +/* TINFL_HUFF_BITBUF_FILL() is only used rarely, when the number of bytes remaining in the input buffer falls below 2. */ +/* It reads just enough bytes from the input stream that are needed to decode the next Huffman code (and absolutely no more). It works by trying to fully decode a */ +/* Huffman code by using whatever bits are currently present in the bit buffer. If this fails, it reads another byte, and tries again until it succeeds or until the */ +/* bit buffer contains >=15 bits (deflate's max. Huffman code size). */ +#define TINFL_HUFF_BITBUF_FILL(state_index, pLookUp, pTree) \ + do \ + { \ + temp = pLookUp[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]; \ + if (temp >= 0) \ + { \ + code_len = temp >> 9; \ + if ((code_len) && (num_bits >= code_len)) \ + break; \ + } \ + else if (num_bits > TINFL_FAST_LOOKUP_BITS) \ + { \ + code_len = TINFL_FAST_LOOKUP_BITS; \ + do \ + { \ + temp = pTree[~temp + ((bit_buf >> code_len++) & 1)]; \ + } while ((temp < 0) && (num_bits >= (code_len + 1))); \ + if (temp >= 0) \ + break; \ + } \ + TINFL_GET_BYTE(state_index, c); \ + bit_buf |= (((tinfl_bit_buf_t)c) << num_bits); \ + num_bits += 8; \ + } while (num_bits < 15); + +/* TINFL_HUFF_DECODE() decodes the next Huffman coded symbol. It's more complex than you would initially expect because the zlib API expects the decompressor to never read */ +/* beyond the final byte of the deflate stream. (In other words, when this macro wants to read another byte from the input, it REALLY needs another byte in order to fully */ +/* decode the next Huffman code.) Handling this properly is particularly important on raw deflate (non-zlib) streams, which aren't followed by a byte aligned adler-32. */ +/* The slow path is only executed at the very end of the input buffer. */ +/* v1.16: The original macro handled the case at the very end of the passed-in input buffer, but we also need to handle the case where the user passes in 1+zillion bytes */ +/* following the deflate data and our non-conservative read-ahead path won't kick in here on this code. This is much trickier. */ +#define TINFL_HUFF_DECODE(state_index, sym, pLookUp, pTree) \ + do \ + { \ + int temp; \ + mz_uint code_len, c; \ + if (num_bits < 15) \ + { \ + if ((pIn_buf_end - pIn_buf_cur) < 2) \ + { \ + TINFL_HUFF_BITBUF_FILL(state_index, pLookUp, pTree); \ + } \ + else \ + { \ + bit_buf |= (((tinfl_bit_buf_t)pIn_buf_cur[0]) << num_bits) | (((tinfl_bit_buf_t)pIn_buf_cur[1]) << (num_bits + 8)); \ + pIn_buf_cur += 2; \ + num_bits += 16; \ + } \ + } \ + if ((temp = pLookUp[bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) \ + code_len = temp >> 9, temp &= 511; \ + else \ + { \ + code_len = TINFL_FAST_LOOKUP_BITS; \ + do \ + { \ + temp = pTree[~temp + ((bit_buf >> code_len++) & 1)]; \ + } while (temp < 0); \ + } \ + sym = temp; \ + bit_buf >>= code_len; \ + num_bits -= code_len; \ + } \ + MZ_MACRO_END + + static void tinfl_clear_tree(tinfl_decompressor *r) + { + if (r->m_type == 0) + MZ_CLEAR_ARR(r->m_tree_0); + else if (r->m_type == 1) + MZ_CLEAR_ARR(r->m_tree_1); + else + MZ_CLEAR_ARR(r->m_tree_2); + } + + tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags) + { + static const mz_uint16 s_length_base[31] = { 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258, 0, 0 }; + static const mz_uint8 s_length_extra[31] = { 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0 }; + static const mz_uint16 s_dist_base[32] = { 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 0, 0 }; + static const mz_uint8 s_dist_extra[32] = { 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13 }; + static const mz_uint8 s_length_dezigzag[19] = { 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 }; + static const mz_uint16 s_min_table_sizes[3] = { 257, 1, 4 }; + + mz_int16 *pTrees[3]; + mz_uint8 *pCode_sizes[3]; + + tinfl_status status = TINFL_STATUS_FAILED; + mz_uint32 num_bits, dist, counter, num_extra; + tinfl_bit_buf_t bit_buf; + const mz_uint8 *pIn_buf_cur = pIn_buf_next, *const pIn_buf_end = pIn_buf_next + *pIn_buf_size; + mz_uint8 *pOut_buf_cur = pOut_buf_next, *const pOut_buf_end = pOut_buf_next ? pOut_buf_next + *pOut_buf_size : NULL; + size_t out_buf_size_mask = (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF) ? (size_t)-1 : ((pOut_buf_next - pOut_buf_start) + *pOut_buf_size) - 1, dist_from_out_buf_start; + + /* Ensure the output buffer's size is a power of 2, unless the output buffer is large enough to hold the entire output file (in which case it doesn't matter). */ + if (((out_buf_size_mask + 1) & out_buf_size_mask) || (pOut_buf_next < pOut_buf_start)) + { + *pIn_buf_size = *pOut_buf_size = 0; + return TINFL_STATUS_BAD_PARAM; + } + + pTrees[0] = r->m_tree_0; + pTrees[1] = r->m_tree_1; + pTrees[2] = r->m_tree_2; + pCode_sizes[0] = r->m_code_size_0; + pCode_sizes[1] = r->m_code_size_1; + pCode_sizes[2] = r->m_code_size_2; + + num_bits = r->m_num_bits; + bit_buf = r->m_bit_buf; + dist = r->m_dist; + counter = r->m_counter; + num_extra = r->m_num_extra; + dist_from_out_buf_start = r->m_dist_from_out_buf_start; + TINFL_CR_BEGIN + + bit_buf = num_bits = dist = counter = num_extra = r->m_zhdr0 = r->m_zhdr1 = 0; + r->m_z_adler32 = r->m_check_adler32 = 1; + if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) + { + TINFL_GET_BYTE(1, r->m_zhdr0); + TINFL_GET_BYTE(2, r->m_zhdr1); + counter = (((r->m_zhdr0 * 256 + r->m_zhdr1) % 31 != 0) || (r->m_zhdr1 & 32) || ((r->m_zhdr0 & 15) != 8)); + if (!(decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)) + counter |= (((1U << (8U + (r->m_zhdr0 >> 4))) > 32768U) || ((out_buf_size_mask + 1) < (size_t)((size_t)1 << (8U + (r->m_zhdr0 >> 4))))); + if (counter) + { + TINFL_CR_RETURN_FOREVER(36, TINFL_STATUS_FAILED); + } + } + + do + { + TINFL_GET_BITS(3, r->m_final, 3); + r->m_type = r->m_final >> 1; + if (r->m_type == 0) + { + TINFL_SKIP_BITS(5, num_bits & 7); + for (counter = 0; counter < 4; ++counter) + { + if (num_bits) + TINFL_GET_BITS(6, r->m_raw_header[counter], 8); + else + TINFL_GET_BYTE(7, r->m_raw_header[counter]); + } + if ((counter = (r->m_raw_header[0] | (r->m_raw_header[1] << 8))) != (mz_uint)(0xFFFF ^ (r->m_raw_header[2] | (r->m_raw_header[3] << 8)))) + { + TINFL_CR_RETURN_FOREVER(39, TINFL_STATUS_FAILED); + } + while ((counter) && (num_bits)) + { + TINFL_GET_BITS(51, dist, 8); + while (pOut_buf_cur >= pOut_buf_end) + { + TINFL_CR_RETURN(52, TINFL_STATUS_HAS_MORE_OUTPUT); + } + *pOut_buf_cur++ = (mz_uint8)dist; + counter--; + } + while (counter) + { + size_t n; + while (pOut_buf_cur >= pOut_buf_end) + { + TINFL_CR_RETURN(9, TINFL_STATUS_HAS_MORE_OUTPUT); + } + while (pIn_buf_cur >= pIn_buf_end) + { + TINFL_CR_RETURN(38, (decomp_flags & TINFL_FLAG_HAS_MORE_INPUT) ? TINFL_STATUS_NEEDS_MORE_INPUT : TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS); + } + n = MZ_MIN(MZ_MIN((size_t)(pOut_buf_end - pOut_buf_cur), (size_t)(pIn_buf_end - pIn_buf_cur)), counter); + TINFL_MEMCPY(pOut_buf_cur, pIn_buf_cur, n); + pIn_buf_cur += n; + pOut_buf_cur += n; + counter -= (mz_uint)n; + } + } + else if (r->m_type == 3) + { + TINFL_CR_RETURN_FOREVER(10, TINFL_STATUS_FAILED); + } + else + { + if (r->m_type == 1) + { + mz_uint8 *p = r->m_code_size_0; + mz_uint i; + r->m_table_sizes[0] = 288; + r->m_table_sizes[1] = 32; + TINFL_MEMSET(r->m_code_size_1, 5, 32); + for (i = 0; i <= 143; ++i) + *p++ = 8; + for (; i <= 255; ++i) + *p++ = 9; + for (; i <= 279; ++i) + *p++ = 7; + for (; i <= 287; ++i) + *p++ = 8; + } + else + { + for (counter = 0; counter < 3; counter++) + { + TINFL_GET_BITS(11, r->m_table_sizes[counter], "\05\05\04"[counter]); + r->m_table_sizes[counter] += s_min_table_sizes[counter]; + } + MZ_CLEAR_ARR(r->m_code_size_2); + for (counter = 0; counter < r->m_table_sizes[2]; counter++) + { + mz_uint s; + TINFL_GET_BITS(14, s, 3); + r->m_code_size_2[s_length_dezigzag[counter]] = (mz_uint8)s; + } + r->m_table_sizes[2] = 19; + } + for (; (int)r->m_type >= 0; r->m_type--) + { + int tree_next, tree_cur; + mz_int16 *pLookUp; + mz_int16 *pTree; + mz_uint8 *pCode_size; + mz_uint i, j, used_syms, total, sym_index, next_code[17], total_syms[16]; + pLookUp = r->m_look_up[r->m_type]; + pTree = pTrees[r->m_type]; + pCode_size = pCode_sizes[r->m_type]; + MZ_CLEAR_ARR(total_syms); + TINFL_MEMSET(pLookUp, 0, sizeof(r->m_look_up[0])); + tinfl_clear_tree(r); + for (i = 0; i < r->m_table_sizes[r->m_type]; ++i) + total_syms[pCode_size[i]]++; + used_syms = 0, total = 0; + next_code[0] = next_code[1] = 0; + for (i = 1; i <= 15; ++i) + { + used_syms += total_syms[i]; + next_code[i + 1] = (total = ((total + total_syms[i]) << 1)); + } + if ((65536 != total) && (used_syms > 1)) + { + TINFL_CR_RETURN_FOREVER(35, TINFL_STATUS_FAILED); + } + for (tree_next = -1, sym_index = 0; sym_index < r->m_table_sizes[r->m_type]; ++sym_index) + { + mz_uint rev_code = 0, l, cur_code, code_size = pCode_size[sym_index]; + if (!code_size) + continue; + cur_code = next_code[code_size]++; + for (l = code_size; l > 0; l--, cur_code >>= 1) + rev_code = (rev_code << 1) | (cur_code & 1); + if (code_size <= TINFL_FAST_LOOKUP_BITS) + { + mz_int16 k = (mz_int16)((code_size << 9) | sym_index); + while (rev_code < TINFL_FAST_LOOKUP_SIZE) + { + pLookUp[rev_code] = k; + rev_code += (1 << code_size); + } + continue; + } + if (0 == (tree_cur = pLookUp[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)])) + { + pLookUp[rev_code & (TINFL_FAST_LOOKUP_SIZE - 1)] = (mz_int16)tree_next; + tree_cur = tree_next; + tree_next -= 2; + } + rev_code >>= (TINFL_FAST_LOOKUP_BITS - 1); + for (j = code_size; j > (TINFL_FAST_LOOKUP_BITS + 1); j--) + { + tree_cur -= ((rev_code >>= 1) & 1); + if (!pTree[-tree_cur - 1]) + { + pTree[-tree_cur - 1] = (mz_int16)tree_next; + tree_cur = tree_next; + tree_next -= 2; + } + else + tree_cur = pTree[-tree_cur - 1]; + } + tree_cur -= ((rev_code >>= 1) & 1); + pTree[-tree_cur - 1] = (mz_int16)sym_index; + } + if (r->m_type == 2) + { + for (counter = 0; counter < (r->m_table_sizes[0] + r->m_table_sizes[1]);) + { + mz_uint s; + TINFL_HUFF_DECODE(16, dist, r->m_look_up[2], r->m_tree_2); + if (dist < 16) + { + r->m_len_codes[counter++] = (mz_uint8)dist; + continue; + } + if ((dist == 16) && (!counter)) + { + TINFL_CR_RETURN_FOREVER(17, TINFL_STATUS_FAILED); + } + num_extra = "\02\03\07"[dist - 16]; + TINFL_GET_BITS(18, s, num_extra); + s += "\03\03\013"[dist - 16]; + TINFL_MEMSET(r->m_len_codes + counter, (dist == 16) ? r->m_len_codes[counter - 1] : 0, s); + counter += s; + } + if ((r->m_table_sizes[0] + r->m_table_sizes[1]) != counter) + { + TINFL_CR_RETURN_FOREVER(21, TINFL_STATUS_FAILED); + } + TINFL_MEMCPY(r->m_code_size_0, r->m_len_codes, r->m_table_sizes[0]); + TINFL_MEMCPY(r->m_code_size_1, r->m_len_codes + r->m_table_sizes[0], r->m_table_sizes[1]); + } + } + for (;;) + { + mz_uint8 *pSrc; + for (;;) + { + if (((pIn_buf_end - pIn_buf_cur) < 4) || ((pOut_buf_end - pOut_buf_cur) < 2)) + { + TINFL_HUFF_DECODE(23, counter, r->m_look_up[0], r->m_tree_0); + if (counter >= 256) + break; + while (pOut_buf_cur >= pOut_buf_end) + { + TINFL_CR_RETURN(24, TINFL_STATUS_HAS_MORE_OUTPUT); + } + *pOut_buf_cur++ = (mz_uint8)counter; + } + else + { + int sym2; + mz_uint code_len; +#if TINFL_USE_64BIT_BITBUF + if (num_bits < 30) + { + bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE32(pIn_buf_cur)) << num_bits); + pIn_buf_cur += 4; + num_bits += 32; + } +#else + if (num_bits < 15) + { + bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits); + pIn_buf_cur += 2; + num_bits += 16; + } +#endif + if ((sym2 = r->m_look_up[0][bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) + code_len = sym2 >> 9; + else + { + code_len = TINFL_FAST_LOOKUP_BITS; + do + { + sym2 = r->m_tree_0[~sym2 + ((bit_buf >> code_len++) & 1)]; + } while (sym2 < 0); + } + counter = sym2; + bit_buf >>= code_len; + num_bits -= code_len; + if (counter & 256) + break; + +#if !TINFL_USE_64BIT_BITBUF + if (num_bits < 15) + { + bit_buf |= (((tinfl_bit_buf_t)MZ_READ_LE16(pIn_buf_cur)) << num_bits); + pIn_buf_cur += 2; + num_bits += 16; + } +#endif + if ((sym2 = r->m_look_up[0][bit_buf & (TINFL_FAST_LOOKUP_SIZE - 1)]) >= 0) + code_len = sym2 >> 9; + else + { + code_len = TINFL_FAST_LOOKUP_BITS; + do + { + sym2 = r->m_tree_0[~sym2 + ((bit_buf >> code_len++) & 1)]; + } while (sym2 < 0); + } + bit_buf >>= code_len; + num_bits -= code_len; + + pOut_buf_cur[0] = (mz_uint8)counter; + if (sym2 & 256) + { + pOut_buf_cur++; + counter = sym2; + break; + } + pOut_buf_cur[1] = (mz_uint8)sym2; + pOut_buf_cur += 2; + } + } + if ((counter &= 511) == 256) + break; + + num_extra = s_length_extra[counter - 257]; + counter = s_length_base[counter - 257]; + if (num_extra) + { + mz_uint extra_bits; + TINFL_GET_BITS(25, extra_bits, num_extra); + counter += extra_bits; + } + + TINFL_HUFF_DECODE(26, dist, r->m_look_up[1], r->m_tree_1); + num_extra = s_dist_extra[dist]; + dist = s_dist_base[dist]; + if (num_extra) + { + mz_uint extra_bits; + TINFL_GET_BITS(27, extra_bits, num_extra); + dist += extra_bits; + } + + dist_from_out_buf_start = pOut_buf_cur - pOut_buf_start; + if ((dist == 0 || dist > dist_from_out_buf_start || dist_from_out_buf_start == 0) && (decomp_flags & TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)) + { + TINFL_CR_RETURN_FOREVER(37, TINFL_STATUS_FAILED); + } + + pSrc = pOut_buf_start + ((dist_from_out_buf_start - dist) & out_buf_size_mask); + + if ((MZ_MAX(pOut_buf_cur, pSrc) + counter) > pOut_buf_end) + { + while (counter--) + { + while (pOut_buf_cur >= pOut_buf_end) + { + TINFL_CR_RETURN(53, TINFL_STATUS_HAS_MORE_OUTPUT); + } + *pOut_buf_cur++ = pOut_buf_start[(dist_from_out_buf_start++ - dist) & out_buf_size_mask]; + } + continue; + } +#if MINIZ_USE_UNALIGNED_LOADS_AND_STORES + else if ((counter >= 9) && (counter <= dist)) + { + const mz_uint8 *pSrc_end = pSrc + (counter & ~7); + do + { +#ifdef MINIZ_UNALIGNED_USE_MEMCPY + memcpy(pOut_buf_cur, pSrc, sizeof(mz_uint32) * 2); +#else + ((mz_uint32 *)pOut_buf_cur)[0] = ((const mz_uint32 *)pSrc)[0]; + ((mz_uint32 *)pOut_buf_cur)[1] = ((const mz_uint32 *)pSrc)[1]; +#endif + pOut_buf_cur += 8; + } while ((pSrc += 8) < pSrc_end); + if ((counter &= 7) < 3) + { + if (counter) + { + pOut_buf_cur[0] = pSrc[0]; + if (counter > 1) + pOut_buf_cur[1] = pSrc[1]; + pOut_buf_cur += counter; + } + continue; + } + } +#endif + while (counter > 2) + { + pOut_buf_cur[0] = pSrc[0]; + pOut_buf_cur[1] = pSrc[1]; + pOut_buf_cur[2] = pSrc[2]; + pOut_buf_cur += 3; + pSrc += 3; + counter -= 3; + } + if (counter > 0) + { + pOut_buf_cur[0] = pSrc[0]; + if (counter > 1) + pOut_buf_cur[1] = pSrc[1]; + pOut_buf_cur += counter; + } + } + } + } while (!(r->m_final & 1)); + + /* Ensure byte alignment and put back any bytes from the bitbuf if we've looked ahead too far on gzip, or other Deflate streams followed by arbitrary data. */ + /* I'm being super conservative here. A number of simplifications can be made to the byte alignment part, and the Adler32 check shouldn't ever need to worry about reading from the bitbuf now. */ + TINFL_SKIP_BITS(32, num_bits & 7); + while ((pIn_buf_cur > pIn_buf_next) && (num_bits >= 8)) + { + --pIn_buf_cur; + num_bits -= 8; + } + bit_buf &= ~(~(tinfl_bit_buf_t)0 << num_bits); + MZ_ASSERT(!num_bits); /* if this assert fires then we've read beyond the end of non-deflate/zlib streams with following data (such as gzip streams). */ + + if (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) + { + for (counter = 0; counter < 4; ++counter) + { + mz_uint s; + if (num_bits) + TINFL_GET_BITS(41, s, 8); + else + TINFL_GET_BYTE(42, s); + r->m_z_adler32 = (r->m_z_adler32 << 8) | s; + } + } + TINFL_CR_RETURN_FOREVER(34, TINFL_STATUS_DONE); + + TINFL_CR_FINISH + + common_exit: + /* As long as we aren't telling the caller that we NEED more input to make forward progress: */ + /* Put back any bytes from the bitbuf in case we've looked ahead too far on gzip, or other Deflate streams followed by arbitrary data. */ + /* We need to be very careful here to NOT push back any bytes we definitely know we need to make forward progress, though, or we'll lock the caller up into an inf loop. */ + if ((status != TINFL_STATUS_NEEDS_MORE_INPUT) && (status != TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS)) + { + while ((pIn_buf_cur > pIn_buf_next) && (num_bits >= 8)) + { + --pIn_buf_cur; + num_bits -= 8; + } + } + r->m_num_bits = num_bits; + r->m_bit_buf = bit_buf & ~(~(tinfl_bit_buf_t)0 << num_bits); + r->m_dist = dist; + r->m_counter = counter; + r->m_num_extra = num_extra; + r->m_dist_from_out_buf_start = dist_from_out_buf_start; + *pIn_buf_size = pIn_buf_cur - pIn_buf_next; + *pOut_buf_size = pOut_buf_cur - pOut_buf_next; + if ((decomp_flags & (TINFL_FLAG_PARSE_ZLIB_HEADER | TINFL_FLAG_COMPUTE_ADLER32)) && (status >= 0)) + { + const mz_uint8 *ptr = pOut_buf_next; + size_t buf_len = *pOut_buf_size; + mz_uint32 i, s1 = r->m_check_adler32 & 0xffff, s2 = r->m_check_adler32 >> 16; + size_t block_len = buf_len % 5552; + while (buf_len) + { + for (i = 0; i + 7 < block_len; i += 8, ptr += 8) + { + s1 += ptr[0], s2 += s1; + s1 += ptr[1], s2 += s1; + s1 += ptr[2], s2 += s1; + s1 += ptr[3], s2 += s1; + s1 += ptr[4], s2 += s1; + s1 += ptr[5], s2 += s1; + s1 += ptr[6], s2 += s1; + s1 += ptr[7], s2 += s1; + } + for (; i < block_len; ++i) + s1 += *ptr++, s2 += s1; + s1 %= 65521U, s2 %= 65521U; + buf_len -= block_len; + block_len = 5552; + } + r->m_check_adler32 = (s2 << 16) + s1; + if ((status == TINFL_STATUS_DONE) && (decomp_flags & TINFL_FLAG_PARSE_ZLIB_HEADER) && (r->m_check_adler32 != r->m_z_adler32)) + status = TINFL_STATUS_ADLER32_MISMATCH; + } + return status; + } + + /* Higher level helper functions. */ + void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags) + { + tinfl_decompressor decomp; + void *pBuf = NULL, *pNew_buf; + size_t src_buf_ofs = 0, out_buf_capacity = 0; + *pOut_len = 0; + tinfl_init(&decomp); + for (;;) + { + size_t src_buf_size = src_buf_len - src_buf_ofs, dst_buf_size = out_buf_capacity - *pOut_len, new_out_buf_capacity; + tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8 *)pSrc_buf + src_buf_ofs, &src_buf_size, (mz_uint8 *)pBuf, pBuf ? (mz_uint8 *)pBuf + *pOut_len : NULL, &dst_buf_size, + (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); + if ((status < 0) || (status == TINFL_STATUS_NEEDS_MORE_INPUT)) + { + MZ_FREE(pBuf); + *pOut_len = 0; + return NULL; + } + src_buf_ofs += src_buf_size; + *pOut_len += dst_buf_size; + if (status == TINFL_STATUS_DONE) + break; + new_out_buf_capacity = out_buf_capacity * 2; + if (new_out_buf_capacity < 128) + new_out_buf_capacity = 128; + pNew_buf = MZ_REALLOC(pBuf, new_out_buf_capacity); + if (!pNew_buf) + { + MZ_FREE(pBuf); + *pOut_len = 0; + return NULL; + } + pBuf = pNew_buf; + out_buf_capacity = new_out_buf_capacity; + } + return pBuf; + } + + size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags) + { + tinfl_decompressor decomp; + tinfl_status status; + tinfl_init(&decomp); + status = tinfl_decompress(&decomp, (const mz_uint8 *)pSrc_buf, &src_buf_len, (mz_uint8 *)pOut_buf, (mz_uint8 *)pOut_buf, &out_buf_len, (flags & ~TINFL_FLAG_HAS_MORE_INPUT) | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF); + return (status != TINFL_STATUS_DONE) ? TINFL_DECOMPRESS_MEM_TO_MEM_FAILED : out_buf_len; + } + + int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) + { + int result = 0; + tinfl_decompressor decomp; + mz_uint8 *pDict = (mz_uint8 *)MZ_MALLOC(TINFL_LZ_DICT_SIZE); + size_t in_buf_ofs = 0, dict_ofs = 0; + if (!pDict) + return TINFL_STATUS_FAILED; + memset(pDict, 0, TINFL_LZ_DICT_SIZE); + tinfl_init(&decomp); + for (;;) + { + size_t in_buf_size = *pIn_buf_size - in_buf_ofs, dst_buf_size = TINFL_LZ_DICT_SIZE - dict_ofs; + tinfl_status status = tinfl_decompress(&decomp, (const mz_uint8 *)pIn_buf + in_buf_ofs, &in_buf_size, pDict, pDict + dict_ofs, &dst_buf_size, + (flags & ~(TINFL_FLAG_HAS_MORE_INPUT | TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF))); + in_buf_ofs += in_buf_size; + if ((dst_buf_size) && (!(*pPut_buf_func)(pDict + dict_ofs, (int)dst_buf_size, pPut_buf_user))) + break; + if (status != TINFL_STATUS_HAS_MORE_OUTPUT) + { + result = (status == TINFL_STATUS_DONE); + break; + } + dict_ofs = (dict_ofs + dst_buf_size) & (TINFL_LZ_DICT_SIZE - 1); + } + MZ_FREE(pDict); + *pIn_buf_size = in_buf_ofs; + return result; + } + +#ifndef MINIZ_NO_MALLOC + tinfl_decompressor *tinfl_decompressor_alloc(void) + { + tinfl_decompressor *pDecomp = (tinfl_decompressor *)MZ_MALLOC(sizeof(tinfl_decompressor)); + if (pDecomp) + tinfl_init(pDecomp); + return pDecomp; + } + + void tinfl_decompressor_free(tinfl_decompressor *pDecomp) + { + MZ_FREE(pDecomp); + } +#endif + +#ifdef __cplusplus +} +#endif + +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ diff --git a/src/third_party/miniz/miniz_tinfl.h b/src/third_party/miniz/miniz_tinfl.h new file mode 100644 index 0000000..7edca60 --- /dev/null +++ b/src/third_party/miniz/miniz_tinfl.h @@ -0,0 +1,150 @@ +#pragma once +#include "miniz_common.h" +/* ------------------- Low-level Decompression API Definitions */ + +#ifndef MINIZ_NO_INFLATE_APIS + +#ifdef __cplusplus +extern "C" +{ +#endif + /* Decompression flags used by tinfl_decompress(). */ + /* TINFL_FLAG_PARSE_ZLIB_HEADER: If set, the input has a valid zlib header and ends with an adler32 checksum (it's a valid zlib stream). Otherwise, the input is a raw deflate stream. */ + /* TINFL_FLAG_HAS_MORE_INPUT: If set, there are more input bytes available beyond the end of the supplied input buffer. If clear, the input buffer contains all remaining input. */ + /* TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF: If set, the output buffer is large enough to hold the entire decompressed stream. If clear, the output buffer is at least the size of the dictionary (typically 32KB). */ + /* TINFL_FLAG_COMPUTE_ADLER32: Force adler-32 checksum computation of the decompressed bytes. */ + enum + { + TINFL_FLAG_PARSE_ZLIB_HEADER = 1, + TINFL_FLAG_HAS_MORE_INPUT = 2, + TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF = 4, + TINFL_FLAG_COMPUTE_ADLER32 = 8 + }; + + /* High level decompression functions: */ + /* tinfl_decompress_mem_to_heap() decompresses a block in memory to a heap block allocated via malloc(). */ + /* On entry: */ + /* pSrc_buf, src_buf_len: Pointer and size of the Deflate or zlib source data to decompress. */ + /* On return: */ + /* Function returns a pointer to the decompressed data, or NULL on failure. */ + /* *pOut_len will be set to the decompressed data's size, which could be larger than src_buf_len on uncompressible data. */ + /* The caller must call mz_free() on the returned block when it's no longer needed. */ + MINIZ_EXPORT void *tinfl_decompress_mem_to_heap(const void *pSrc_buf, size_t src_buf_len, size_t *pOut_len, int flags); + +/* tinfl_decompress_mem_to_mem() decompresses a block in memory to another block in memory. */ +/* Returns TINFL_DECOMPRESS_MEM_TO_MEM_FAILED on failure, or the number of bytes written on success. */ +#define TINFL_DECOMPRESS_MEM_TO_MEM_FAILED ((size_t)(-1)) + MINIZ_EXPORT size_t tinfl_decompress_mem_to_mem(void *pOut_buf, size_t out_buf_len, const void *pSrc_buf, size_t src_buf_len, int flags); + + /* tinfl_decompress_mem_to_callback() decompresses a block in memory to an internal 32KB buffer, and a user provided callback function will be called to flush the buffer. */ + /* Returns 1 on success or 0 on failure. */ + typedef int (*tinfl_put_buf_func_ptr)(const void *pBuf, int len, void *pUser); + MINIZ_EXPORT int tinfl_decompress_mem_to_callback(const void *pIn_buf, size_t *pIn_buf_size, tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags); + + struct tinfl_decompressor_tag; + typedef struct tinfl_decompressor_tag tinfl_decompressor; + +#ifndef MINIZ_NO_MALLOC + /* Allocate the tinfl_decompressor structure in C so that */ + /* non-C language bindings to tinfl_ API don't need to worry about */ + /* structure size and allocation mechanism. */ + MINIZ_EXPORT tinfl_decompressor *tinfl_decompressor_alloc(void); + MINIZ_EXPORT void tinfl_decompressor_free(tinfl_decompressor *pDecomp); +#endif + +/* Max size of LZ dictionary. */ +#define TINFL_LZ_DICT_SIZE 32768 + + /* Return status. */ + typedef enum + { + /* This flags indicates the inflator needs 1 or more input bytes to make forward progress, but the caller is indicating that no more are available. The compressed data */ + /* is probably corrupted. If you call the inflator again with more bytes it'll try to continue processing the input but this is a BAD sign (either the data is corrupted or you called it incorrectly). */ + /* If you call it again with no input you'll just get TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS again. */ + TINFL_STATUS_FAILED_CANNOT_MAKE_PROGRESS = -4, + + /* This flag indicates that one or more of the input parameters was obviously bogus. (You can try calling it again, but if you get this error the calling code is wrong.) */ + TINFL_STATUS_BAD_PARAM = -3, + + /* This flags indicate the inflator is finished but the adler32 check of the uncompressed data didn't match. If you call it again it'll return TINFL_STATUS_DONE. */ + TINFL_STATUS_ADLER32_MISMATCH = -2, + + /* This flags indicate the inflator has somehow failed (bad code, corrupted input, etc.). If you call it again without resetting via tinfl_init() it it'll just keep on returning the same status failure code. */ + TINFL_STATUS_FAILED = -1, + + /* Any status code less than TINFL_STATUS_DONE must indicate a failure. */ + + /* This flag indicates the inflator has returned every byte of uncompressed data that it can, has consumed every byte that it needed, has successfully reached the end of the deflate stream, and */ + /* if zlib headers and adler32 checking enabled that it has successfully checked the uncompressed data's adler32. If you call it again you'll just get TINFL_STATUS_DONE over and over again. */ + TINFL_STATUS_DONE = 0, + + /* This flag indicates the inflator MUST have more input data (even 1 byte) before it can make any more forward progress, or you need to clear the TINFL_FLAG_HAS_MORE_INPUT */ + /* flag on the next call if you don't have any more source data. If the source data was somehow corrupted it's also possible (but unlikely) for the inflator to keep on demanding input to */ + /* proceed, so be sure to properly set the TINFL_FLAG_HAS_MORE_INPUT flag. */ + TINFL_STATUS_NEEDS_MORE_INPUT = 1, + + /* This flag indicates the inflator definitely has 1 or more bytes of uncompressed data available, but it cannot write this data into the output buffer. */ + /* Note if the source compressed data was corrupted it's possible for the inflator to return a lot of uncompressed data to the caller. I've been assuming you know how much uncompressed data to expect */ + /* (either exact or worst case) and will stop calling the inflator and fail after receiving too much. In pure streaming scenarios where you have no idea how many bytes to expect this may not be possible */ + /* so I may need to add some code to address this. */ + TINFL_STATUS_HAS_MORE_OUTPUT = 2 + } tinfl_status; + +/* Initializes the decompressor to its initial state. */ +#define tinfl_init(r) \ + do \ + { \ + (r)->m_state = 0; \ + } \ + MZ_MACRO_END +#define tinfl_get_adler32(r) (r)->m_check_adler32 + + /* Main low-level decompressor coroutine function. This is the only function actually needed for decompression. All the other functions are just high-level helpers for improved usability. */ + /* This is a universal API, i.e. it can be used as a building block to build any desired higher level decompression API. In the limit case, it can be called once per every byte input or output. */ + MINIZ_EXPORT tinfl_status tinfl_decompress(tinfl_decompressor *r, const mz_uint8 *pIn_buf_next, size_t *pIn_buf_size, mz_uint8 *pOut_buf_start, mz_uint8 *pOut_buf_next, size_t *pOut_buf_size, const mz_uint32 decomp_flags); + + /* Internal/private bits follow. */ + enum + { + TINFL_MAX_HUFF_TABLES = 3, + TINFL_MAX_HUFF_SYMBOLS_0 = 288, + TINFL_MAX_HUFF_SYMBOLS_1 = 32, + TINFL_MAX_HUFF_SYMBOLS_2 = 19, + TINFL_FAST_LOOKUP_BITS = 10, + TINFL_FAST_LOOKUP_SIZE = 1 << TINFL_FAST_LOOKUP_BITS + }; + +#if MINIZ_HAS_64BIT_REGISTERS +#define TINFL_USE_64BIT_BITBUF 1 +#else +#define TINFL_USE_64BIT_BITBUF 0 +#endif + +#if TINFL_USE_64BIT_BITBUF + typedef mz_uint64 tinfl_bit_buf_t; +#define TINFL_BITBUF_SIZE (64) +#else +typedef mz_uint32 tinfl_bit_buf_t; +#define TINFL_BITBUF_SIZE (32) +#endif + + struct tinfl_decompressor_tag + { + mz_uint32 m_state, m_num_bits, m_zhdr0, m_zhdr1, m_z_adler32, m_final, m_type, m_check_adler32, m_dist, m_counter, m_num_extra, m_table_sizes[TINFL_MAX_HUFF_TABLES]; + tinfl_bit_buf_t m_bit_buf; + size_t m_dist_from_out_buf_start; + mz_int16 m_look_up[TINFL_MAX_HUFF_TABLES][TINFL_FAST_LOOKUP_SIZE]; + mz_int16 m_tree_0[TINFL_MAX_HUFF_SYMBOLS_0 * 2]; + mz_int16 m_tree_1[TINFL_MAX_HUFF_SYMBOLS_1 * 2]; + mz_int16 m_tree_2[TINFL_MAX_HUFF_SYMBOLS_2 * 2]; + mz_uint8 m_code_size_0[TINFL_MAX_HUFF_SYMBOLS_0]; + mz_uint8 m_code_size_1[TINFL_MAX_HUFF_SYMBOLS_1]; + mz_uint8 m_code_size_2[TINFL_MAX_HUFF_SYMBOLS_2]; + mz_uint8 m_raw_header[4], m_len_codes[TINFL_MAX_HUFF_SYMBOLS_0 + TINFL_MAX_HUFF_SYMBOLS_1 + 137]; + }; + +#ifdef __cplusplus +} +#endif + +#endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ diff --git a/test/test_gzip_decode_native/test_main.cpp b/test/test_gzip_decode_native/test_main.cpp new file mode 100644 index 0000000..23b3bfe --- /dev/null +++ b/test/test_gzip_decode_native/test_main.cpp @@ -0,0 +1,122 @@ +#include + +#include +#include +#include +#include +#include + +#include "GzipDecoder.h" + +static const uint8_t kGzipHello[] = { + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0xd7, 0x51, + 0x48, 0xaf, 0xca, 0x2c, 0x50, 0xe4, 0x02, 0x00, 0x05, 0x14, 0xa6, 0xf3, 0x0d, 0x00, 0x00, 0x00, +}; + +static std::string decodeGzipInChunks(const std::vector& gz, size_t chunkSize) { + GzipDecoder dec; + TEST_ASSERT_TRUE(dec.begin()); + + std::string out; + size_t offset = 0; + while (offset < gz.size()) { + size_t n = std::min(chunkSize, gz.size() - offset); + size_t inner = 0; + while (inner < n) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + size_t consumed = 0; + GzipDecoder::Result r = + dec.write(gz.data() + offset + inner, n - inner, &consumed, &outPtr, &outLen, true); + if (outLen > 0) { + out.append(reinterpret_cast(outPtr), outLen); + } + TEST_ASSERT_NOT_EQUAL_MESSAGE(GzipDecoder::Result::kError, r, dec.lastError()); + TEST_ASSERT_TRUE_MESSAGE(consumed > 0 || outLen > 0, "decoder stalled"); + inner += consumed; + if (r == GzipDecoder::Result::kNeedMoreInput && inner >= n) { + break; + } + } + offset += n; + } + + for (;;) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + GzipDecoder::Result r = dec.finish(&outPtr, &outLen); + if (outLen > 0) { + out.append(reinterpret_cast(outPtr), outLen); + } + if (r == GzipDecoder::Result::kDone) + break; + TEST_ASSERT_EQUAL_MESSAGE(GzipDecoder::Result::kOk, r, dec.lastError()); + } + + TEST_ASSERT_TRUE(dec.isDone()); + return out; +} + +static void test_gzip_decode_single_chunk() { + const std::vector gz(kGzipHello, kGzipHello + sizeof(kGzipHello)); + std::string out = decodeGzipInChunks(gz, gz.size()); + TEST_ASSERT_EQUAL_STRING("Hello, gzip!\n", out.c_str()); +} + +static void test_gzip_decode_byte_by_byte() { + const std::vector gz(kGzipHello, kGzipHello + sizeof(kGzipHello)); + std::string out = decodeGzipInChunks(gz, 1); + TEST_ASSERT_EQUAL_STRING("Hello, gzip!\n", out.c_str()); +} + +static void test_gzip_header_with_fname() { + std::vector gz(kGzipHello, kGzipHello + sizeof(kGzipHello)); + // Set FNAME flag and insert a null-terminated filename after the fixed 10-byte header. + gz[3] |= 0x08; + const char name[] = "x.txt"; + gz.insert(gz.begin() + 10, name, name + sizeof(name)); // includes trailing NUL + + std::string out = decodeGzipInChunks(gz, 3); + TEST_ASSERT_EQUAL_STRING("Hello, gzip!\n", out.c_str()); +} + +static void test_truncated_gzip_fails() { + const std::vector gz(kGzipHello, kGzipHello + sizeof(kGzipHello)); + TEST_ASSERT_TRUE(gz.size() > 8); + + GzipDecoder dec; + TEST_ASSERT_TRUE(dec.begin()); + + const std::vector truncated(gz.begin(), gz.end() - 3); + size_t offset = 0; + while (offset < truncated.size()) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + size_t consumed = 0; + GzipDecoder::Result r = + dec.write(truncated.data() + offset, truncated.size() - offset, &consumed, &outPtr, &outLen, true); + TEST_ASSERT_NOT_EQUAL_MESSAGE(GzipDecoder::Result::kError, r, dec.lastError()); + TEST_ASSERT_TRUE(consumed > 0 || outLen > 0); + offset += consumed; + if (r == GzipDecoder::Result::kNeedMoreInput) + break; + } + + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + GzipDecoder::Result r = dec.finish(&outPtr, &outLen); + TEST_ASSERT_EQUAL(GzipDecoder::Result::kError, r); + TEST_ASSERT_TRUE(strlen(dec.lastError()) > 0); +} + +int main(int argc, char** argv) { + (void)argc; + (void)argv; + + UNITY_BEGIN(); + RUN_TEST(test_gzip_decode_single_chunk); + RUN_TEST(test_gzip_decode_byte_by_byte); + RUN_TEST(test_gzip_header_with_fname); + RUN_TEST(test_truncated_gzip_fails); + return UNITY_END(); +} From cf5385e82b5d6e94eaaa9d84fb164a7e71107a04 Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 18 Dec 2025 13:00:42 +0100 Subject: [PATCH 5/6] add ci gzip et clang --- .github/workflows/lint.yml | 11 +++- .github/workflows/test.yml | 7 ++- platformio.ini | 1 + src/AsyncHttpClient.cpp | 2 +- src/AsyncHttpClient.h | 7 +-- src/GzipDecoder.cpp | 64 +++++++++++++++++++++- src/GzipDecoder.h | 1 - src/third_party/miniz/miniz_tinfl.c | 8 +++ test/test_gzip_decode_native/test_main.cpp | 3 +- 9 files changed, 88 insertions(+), 16 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4138086..a27fb2c 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,8 @@ jobs: run: sudo apt-get update && sudo apt-get install -y clang-format - name: Check formatting run: | - files=$(git ls-files '*.h' '*.cpp' '*.ino') + # Exclude vendored third-party sources from formatting checks. + files=$(git ls-files '*.h' '*.cpp' '*.ino' | grep -v '^src/third_party/' || true) echo "Checking clang-format on: $files" diff_found=0 for f in $files; do @@ -33,7 +34,11 @@ jobs: run: pip install cpplint - name: Run cpplint run: | - cpplint --recursive --extensions=h,cpp src || true + # Keep third_party out of lint noise. + files=$(git ls-files src | grep -E '\\.(h|cpp)$' | grep -v '^src/third_party/' || true) + if [ -n "$files" ]; then + cpplint $files || true + fi # We don't fail hard yet; adjust policy later cppcheck: runs-on: ubuntu-latest @@ -45,7 +50,7 @@ jobs: run: | cppcheck --enable=warning,style,performance --inline-suppr \ --suppress=missingIncludeSystem \ - -I src --quiet src || true + -I src --quiet src --exclude=src/third_party || true version-sync: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8ecb0a6..404fbce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -199,14 +199,17 @@ jobs: path: | ~/.platformio/.cache key: ${{ runner.os }}-pio-${{ hashFiles('platformio.ini') }} - - name: Run native unit tests (UrlParser) + - name: Run native unit tests (UrlParser + gzip) if: ${{ hashFiles('test/**') != '' }} run: | echo "Detected test files:" $(ls test || true) - pio test -e native -f test_urlparser_native -v + pio test -e native -f test_urlparser_native -f test_gzip_decode_native -v - name: Skip notice (no tests found) if: ${{ hashFiles('test/**') == '' }} run: echo "No test directory present in this ref; skipping native tests." + - name: Compile ESP32 (gzip enabled) + run: | + pio run -e compile_test_gzip -v python-parse-tests: runs-on: ubuntu-latest diff --git a/platformio.ini b/platformio.ini index 28cf349..58550fa 100644 --- a/platformio.ini +++ b/platformio.ini @@ -92,3 +92,4 @@ build_src_filter = -<*> + + +gzipDecoder.write(reinterpret_cast(wire + offset), - wireLen - offset, &consumed, &outPtr, &outLen, true); + wireLen - offset, &consumed, &outPtr, &outLen, true); if (outLen > 0) { if (!emitBodyBytes(reinterpret_cast(outPtr), outLen)) return false; diff --git a/src/AsyncHttpClient.h b/src/AsyncHttpClient.h index 309f4d0..1ce9b38 100644 --- a/src/AsyncHttpClient.h +++ b/src/AsyncHttpClient.h @@ -179,10 +179,9 @@ class AsyncHttpClient { RequestContext() : request(nullptr), response(nullptr), transport(nullptr), headersComplete(false), responseProcessed(false), expectedContentLength(0), receivedContentLength(0), receivedBodyLength(0), chunked(false), - chunkedComplete(false), - currentChunkRemaining(0), awaitingFinalChunkTerminator(false), id(0), trailerLineCount(0), - redirectCount(0), notifiedEndCallback(false), connectStartMs(0), connectTimeoutMs(0), headersSent(false), - streamingBodyInProgress(false), requestKeepAlive(false), serverRequestedClose(false), + chunkedComplete(false), currentChunkRemaining(0), awaitingFinalChunkTerminator(false), id(0), + trailerLineCount(0), redirectCount(0), notifiedEndCallback(false), connectStartMs(0), connectTimeoutMs(0), + headersSent(false), streamingBodyInProgress(false), requestKeepAlive(false), serverRequestedClose(false), usingPooledConnection(false) #if ASYNC_HTTP_ENABLE_GZIP_DECODE , diff --git a/src/GzipDecoder.cpp b/src/GzipDecoder.cpp index 51e41d2..a7a210b 100644 --- a/src/GzipDecoder.cpp +++ b/src/GzipDecoder.cpp @@ -1,5 +1,60 @@ #include "GzipDecoder.h" +#if !ASYNC_HTTP_ENABLE_GZIP_DECODE + +#include + +GzipDecoder::GzipDecoder() + : _state(State::kError), _headerStage(HeaderStage::kFixed10), _error("Gzip decode disabled"), _fixedLen(0), + _flags(0), _extraLenRead(0), _extraRemaining(0), _needName(false), _needComment(false), _needHcrc(false), + _trailerLen(0), _dict(nullptr), _dictOfs(0), _decomp(nullptr) { + memset(_fixed, 0, sizeof(_fixed)); + memset(_extraLenBytes, 0, sizeof(_extraLenBytes)); + memset(_trailer, 0, sizeof(_trailer)); +} + +GzipDecoder::~GzipDecoder() {} + +void GzipDecoder::reset() {} + +bool GzipDecoder::begin() { + return false; +} + +GzipDecoder::Result GzipDecoder::write(const uint8_t* in, size_t inLen, size_t* inConsumed, const uint8_t** outPtr, + size_t* outLen, bool hasMoreInput) { + (void)in; + (void)inLen; + (void)hasMoreInput; + if (inConsumed) + *inConsumed = 0; + if (outPtr) + *outPtr = nullptr; + if (outLen) + *outLen = 0; + _error = "Gzip decode disabled (build with -DASYNC_HTTP_ENABLE_GZIP_DECODE=1)"; + return Result::kError; +} + +GzipDecoder::Result GzipDecoder::finish(const uint8_t** outPtr, size_t* outLen) { + if (outPtr) + *outPtr = nullptr; + if (outLen) + *outLen = 0; + _error = "Gzip decode disabled (build with -DASYNC_HTTP_ENABLE_GZIP_DECODE=1)"; + return Result::kError; +} + +bool GzipDecoder::isDone() const { + return false; +} + +const char* GzipDecoder::lastError() const { + return _error ? _error : ""; +} + +#else + #include #include @@ -18,9 +73,10 @@ static constexpr uint8_t kGzipFlagExtra = 0x04; static constexpr uint8_t kGzipFlagName = 0x08; static constexpr uint8_t kGzipFlagComment = 0x10; -GzipDecoder::GzipDecoder() : _state(State::kHeader), _headerStage(HeaderStage::kFixed10), _error(nullptr), _fixedLen(0), - _flags(0), _extraLenRead(0), _extraRemaining(0), _needName(false), _needComment(false), - _needHcrc(false), _trailerLen(0), _dict(nullptr), _dictOfs(0), _decomp(nullptr) { +GzipDecoder::GzipDecoder() + : _state(State::kHeader), _headerStage(HeaderStage::kFixed10), _error(nullptr), _fixedLen(0), _flags(0), + _extraLenRead(0), _extraRemaining(0), _needName(false), _needComment(false), _needHcrc(false), _trailerLen(0), + _dict(nullptr), _dictOfs(0), _decomp(nullptr) { memset(_fixed, 0, sizeof(_fixed)); memset(_extraLenBytes, 0, sizeof(_extraLenBytes)); memset(_trailer, 0, sizeof(_trailer)); @@ -352,3 +408,5 @@ GzipDecoder::Result GzipDecoder::finish(const uint8_t** outPtr, size_t* outLen) return r; } + +#endif // ASYNC_HTTP_ENABLE_GZIP_DECODE diff --git a/src/GzipDecoder.h b/src/GzipDecoder.h index edcc246..0971ef1 100644 --- a/src/GzipDecoder.h +++ b/src/GzipDecoder.h @@ -78,4 +78,3 @@ class GzipDecoder { }; #endif // GZIP_DECODER_H - diff --git a/src/third_party/miniz/miniz_tinfl.c b/src/third_party/miniz/miniz_tinfl.c index 3de465f..c415fdf 100644 --- a/src/third_party/miniz/miniz_tinfl.c +++ b/src/third_party/miniz/miniz_tinfl.c @@ -24,6 +24,12 @@ * **************************************************************************/ +#ifndef ASYNC_HTTP_ENABLE_GZIP_DECODE +#define ASYNC_HTTP_ENABLE_GZIP_DECODE 0 +#endif + +#if ASYNC_HTTP_ENABLE_GZIP_DECODE + #include "miniz.h" #ifndef MINIZ_NO_INFLATE_APIS @@ -768,3 +774,5 @@ extern "C" #endif #endif /*#ifndef MINIZ_NO_INFLATE_APIS*/ + +#endif // ASYNC_HTTP_ENABLE_GZIP_DECODE diff --git a/test/test_gzip_decode_native/test_main.cpp b/test/test_gzip_decode_native/test_main.cpp index 23b3bfe..8fc3bbb 100644 --- a/test/test_gzip_decode_native/test_main.cpp +++ b/test/test_gzip_decode_native/test_main.cpp @@ -26,8 +26,7 @@ static std::string decodeGzipInChunks(const std::vector& gz, size_t chu const uint8_t* outPtr = nullptr; size_t outLen = 0; size_t consumed = 0; - GzipDecoder::Result r = - dec.write(gz.data() + offset + inner, n - inner, &consumed, &outPtr, &outLen, true); + GzipDecoder::Result r = dec.write(gz.data() + offset + inner, n - inner, &consumed, &outPtr, &outLen, true); if (outLen > 0) { out.append(reinterpret_cast(outPtr), outLen); } From 5e32254729614220a2121be01f1e0f80704d0f3b Mon Sep 17 00:00:00 2001 From: Denis Date: Thu, 18 Dec 2025 14:14:15 +0100 Subject: [PATCH 6/6] correction gzip --- README.md | 1 + src/GzipDecoder.cpp | 57 ++++++++++++++++++++-- src/GzipDecoder.h | 3 ++ test/test_gzip_decode_native/test_main.cpp | 53 ++++++++++++++++++++ 4 files changed, 111 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4edf2e0..f0fe403 100644 --- a/README.md +++ b/README.md @@ -597,6 +597,7 @@ Notes: - `enableGzipAcceptEncoding(false)` removes `Accept-Encoding` from the request's header list (or call `request.removeHeader("Accept-Encoding")`). - `Content-Length` (when present) refers to the *compressed* payload size; completion detection still follows the wire length. - RAM impact: enabling gzip decode allocates an internal 32KB sliding window per active gzip-decoded response (plus small state). +- Integrity: the gzip trailer is verified (CRC32 + ISIZE); corrupted payloads raise `GZIP_DECODE_FAILED`. ### HTTPS quick reference diff --git a/src/GzipDecoder.cpp b/src/GzipDecoder.cpp index a7a210b..b579e59 100644 --- a/src/GzipDecoder.cpp +++ b/src/GzipDecoder.cpp @@ -7,7 +7,7 @@ GzipDecoder::GzipDecoder() : _state(State::kError), _headerStage(HeaderStage::kFixed10), _error("Gzip decode disabled"), _fixedLen(0), _flags(0), _extraLenRead(0), _extraRemaining(0), _needName(false), _needComment(false), _needHcrc(false), - _trailerLen(0), _dict(nullptr), _dictOfs(0), _decomp(nullptr) { + _trailerLen(0), _crc32(0), _outSize(0), _dict(nullptr), _dictOfs(0), _decomp(nullptr) { memset(_fixed, 0, sizeof(_fixed)); memset(_extraLenBytes, 0, sizeof(_extraLenBytes)); memset(_trailer, 0, sizeof(_trailer)); @@ -73,10 +73,42 @@ static constexpr uint8_t kGzipFlagExtra = 0x04; static constexpr uint8_t kGzipFlagName = 0x08; static constexpr uint8_t kGzipFlagComment = 0x10; +static uint32_t readLe32(const uint8_t* p) { + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static uint32_t* crc32Table() { + static bool inited = false; + static uint32_t table[256]; + if (inited) + return table; + for (uint32_t i = 0; i < 256; ++i) { + uint32_t c = i; + for (uint32_t k = 0; k < 8; ++k) { + if (c & 1U) + c = 0xEDB88320U ^ (c >> 1); + else + c >>= 1; + } + table[i] = c; + } + inited = true; + return table; +} + +static uint32_t crc32Update(uint32_t crc, const uint8_t* data, size_t len) { + uint32_t c = crc; + uint32_t* t = crc32Table(); + for (size_t i = 0; i < len; ++i) { + c = t[(c ^ data[i]) & 0xFFU] ^ (c >> 8); + } + return c; +} + GzipDecoder::GzipDecoder() : _state(State::kHeader), _headerStage(HeaderStage::kFixed10), _error(nullptr), _fixedLen(0), _flags(0), _extraLenRead(0), _extraRemaining(0), _needName(false), _needComment(false), _needHcrc(false), _trailerLen(0), - _dict(nullptr), _dictOfs(0), _decomp(nullptr) { + _crc32(0), _outSize(0), _dict(nullptr), _dictOfs(0), _decomp(nullptr) { memset(_fixed, 0, sizeof(_fixed)); memset(_extraLenBytes, 0, sizeof(_extraLenBytes)); memset(_trailer, 0, sizeof(_trailer)); @@ -110,6 +142,8 @@ void GzipDecoder::reset() { _trailerLen = 0; _dictOfs = 0; + _crc32 = 0xFFFFFFFFU; + _outSize = 0; } bool GzipDecoder::begin() { @@ -288,6 +322,20 @@ GzipDecoder::Result GzipDecoder::consumeTrailer(const uint8_t* in, size_t inLen, } if (_trailerLen < kGzipTrailerSize) return Result::kNeedMoreInput; + uint32_t expectedCrc = readLe32(_trailer); + uint32_t expectedISize = readLe32(_trailer + 4); + uint32_t gotCrc = _crc32 ^ 0xFFFFFFFFU; + uint32_t gotISize = _outSize; + + if (expectedCrc != gotCrc) { + setError("Gzip CRC32 mismatch"); + return Result::kError; + } + if (expectedISize != gotISize) { + setError("Gzip ISIZE mismatch"); + return Result::kError; + } + _state = State::kDone; return Result::kDone; } @@ -354,8 +402,11 @@ GzipDecoder::Result GzipDecoder::write(const uint8_t* in, size_t inLen, size_t* totalConsumed += srcBufSize; *inConsumed = totalConsumed; if (dstBufSize > 0) { - *outPtr = reinterpret_cast(dict + _dictOfs); + const uint8_t* produced = reinterpret_cast(dict + _dictOfs); + *outPtr = produced; *outLen = dstBufSize; + _crc32 = crc32Update(_crc32, produced, dstBufSize); + _outSize += (uint32_t)dstBufSize; _dictOfs = (_dictOfs + dstBufSize) & (kTinflDictSize - 1); } diff --git a/src/GzipDecoder.h b/src/GzipDecoder.h index 0971ef1..e303de4 100644 --- a/src/GzipDecoder.h +++ b/src/GzipDecoder.h @@ -71,6 +71,9 @@ class GzipDecoder { uint8_t _trailer[8]; size_t _trailerLen; + uint32_t _crc32; + uint32_t _outSize; + // Deflate (tinfl) state void* _dict; size_t _dictOfs; diff --git a/test/test_gzip_decode_native/test_main.cpp b/test/test_gzip_decode_native/test_main.cpp index 8fc3bbb..746c8e9 100644 --- a/test/test_gzip_decode_native/test_main.cpp +++ b/test/test_gzip_decode_native/test_main.cpp @@ -108,6 +108,57 @@ static void test_truncated_gzip_fails() { TEST_ASSERT_TRUE(strlen(dec.lastError()) > 0); } +static void test_gzip_crc_mismatch_fails() { + std::vector gz(kGzipHello, kGzipHello + sizeof(kGzipHello)); + // Flip one CRC byte in trailer (bytes 4 from end..) + gz[gz.size() - 8] ^= 0x01; + + GzipDecoder dec; + TEST_ASSERT_TRUE(dec.begin()); + + size_t offset = 0; + std::string out; + while (offset < gz.size()) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + size_t consumed = 0; + GzipDecoder::Result r = dec.write(gz.data() + offset, gz.size() - offset, &consumed, &outPtr, &outLen, true); + if (outLen > 0) + out.append(reinterpret_cast(outPtr), outLen); + if (r == GzipDecoder::Result::kError) + break; + TEST_ASSERT_TRUE(consumed > 0 || outLen > 0); + offset += consumed; + } + + TEST_ASSERT_FALSE_MESSAGE(dec.isDone(), "should not be done on CRC mismatch"); + TEST_ASSERT_TRUE(strlen(dec.lastError()) > 0); +} + +static void test_gzip_isize_mismatch_fails() { + std::vector gz(kGzipHello, kGzipHello + sizeof(kGzipHello)); + // Flip one ISIZE byte in trailer (last 4 bytes). + gz[gz.size() - 1] ^= 0x01; + + GzipDecoder dec; + TEST_ASSERT_TRUE(dec.begin()); + + size_t offset = 0; + while (offset < gz.size()) { + const uint8_t* outPtr = nullptr; + size_t outLen = 0; + size_t consumed = 0; + GzipDecoder::Result r = dec.write(gz.data() + offset, gz.size() - offset, &consumed, &outPtr, &outLen, true); + if (r == GzipDecoder::Result::kError) + break; + TEST_ASSERT_TRUE(consumed > 0 || outLen > 0); + offset += consumed; + } + + TEST_ASSERT_FALSE(dec.isDone()); + TEST_ASSERT_TRUE(strlen(dec.lastError()) > 0); +} + int main(int argc, char** argv) { (void)argc; (void)argv; @@ -117,5 +168,7 @@ int main(int argc, char** argv) { RUN_TEST(test_gzip_decode_byte_by_byte); RUN_TEST(test_gzip_header_with_fname); RUN_TEST(test_truncated_gzip_fails); + RUN_TEST(test_gzip_crc_mismatch_fails); + RUN_TEST(test_gzip_isize_mismatch_fails); return UNITY_END(); }