diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 1e785a5b3f7..dffe791581b 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -875,30 +875,96 @@ bool HTTPRequest::LoadHeaders(LineReader& reader) bool HTTPRequest::LoadBody(LineReader& reader) { // https://httpwg.org/specs/rfc9112.html#message.body + auto transfer_encoding_header = m_headers.FindFirst("Transfer-Encoding"); + if (transfer_encoding_header && ToLower(transfer_encoding_header.value()) == "chunked") { + // Transfer-Encoding: https://datatracker.ietf.org/doc/html/rfc7230.html#section-3.3.1 + // Chunked Transfer Coding: https://datatracker.ietf.org/doc/html/rfc7230.html#section-4.1 + // see evhttp_handle_chunked_read() in libevent http.c + while (reader.Remaining() > 0) { + auto maybe_chunk_size = reader.ReadLine(); + if (!maybe_chunk_size) return false; - // No Content-length or Transfer-Encoding header means no body, see libevent evhttp_get_body() - // TODO: we must also implement Transfer-Encoding for chunk-reading - auto content_length_values{m_headers.FindAll("Content-Length")}; - if (content_length_values.empty()) return true; + // Allow (but ignore) Chunk Extensions + // See https://www.rfc-editor.org/rfc/rfc9112.html#name-chunk-extensions + std::string_view chunk_size_noext{maybe_chunk_size.value()}; + const auto semicolon_pos = chunk_size_noext.find(';'); + if (semicolon_pos != chunk_size_noext.npos) { + chunk_size_noext.remove_suffix(chunk_size_noext.size() - semicolon_pos); + } - // Duplicate Content-Length headers are allowed only if they all have the same value - // https://www.rfc-editor.org/rfc/rfc7230#section-3.3.3 - const auto& first_content_length_value{content_length_values[0]}; - for (size_t i = 1; i < content_length_values.size(); ++i) { - if (content_length_values[i] != first_content_length_value) throw std::runtime_error("Differing Content-Length values"); + const auto chunk_size{ToIntegral(util::TrimStringView(chunk_size_noext), /*base=*/16)}; + if (!chunk_size) throw std::runtime_error("Cannot parse chunk length value"); + + if ((m_body.size() > MAX_BODY_SIZE) || + (*chunk_size > MAX_BODY_SIZE - m_body.size())) + throw ContentTooLargeError("Chunk will exceed max body size"); + + // Last chunk has size 0 + if (*chunk_size == 0) { + // Allow (but ignore) Chunked Trailer section, by + // reading CRLF-terminated lines until we read an empty line, + // which indicates the end of this request. + // See https://httpwg.org/specs/rfc9112.html#rfc.section.7.1.2 + const size_t trailer_start{reader.Consumed()}; + while (true) { + auto maybe_trailer = reader.ReadLine(); + if (reader.Consumed() - trailer_start > MAX_HEADERS_SIZE) { + throw std::runtime_error("HTTP chunked trailer exceeds size limit"); + } + if (!maybe_trailer) return false; + if (maybe_trailer->empty()) break; + } + // Complete request has been parsed, reader is now pointing + // to beginning of next request or end of the buffer. + return true; + } + + // We are still expecting more data for this chunk + if (reader.Remaining() < *chunk_size) { + return false; + } + + // Pack chunk onto body + m_body += reader.ReadLength(*chunk_size); + + // Even though every chunk size is explicitly declared, + // they are still terminated by a CRLF we don't need, + // just consume it here. + auto crlf = reader.ReadLine(); + if (!crlf) { + // CRLF not found before end of buffer: it has not been received by our socket yet. + return false; + } + // CRLF was found but there was unexpected data after the chunk_sized chunk + if (!crlf.value().empty()) throw std::runtime_error("Improperly terminated chunk"); + } + + // We read all the chunks but never got the last chunk, wait for client to send more + return false; + } else { + // No Content-length or Transfer-Encoding header means no body, see libevent evhttp_get_body() + auto content_length_values{m_headers.FindAll("Content-Length")}; + if (content_length_values.empty()) return true; + + // Duplicate Content-Length headers are allowed only if they all have the same value + // https://www.rfc-editor.org/rfc/rfc7230#section-3.3.3 + const auto& first_content_length_value{content_length_values[0]}; + for (size_t i = 1; i < content_length_values.size(); ++i) { + if (content_length_values[i] != first_content_length_value) throw std::runtime_error("Differing Content-Length values"); + } + + const auto content_length{ToIntegral(first_content_length_value)}; + if (!content_length) throw std::runtime_error("Cannot parse Content-Length value"); + + if (*content_length > MAX_BODY_SIZE) throw ContentTooLargeError("Max body size exceeded"); + + // Not enough data in buffer for expected body + if (reader.Remaining() < *content_length) return false; + + m_body = reader.ReadLength(*content_length); + + return true; } - - const auto content_length{ToIntegral(first_content_length_value)}; - if (!content_length) throw std::runtime_error("Cannot parse Content-Length value"); - - if (*content_length > MAX_BODY_SIZE) throw ContentTooLargeError("Max body size exceeded"); - - // Not enough data in buffer for expected body - if (reader.Remaining() < *content_length) return false; - - m_body = reader.ReadLength(*content_length); - - return true; } util::Expected HTTPServer::BindAndStartListening(const CService& to) diff --git a/src/httpserver.h b/src/httpserver.h index 092c593533b..01a97922924 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -215,7 +215,7 @@ constexpr size_t MAX_HEADERS_SIZE{8192}; //! Maximum size of an HTTP request body constexpr uint64_t MAX_BODY_SIZE{32_MiB}; -//! Thrown when a request body exceeds MAX_BODY_SIZE +//! Thrown when a request body exceeds MAX_BODY_SIZE (or *will* exceed, in chunked transfer) //! so the server can reply with more specific code 413 (content too large) vs general 400 (bad request) struct ContentTooLargeError : std::runtime_error { using std::runtime_error::runtime_error; diff --git a/src/test/httpserver_tests.cpp b/src/test/httpserver_tests.cpp index bb9da546d74..1c90cdc29d0 100644 --- a/src/test/httpserver_tests.cpp +++ b/src/test/httpserver_tests.cpp @@ -374,6 +374,117 @@ BOOST_AUTO_TEST_CASE(http_request_tests) BOOST_CHECK(req.LoadHeaders(reader)); BOOST_CHECK(!req.LoadBody(reader)); } + { + // Support "chunked" transfer. Chunk lengths are ascii-encoded hex integers, whitespace ignored + HTTPRequest req; + std::string_view ok_chunked = "GET / HTTP/1.0\n" + "Transfer-Encoding: chunked\n" + "\n" + "10\n" + R"({"method":"getbl)""\n" + " a \n" + R"(ockcount"})""\n" + "0\n" + "\n"; + LineReader reader(ok_chunked, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader)); + BOOST_CHECK(req.LoadHeaders(reader)); + BOOST_CHECK(req.LoadBody(reader)); + BOOST_CHECK_EQUAL(req.m_body, R"({"method":"getblockcount"})"); + } + { + // Prevent "chunked" transfer from exceeding size limit + HTTPRequest req; + std::string_view excessive_chunk_size = "GET / HTTP/1.0\n" + "Transfer-Encoding: chunked\n" + "\n" + "10\n" + R"({"method":"getbl)""\n" + "20000000\n" + R"(ockcount"})""\n" + "0\n" + "\n"; + LineReader reader(excessive_chunk_size, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader)); + BOOST_CHECK(req.LoadHeaders(reader)); + BOOST_CHECK_EXCEPTION(req.LoadBody(reader), http_bitcoin::ContentTooLargeError, HasReason{"Chunk will exceed max body size"}); + } + { + // Allow (but ignore) Chunk Extensions + HTTPRequest req; + std::string_view ok_chunked = "GET / HTTP/1.0\n" + "Transfer-Encoding: chunked\n" + "\n" + "10;sha256=715790e8a3b09d704ac9641f42d183a5ebc5fd939663de23da548519ac2165e5\n" + R"({"method":"getbl)""\n" + " a ; compressed\n" + R"(ockcount"})""\n" + "0;why;would;anyone;do;this;\n" + "Expires: Wed, 21 Oct 2026 07:28:00 GMT\n" + "\n"; + LineReader reader(ok_chunked, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader)); + BOOST_CHECK(req.LoadHeaders(reader)); + BOOST_CHECK(req.LoadBody(reader)); + BOOST_CHECK_EQUAL(req.m_body, R"({"method":"getblockcount"})"); + // Chunk Trailer was cleared + BOOST_CHECK_EQUAL(reader.Remaining(), 0); + } + { + // Invalid "chunked" transfer, using roman numerals instead of hex for chunk length + HTTPRequest req; + std::string_view invalid_chunked = "GET / HTTP/1.0\n" + "Transfer-Encoding: chunked\n" + "\n" + "XVI\n" + R"({"method":"getbl)""\n" + "X\n" + R"(ockcount"})""\n" + "0\n" + "\n"; + LineReader reader(invalid_chunked, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader)); + BOOST_CHECK(req.LoadHeaders(reader)); + BOOST_CHECK_EXCEPTION(req.LoadBody(reader), std::runtime_error, HasReason{"Cannot parse chunk length value"}); + } + { + // Invalid "chunked" transfer, missing chunk termination \n + HTTPRequest req; + std::string_view invalid_chunked = "GET / HTTP/1.0\n" + "Transfer-Encoding: chunked\n" + "\n" + "10\n" + R"({"method":"getbl)" + "a\n" // interpreted as extra data at the end of `0x10`-sized chunk + R"(ockcount"})" + "0\n" + "\n"; + LineReader reader(invalid_chunked, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader)); + BOOST_CHECK(req.LoadHeaders(reader)); + BOOST_CHECK_EXCEPTION(req.LoadBody(reader), std::runtime_error, HasReason{"Improperly terminated chunk"}); + } + { + // End of buffer reached without chunk termination, caller must wait for more data to arrive + HTTPRequest req; + std::string delayed_chunked = "GET / HTTP/1.0\n" + "Transfer-Encoding: chunked\n" + "\n" + "10\n" + R"({"method":"getbl)""\n" + "a\n" + R"(ockcount"})"; + LineReader reader1(delayed_chunked, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader1)); + BOOST_CHECK(req.LoadHeaders(reader1)); + BOOST_CHECK(!req.LoadBody(reader1)); + // more data arrives! + delayed_chunked += "\n0\n\n"; + LineReader reader2(delayed_chunked, MAX_HEADERS_SIZE); + BOOST_CHECK(req.LoadControlData(reader2)); + BOOST_CHECK(req.LoadHeaders(reader2)); + BOOST_CHECK(req.LoadBody(reader2)); + } } BOOST_AUTO_TEST_CASE(http_server_socket_tests)