HTTPServer: compose and send replies to connected clients

Sockets-touching bits copied and adapted from `CConnman::SocketSendData()`

Testing this requires adding a new feature to the SocketTestingSetup,
returning the DynSock I/O pipes from the mock socket so the received
data can be checked.

Co-authored-by: Vasil Dimov <vd@FreeBSD.org>

src/httpserver.h

@@ -295,6 +295,9 @@ public:
     //! Pointer to the client that made the request so we know who to respond to.
     std::shared_ptr<HTTPRemoteClient> m_client;
 
+    //! Response headers may be set in advance before response body is known
+    HTTPHeaders m_response_headers;
+
     explicit HTTPRequest(std::shared_ptr<HTTPRemoteClient> client) : m_client{std::move(client)} {}
     //! Construct with a null client for unit tests
     explicit HTTPRequest() : m_client{} {}
@@ -312,6 +315,12 @@ public:
     bool LoadHeaders(LineReader& reader);
     bool LoadBody(LineReader& reader);
     /// @}
+
+    void WriteReply(HTTPStatusCode status, std::span<const std::byte> reply_body = {});
+    void WriteReply(HTTPStatusCode status, std::string_view reply_body_view)
+    {
+        WriteReply(status, std::as_bytes(std::span{reply_body_view}));
+    }
 };
 
 class HTTPServer
@@ -518,6 +527,22 @@ public:
      */
     std::vector<std::byte> m_recv_buffer{};
 
+    /**
+     * Response data destined for this client.
+     * Written to by http worker threads, read and erased by HTTPServer I/O thread
+     */
+    /// @{
+    Mutex m_send_mutex;
+    std::vector<std::byte> m_send_buffer GUARDED_BY(m_send_mutex);
+    /// @}
+
+    /**
+     * Set true by worker threads after writing a response to m_send_buffer.
+     * Set false by the HTTPServer I/O thread after flushing m_send_buffer.
+     * Checked in the HTTPServer I/O loop to avoid locking m_send_mutex if there's nothing to send.
+     */
+    std::atomic_bool m_send_ready{false};
+
     /**
      * Mutex that serializes the Send() and Recv() calls on `m_sock`. Reading
      * from the client occurs in the I/O thread but writing back to a client
@@ -547,6 +572,12 @@ public:
      * @returns true upon reading a complete request, otherwise false (may throw).
      */
     bool ReadRequest(HTTPRequest& req);
+
+    /**
+     * Push data (if there is any) from client's m_send_buffer to the connected socket.
+     * @returns false if we are done with this client and HTTPServer can skip the next read operation from it.
+     */
+    bool MaybeSendBytesFromBuffer() EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex, !m_sock_mutex);
 };
 } // namespace http_bitcoin