From d77843e2f0d266b175ec1addb288b9c1f49b89f7 Mon Sep 17 00:00:00 2001 From: Andreas Kling Date: Thu, 19 Sep 2024 08:06:17 +0200 Subject: [PATCH] LibHTTP: Remove the now-unused job classes The request/response classes are still used in a couple places and will need more careful removal. --- Meta/Lagom/Fuzzers/FuzzHttpRequest.cpp | 22 - Meta/Lagom/Fuzzers/fuzzers.cmake | 2 - Userland/Libraries/LibCore/CMakeLists.txt | 1 - Userland/Libraries/LibCore/NetworkJob.cpp | 82 --- Userland/Libraries/LibCore/NetworkJob.h | 75 --- Userland/Libraries/LibHTTP/CMakeLists.txt | 2 - Userland/Libraries/LibHTTP/Forward.h | 2 - Userland/Libraries/LibHTTP/HttpRequest.cpp | 1 - Userland/Libraries/LibHTTP/HttpsJob.cpp | 16 - Userland/Libraries/LibHTTP/HttpsJob.h | 41 -- Userland/Libraries/LibHTTP/Job.cpp | 638 --------------------- Userland/Libraries/LibHTTP/Job.h | 84 --- 12 files changed, 966 deletions(-) delete mode 100644 Meta/Lagom/Fuzzers/FuzzHttpRequest.cpp delete mode 100644 Userland/Libraries/LibCore/NetworkJob.cpp delete mode 100644 Userland/Libraries/LibCore/NetworkJob.h delete mode 100644 Userland/Libraries/LibHTTP/HttpsJob.cpp delete mode 100644 Userland/Libraries/LibHTTP/HttpsJob.h delete mode 100644 Userland/Libraries/LibHTTP/Job.cpp delete mode 100644 Userland/Libraries/LibHTTP/Job.h diff --git a/Meta/Lagom/Fuzzers/FuzzHttpRequest.cpp b/Meta/Lagom/Fuzzers/FuzzHttpRequest.cpp deleted file mode 100644 index c677e6f8acb..00000000000 --- a/Meta/Lagom/Fuzzers/FuzzHttpRequest.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include - -extern "C" int LLVMFuzzerTestOneInput(uint8_t const* data, size_t size) -{ - AK::set_debug_enabled(false); - auto request_wrapper = HTTP::HttpRequest::from_raw_request(ReadonlyBytes { data, size }); - if (request_wrapper.is_error()) - return 0; - - auto& request = request_wrapper.value(); - VERIFY(request.method() != HTTP::HttpRequest::Method::Invalid); - - return 0; -} diff --git a/Meta/Lagom/Fuzzers/fuzzers.cmake b/Meta/Lagom/Fuzzers/fuzzers.cmake index 3334e3219de..c941fb414e3 100644 --- a/Meta/Lagom/Fuzzers/fuzzers.cmake +++ b/Meta/Lagom/Fuzzers/fuzzers.cmake @@ -10,7 +10,6 @@ set(FUZZER_TARGETS GIFLoader GzipDecompression GzipRoundtrip - HttpRequest ICCProfile ICOLoader JPEGLoader @@ -64,7 +63,6 @@ set(FUZZER_DEPENDENCIES_FlacLoader LibMedia) set(FUZZER_DEPENDENCIES_GIFLoader LibGfx) set(FUZZER_DEPENDENCIES_GzipDecompression LibCompress) set(FUZZER_DEPENDENCIES_GzipRoundtrip LibCompress) -set(FUZZER_DEPENDENCIES_HttpRequest LibHTTP) set(FUZZER_DEPENDENCIES_ICCProfile LibGfx) set(FUZZER_DEPENDENCIES_ICOLoader LibGfx) set(FUZZER_DEPENDENCIES_JPEGLoader LibGfx) diff --git a/Userland/Libraries/LibCore/CMakeLists.txt b/Userland/Libraries/LibCore/CMakeLists.txt index 7fdee4d55aa..c750b8ff13c 100644 --- a/Userland/Libraries/LibCore/CMakeLists.txt +++ b/Userland/Libraries/LibCore/CMakeLists.txt @@ -34,7 +34,6 @@ set(SOURCES LockFile.cpp MappedFile.cpp MimeData.cpp - NetworkJob.cpp Notifier.cpp Process.cpp ProcessStatisticsReader.cpp diff --git a/Userland/Libraries/LibCore/NetworkJob.cpp b/Userland/Libraries/LibCore/NetworkJob.cpp deleted file mode 100644 index cb72a44f94d..00000000000 --- a/Userland/Libraries/LibCore/NetworkJob.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * Copyright (c) 2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include - -namespace Core { - -NetworkJob::NetworkJob(Stream& output_stream) - : m_output_stream(output_stream) -{ -} - -NetworkJob::~NetworkJob() = default; - -void NetworkJob::did_finish(NonnullRefPtr&& response) -{ - if (is_cancelled()) - return; - - // NOTE: We protect ourselves here, since the on_finish callback may otherwise - // trigger destruction of this job somehow. - NonnullRefPtr protector(*this); - - m_response = move(response); - dbgln_if(NETWORKJOB_DEBUG, "{} job did_finish", *this); - VERIFY(on_finish); - on_finish(true); - shutdown(ShutdownMode::DetachFromSocket); -} - -void NetworkJob::did_fail(Error error) -{ - if (is_cancelled()) - return; - - // NOTE: We protect ourselves here, since the on_finish callback may otherwise - // trigger destruction of this job somehow. - NonnullRefPtr protector(*this); - - m_error = error; - dbgln_if(NETWORKJOB_DEBUG, "{}{{{:p}}} job did_fail! error: {} ({})", class_name(), this, (unsigned)error, to_string(error)); - VERIFY(on_finish); - on_finish(false); - shutdown(ShutdownMode::DetachFromSocket); -} - -void NetworkJob::did_progress(Optional total_size, u64 downloaded) -{ - if (is_cancelled()) - return; - - // NOTE: We protect ourselves here, since the callback may otherwise - // trigger destruction of this job somehow. - NonnullRefPtr protector(*this); - - if (on_progress) - on_progress(total_size, downloaded); -} - -char const* to_string(NetworkJob::Error error) -{ - switch (error) { - case NetworkJob::Error::ProtocolFailed: - return "ProtocolFailed"; - case NetworkJob::Error::ConnectionFailed: - return "ConnectionFailed"; - case NetworkJob::Error::TransmissionFailed: - return "TransmissionFailed"; - case NetworkJob::Error::Cancelled: - return "Cancelled"; - default: - return "(Unknown error)"; - } -} - -} diff --git a/Userland/Libraries/LibCore/NetworkJob.h b/Userland/Libraries/LibCore/NetworkJob.h deleted file mode 100644 index 0cc1545a056..00000000000 --- a/Userland/Libraries/LibCore/NetworkJob.h +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * Copyright (c) 2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include - -namespace Core { - -class NetworkJob : public EventReceiver { - C_OBJECT_ABSTRACT(NetworkJob) -public: - enum class Error { - None, - ConnectionFailed, - TransmissionFailed, - ProtocolFailed, - Cancelled, - }; - virtual ~NetworkJob() override; - - // Could fire twice, after Headers and after Trailers! - Function response_code)> on_headers_received; - Function on_finish; - Function, u64)> on_progress; - - bool is_cancelled() const { return m_error == Error::Cancelled; } - bool has_error() const { return m_error != Error::None; } - Error error() const { return m_error; } - NetworkResponse* response() { return m_response.ptr(); } - NetworkResponse const* response() const { return m_response.ptr(); } - - enum class ShutdownMode { - DetachFromSocket, - CloseSocket, - }; - virtual void start(Core::BufferedSocketBase&) = 0; - virtual void shutdown(ShutdownMode) = 0; - virtual void fail(Error error) { did_fail(error); } - - void cancel() - { - shutdown(ShutdownMode::DetachFromSocket); - m_error = Error::Cancelled; - } - -protected: - NetworkJob(Stream&); - void did_finish(NonnullRefPtr&&); - void did_fail(Error); - void did_progress(Optional total_size, u64 downloaded); - - ErrorOr do_write(ReadonlyBytes bytes) { return m_output_stream.write_some(bytes); } - -private: - RefPtr m_response; - Stream& m_output_stream; - Error m_error { Error::None }; -}; - -char const* to_string(NetworkJob::Error); - -} - -template<> -struct AK::Formatter : Formatter { -}; diff --git a/Userland/Libraries/LibHTTP/CMakeLists.txt b/Userland/Libraries/LibHTTP/CMakeLists.txt index 483d7729e71..d2750188162 100644 --- a/Userland/Libraries/LibHTTP/CMakeLists.txt +++ b/Userland/Libraries/LibHTTP/CMakeLists.txt @@ -1,8 +1,6 @@ set(SOURCES HttpRequest.cpp HttpResponse.cpp - HttpsJob.cpp - Job.cpp ) serenity_lib(LibHTTP http) diff --git a/Userland/Libraries/LibHTTP/Forward.h b/Userland/Libraries/LibHTTP/Forward.h index 2c8614a8be7..a01fc9c0967 100644 --- a/Userland/Libraries/LibHTTP/Forward.h +++ b/Userland/Libraries/LibHTTP/Forward.h @@ -10,7 +10,5 @@ namespace HTTP { class HttpRequest; class HttpResponse; -class HttpsJob; -class Job; } diff --git a/Userland/Libraries/LibHTTP/HttpRequest.cpp b/Userland/Libraries/LibHTTP/HttpRequest.cpp index 430c7561fca..a89b80d0d8a 100644 --- a/Userland/Libraries/LibHTTP/HttpRequest.cpp +++ b/Userland/Libraries/LibHTTP/HttpRequest.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include namespace HTTP { diff --git a/Userland/Libraries/LibHTTP/HttpsJob.cpp b/Userland/Libraries/LibHTTP/HttpsJob.cpp deleted file mode 100644 index 28a2a5f6010..00000000000 --- a/Userland/Libraries/LibHTTP/HttpsJob.cpp +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2020-2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include - -namespace HTTP { - -void HttpsJob::set_certificate(ByteString certificate, ByteString key) -{ - m_received_client_certificates = TLS::TLSv12::parse_pem_certificate(certificate.bytes(), key.bytes()); -} - -} diff --git a/Userland/Libraries/LibHTTP/HttpsJob.h b/Userland/Libraries/LibHTTP/HttpsJob.h deleted file mode 100644 index 0be336e0988..00000000000 --- a/Userland/Libraries/LibHTTP/HttpsJob.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) 2020, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace HTTP { - -class HttpsJob final : public Job { - C_OBJECT(HttpsJob) -public: - virtual ~HttpsJob() override - { - } - - bool received_client_certificates() const { return m_received_client_certificates.has_value(); } - Vector take_client_certificates() const { return m_received_client_certificates.release_value(); } - - void set_certificate(ByteString certificate, ByteString key); - - Function()> on_certificate_requested; - -private: - explicit HttpsJob(HttpRequest&& request, Stream& output_stream) - : Job(move(request), output_stream) - { - } - - mutable Optional> m_received_client_certificates; -}; - -} diff --git a/Userland/Libraries/LibHTTP/Job.cpp b/Userland/Libraries/LibHTTP/Job.cpp deleted file mode 100644 index 2d9a60e5c6c..00000000000 --- a/Userland/Libraries/LibHTTP/Job.cpp +++ /dev/null @@ -1,638 +0,0 @@ -/* - * Copyright (c) 2018-2020, Andreas Kling - * Copyright (c) 2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace HTTP { - -static ErrorOr handle_content_encoding(ByteBuffer const& buf, ByteString const& content_encoding) -{ - dbgln_if(JOB_DEBUG, "Job::handle_content_encoding: buf has content_encoding={}", content_encoding); - - // FIXME: Actually do the decompression of the data using streams, instead of all at once when everything has been - // received. This will require that some of the decompression algorithms are implemented in a streaming way. - // Gzip and Deflate are implemented using Stream, while Brotli uses the newer Core::Stream. The Gzip and - // Deflate implementations will likely need to be changed to LibCore::Stream for this to work easily. - - if (content_encoding == "gzip") { - if (!Compress::GzipDecompressor::is_likely_compressed(buf)) { - dbgln("Job::handle_content_encoding: buf is not gzip compressed!"); - } - - dbgln_if(JOB_DEBUG, "Job::handle_content_encoding: buf is gzip compressed!"); - - auto uncompressed = TRY(Compress::GzipDecompressor::decompress_all(buf)); - - if constexpr (JOB_DEBUG) { - dbgln("Job::handle_content_encoding: Gzip::decompress() successful."); - dbgln(" Input size: {}", buf.size()); - dbgln(" Output size: {}", uncompressed.size()); - } - - return uncompressed; - } else if (content_encoding == "deflate") { - dbgln_if(JOB_DEBUG, "Job::handle_content_encoding: buf is deflate compressed!"); - - // Even though the content encoding is "deflate", it's actually deflate with the zlib wrapper. - // https://tools.ietf.org/html/rfc7230#section-4.2.2 - auto memory_stream = make(buf); - auto zlib_decompressor = Compress::ZlibDecompressor::create(move(memory_stream)); - Optional uncompressed; - if (zlib_decompressor.is_error()) { - // From the RFC: - // "Note: Some non-conformant implementations send the "deflate" - // compressed data without the zlib wrapper." - dbgln_if(JOB_DEBUG, "Job::handle_content_encoding: ZlibDecompressor::decompress_all() failed. Trying DeflateDecompressor::decompress_all()"); - uncompressed = TRY(Compress::DeflateDecompressor::decompress_all(buf)); - } else { - uncompressed = TRY(zlib_decompressor.value()->read_until_eof()); - } - - if constexpr (JOB_DEBUG) { - dbgln("Job::handle_content_encoding: Deflate decompression successful."); - dbgln(" Input size: {}", buf.size()); - dbgln(" Output size: {}", uncompressed.value().size()); - } - - return uncompressed.release_value(); - } else if (content_encoding == "br") { - dbgln_if(JOB_DEBUG, "Job::handle_content_encoding: buf is brotli compressed!"); - - FixedMemoryStream bufstream { buf }; - auto brotli_stream = Compress::BrotliDecompressionStream { MaybeOwned(bufstream) }; - - auto uncompressed = TRY(brotli_stream.read_until_eof()); - if constexpr (JOB_DEBUG) { - dbgln("Job::handle_content_encoding: Brotli::decompress() successful."); - dbgln(" Input size: {}", buf.size()); - dbgln(" Output size: {}", uncompressed.size()); - } - - return uncompressed; - } - - return buf; -} - -Job::Job(HttpRequest&& request, Stream& output_stream) - : Core::NetworkJob(output_stream) - , m_request(move(request)) -{ -} - -void Job::start(Core::BufferedSocketBase& socket) -{ - VERIFY(!m_socket); - m_socket = &socket; - dbgln_if(HTTPJOB_DEBUG, "Reusing previous connection for {}", url()); - deferred_invoke([this] { - dbgln_if(HTTPJOB_DEBUG, "HttpJob: on_connected callback"); - on_socket_connected(); - }); -} - -void Job::shutdown(ShutdownMode mode) -{ - if (!m_socket) - return; - if (mode == ShutdownMode::CloseSocket) { - m_socket->close(); - m_socket->on_ready_to_read = nullptr; - } else { - m_socket->on_ready_to_read = nullptr; - m_socket = nullptr; - } -} - -void Job::flush_received_buffers() -{ - if (!m_can_stream_response || m_buffered_size == 0) - return; - dbgln_if(JOB_DEBUG, "Job: Flushing received buffers: have {} bytes in {} buffers for {}", m_buffered_size, m_received_buffers.size(), m_request.url()); - for (size_t i = 0; i < m_received_buffers.size(); ++i) { - auto& payload = m_received_buffers[i]->pending_flush; - auto result = do_write(payload); - if (result.is_error()) { - if (!result.error().is_errno()) { - dbgln_if(JOB_DEBUG, "Job: Failed to flush received buffers: {}", result.error()); - continue; - } - if (result.error().code() == EINTR) { - i--; - continue; - } - break; - } - auto written = result.release_value(); - m_buffered_size -= written; - if (written == payload.size()) { - // FIXME: Make this a take-first-friendly object? - (void)m_received_buffers.take_first(); - --i; - continue; - } - VERIFY(written < payload.size()); - payload = payload.slice(written, payload.size() - written); - break; - } - dbgln_if(JOB_DEBUG, "Job: Flushing received buffers done: have {} bytes in {} buffers for {}", m_buffered_size, m_received_buffers.size(), m_request.url()); -} - -void Job::register_on_ready_to_read(Function callback) -{ - m_socket->on_ready_to_read = [this, callback = move(callback)] { - callback(); - - // As `m_socket` is a buffered object, we might not get notifications for data in the buffer - // so exhaust the buffer to ensure we don't end up waiting forever. - auto can_read_without_blocking = m_socket->can_read_without_blocking(); - if (can_read_without_blocking.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - if (can_read_without_blocking.value() && m_state != State::Finished && !has_error()) { - deferred_invoke([this] { - if (m_socket && m_socket->on_ready_to_read) - m_socket->on_ready_to_read(); - }); - } - }; -} - -ErrorOr Job::read_line(size_t size) -{ - auto buffer = TRY(ByteBuffer::create_uninitialized(size)); - auto bytes_read = TRY(m_socket->read_until(buffer, "\r\n"sv)); - return ByteString::copy(bytes_read); -} - -ErrorOr Job::receive(size_t size) -{ - if (size == 0) - return ByteBuffer {}; - - auto buffer = TRY(ByteBuffer::create_uninitialized(size)); - size_t nread; - do { - auto result = m_socket->read_some(buffer); - if (result.is_error() && result.error().is_errno() && result.error().code() == EINTR) - continue; - nread = TRY(result).size(); - break; - } while (true); - return buffer.slice(0, nread); -} - -void Job::on_socket_connected() -{ - auto raw_request = m_request.to_raw_request().release_value_but_fixme_should_propagate_errors(); - - if constexpr (JOB_DEBUG) { - dbgln("Job: raw_request:"); - dbgln("{}", ByteString::copy(raw_request)); - } - - bool success = !m_socket->write_until_depleted(raw_request).is_error(); - if (!success) - deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - - register_on_ready_to_read([&] { - dbgln_if(JOB_DEBUG, "Ready to read for {}, state = {}, cancelled = {}", m_request.url(), to_underlying(m_state), is_cancelled()); - if (is_cancelled()) - return; - - if (m_state == State::Finished) { - // We have everything we want, at this point, we can either get an EOF, or a bunch of extra newlines - // (unless "Connection: close" isn't specified) - // So just ignore everything after this. - return; - } - - if (m_socket->is_eof()) { - // Some servers really like terminating connections by simply closing them (even TLS ones) - // to signal end-of-data, if there's no: - // - connection - // - content-size - // - transfer-encoding: chunked - // header, simply treat EOF as a termination signal. - if (m_headers.contains("connection"sv) || m_content_length.has_value() || m_current_chunk_total_size.has_value()) { - dbgln_if(JOB_DEBUG, "Read failure: Actually EOF!"); - deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); }); - return; - } - - finish_up(); - return; - } - - while (m_state == State::InStatus) { - auto can_read_line = m_socket->can_read_up_to_delimiter("\r\n"sv.bytes()); - if (can_read_line.is_error()) { - dbgln_if(JOB_DEBUG, "Job {} could not figure out whether we could read a line", m_request.url()); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - - if (!can_read_line.value()) { - dbgln_if(JOB_DEBUG, "Job {} cannot read a full line", m_request.url()); - // TODO: Should we retry here instead of failing instantly? - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - - auto maybe_line = read_line(PAGE_SIZE); - if (maybe_line.is_error()) { - dbgln_if(JOB_DEBUG, "Job {} could not read line: {}", m_request.url(), maybe_line.error()); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - - auto line = maybe_line.release_value(); - - dbgln_if(JOB_DEBUG, "Job {} read line of length {}", m_request.url(), line.length()); - auto parts = line.split_view(' '); - if (parts.size() < 2) { - dbgln("Job: Expected 2-part or 3-part HTTP status line, got '{}'", line); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); }); - } - - if (!parts[0].matches("HTTP/?.?"sv, CaseSensitivity::CaseSensitive) || !is_ascii_digit(parts[0][5]) || !is_ascii_digit(parts[0][7])) { - dbgln("Job: Expected HTTP-Version to be of the form 'HTTP/X.Y', got '{}'", parts[0]); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); }); - } - auto http_major_version = parse_ascii_digit(parts[0][5]); - auto http_minor_version = parse_ascii_digit(parts[0][7]); - m_legacy_connection = http_major_version < 1 || (http_major_version == 1 && http_minor_version == 0); - - auto code = parts[1].to_number(); - if (!code.has_value()) { - dbgln("Job: Expected numeric HTTP status"); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); }); - } - m_code = code.value(); - m_state = State::InHeaders; - - auto can_read_without_blocking = m_socket->can_read_without_blocking(); - if (can_read_without_blocking.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - - if (!can_read_without_blocking.value()) - return; - } - while (m_state == State::InHeaders || m_state == State::Trailers) { - auto can_read_line = m_socket->can_read_up_to_delimiter("\r\n"sv.bytes()); - if (can_read_line.is_error()) { - dbgln_if(JOB_DEBUG, "Job {} could not figure out whether we could read a line", m_request.url()); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - - if (!can_read_line.value()) { - dbgln_if(JOB_DEBUG, "Can't read lines anymore :("); - return; - } - - // There's no max limit defined on headers, but for our sanity, let's limit it to 32K. - auto maybe_line = read_line(32 * KiB); - if (maybe_line.is_error()) { - dbgln_if(JOB_DEBUG, "Job {} could not read a header line: {}", m_request.url(), maybe_line.error()); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - auto line = maybe_line.release_value(); - - if (line.is_empty()) { - if (m_state == State::Trailers) { - return finish_up(); - } - if (on_headers_received) { - on_headers_received(m_headers, m_code > 0 ? m_code : Optional {}); - } - m_state = State::InBody; - - // We've reached the end of the headers, there's a possibility that the server - // responds with nothing (content-length = 0 with normal encoding); if that's the case, - // quit early as we won't be reading anything anyway. - if (auto result = m_headers.get("Content-Length"sv).value_or(""sv).to_number(); result.has_value()) { - if (result.value() == 0) { - auto transfer_encoding = m_headers.get("Transfer-Encoding"sv); - if (!transfer_encoding.has_value() || !transfer_encoding->view().trim_whitespace().equals_ignoring_ascii_case("chunked"sv)) - return finish_up(); - } - } - // There's also the possibility that the server responds with 204 (No Content), - // and manages to set a Content-Length anyway, in such cases ignore Content-Length and quit early; - // As the HTTP spec explicitly prohibits presence of Content-Length when the response code is 204. - if (m_code == 204) - return finish_up(); - - break; - } - auto parts = line.split_view(':'); - if (parts.is_empty()) { - if (m_state == State::Trailers) { - // Some servers like to send two ending chunks - // use this fact as an excuse to ignore anything after the last chunk - // that is not a valid trailing header. - return finish_up(); - } - dbgln("Job: Expected HTTP header with key/value"); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); }); - } - auto name = parts[0]; - if (line.length() < name.length() + 2) { - if (m_state == State::Trailers) { - // Some servers like to send two ending chunks - // use this fact as an excuse to ignore anything after the last chunk - // that is not a valid trailing header. - return finish_up(); - } - dbgln("Job: Malformed HTTP header: '{}' ({})", line, line.length()); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::ProtocolFailed); }); - } - auto value = line.substring(name.length() + 2, line.length() - name.length() - 2); - m_headers.set(name, value); - if (name.equals_ignoring_ascii_case("Content-Encoding"sv)) { - // Assume that any content-encoding means that we can't decode it as a stream :( - dbgln_if(JOB_DEBUG, "Content-Encoding {} detected, cannot stream output :(", value); - m_can_stream_response = false; - } else if (name.equals_ignoring_ascii_case("Content-Length"sv)) { - auto length = value.to_number(); - if (length.has_value()) - m_content_length = length.value(); - } - dbgln_if(JOB_DEBUG, "Job: [{}] = '{}'", name, value); - - auto can_read_without_blocking = m_socket->can_read_without_blocking(); - if (can_read_without_blocking.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - if (!can_read_without_blocking.value()) { - dbgln_if(JOB_DEBUG, "Can't read headers anymore, byebye :("); - return; - } - } - VERIFY(m_state == State::InBody); - - while (true) { - auto can_read_without_blocking = m_socket->can_read_without_blocking(); - if (can_read_without_blocking.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - if (!can_read_without_blocking.value()) - break; - - auto read_size = 64 * KiB; - if (m_current_chunk_remaining_size.has_value()) { - read_chunk_size:; - auto remaining = m_current_chunk_remaining_size.value(); - if (remaining == -1) { - // read size - auto can_read_line = m_socket->can_read_up_to_delimiter("\r\n"sv.bytes()); - if (can_read_line.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - if (!can_read_line.value()) // We'll try later. - return; - auto maybe_size_data = read_line(PAGE_SIZE); - if (maybe_size_data.is_error()) { - dbgln_if(JOB_DEBUG, "Job: Could not receive chunk: {}", maybe_size_data.error()); - } - auto size_data = maybe_size_data.release_value(); - - if (m_should_read_chunk_ending_line) { - // NOTE: Some servers seem to send an extra \r\n here despite there being no size. - // This makes us tolerate that. - size_data = size_data.trim("\r\n"sv, TrimMode::Right); - VERIFY(size_data.is_empty()); - m_should_read_chunk_ending_line = false; - continue; - } - auto size_lines = size_data.view().trim_whitespace().lines(); - dbgln_if(JOB_DEBUG, "Job: Received a chunk with size '{}'", size_data); - if (size_lines.size() == 0) { - if (!m_socket->is_eof()) - break; - dbgln("Job: Reached end of stream"); - finish_up(); - break; - } else { - auto chunk = size_lines[0].split_view(';', SplitBehavior::KeepEmpty); - ByteString size_string = chunk[0]; - char* endptr; - auto size = strtoul(size_string.characters(), &endptr, 16); - if (*endptr) { - // invalid number - deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - break; - } - if (size == 0) { - // This is the last chunk - // '0' *[; chunk-ext-name = chunk-ext-value] - // We're going to ignore _all_ chunk extensions - read_size = 0; - m_current_chunk_total_size = 0; - m_current_chunk_remaining_size = 0; - - dbgln_if(JOB_DEBUG, "Job: Received the last chunk with extensions '{}'", size_string.substring_view(1, size_string.length() - 1)); - } else { - m_current_chunk_total_size = size; - m_current_chunk_remaining_size = size; - read_size = size; - - dbgln_if(JOB_DEBUG, "Job: Chunk of size '{}' started", size); - } - } - } else { - read_size = remaining; - - dbgln_if(JOB_DEBUG, "Job: Resuming chunk with '{}' bytes left over", remaining); - } - } else { - auto transfer_encoding = m_headers.get("Transfer-Encoding"); - if (transfer_encoding.has_value()) { - // HTTP/1.1 3.3.3.3: - // If a message is received with both a Transfer-Encoding and a Content-Length header field, the Transfer-Encoding overrides the Content-Length. [...] - // https://httpwg.org/specs/rfc7230.html#message.body.length - m_content_length = {}; - - // Note: Some servers add extra spaces around 'chunked', see #6302. - auto encoding = transfer_encoding.value().trim_whitespace(); - - dbgln_if(JOB_DEBUG, "Job: This content has transfer encoding '{}'", encoding); - if (encoding.equals_ignoring_ascii_case("chunked"sv)) { - m_current_chunk_remaining_size = -1; - goto read_chunk_size; - } else { - dbgln("Job: Unknown transfer encoding '{}', the result will likely be wrong!", encoding); - } - } - } - - can_read_without_blocking = m_socket->can_read_without_blocking(); - if (can_read_without_blocking.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - if (!can_read_without_blocking.value()) - break; - - dbgln_if(JOB_DEBUG, "Waiting for payload for {}", m_request.url()); - auto maybe_payload = receive(read_size); - if (maybe_payload.is_error()) { - dbgln_if(JOB_DEBUG, "Could not read the payload: {}", maybe_payload.error()); - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - - auto payload = maybe_payload.release_value(); - - if (payload.is_empty() && m_socket->is_eof()) { - finish_up(); - break; - } - - bool read_everything = false; - if (m_content_length.has_value()) { - auto length = m_content_length.value(); - if (m_received_size + payload.size() >= length) { - payload.resize(length - m_received_size); - read_everything = true; - } - } - - m_received_buffers.append(make(payload)); - m_buffered_size += payload.size(); - m_received_size += payload.size(); - flush_received_buffers(); - - deferred_invoke([this] { did_progress(m_content_length, m_received_size); }); - - if (read_everything) { - VERIFY(m_received_size <= m_content_length.value()); - finish_up(); - break; - } - - // Check after reading all the buffered data if we have reached the end of stream - // for cases where the server didn't send a content length, chunked encoding but is - // directly closing the connection. - if (!m_content_length.has_value() && !m_current_chunk_remaining_size.has_value() && m_socket->is_eof()) { - finish_up(); - break; - } - - if (m_current_chunk_remaining_size.has_value()) { - auto size = m_current_chunk_remaining_size.value() - payload.size(); - - dbgln_if(JOB_DEBUG, "Job: We have {} bytes left over in this chunk", size); - if (size == 0) { - dbgln_if(JOB_DEBUG, "Job: Finished a chunk of {} bytes", m_current_chunk_total_size.value()); - - if (m_current_chunk_total_size.value() == 0) { - m_state = State::Trailers; - break; - } - - // we've read everything, now let's get the next chunk - size = -1; - - auto can_read_line = m_socket->can_read_up_to_delimiter("\r\n"sv.bytes()); - if (can_read_line.is_error()) - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - if (can_read_line.value()) { - auto maybe_line = read_line(PAGE_SIZE); - if (maybe_line.is_error()) { - return deferred_invoke([this] { did_fail(Core::NetworkJob::Error::TransmissionFailed); }); - } - - VERIFY(maybe_line.value().is_empty()); - } else { - m_should_read_chunk_ending_line = true; - } - } - m_current_chunk_remaining_size = size; - } - } - - if (!m_socket->is_open()) { - dbgln_if(JOB_DEBUG, "Connection appears to have closed, finishing up"); - finish_up(); - } - }); -} - -void Job::timer_event(Core::TimerEvent& event) -{ - event.accept(); - finish_up(); - if (m_buffered_size == 0) - stop_timer(); -} - -void Job::finish_up() -{ - VERIFY(!m_has_scheduled_finish); - m_state = State::Finished; - - if (is_cancelled()) { - stop_timer(); - m_has_scheduled_finish = true; - return; - } - - if (!m_can_stream_response) { - auto maybe_flattened_buffer = ByteBuffer::create_uninitialized(m_buffered_size); - if (maybe_flattened_buffer.is_error()) - return did_fail(Core::NetworkJob::Error::TransmissionFailed); - auto flattened_buffer = maybe_flattened_buffer.release_value(); - - u8* flat_ptr = flattened_buffer.data(); - for (auto& received_buffer : m_received_buffers) { - memcpy(flat_ptr, received_buffer->pending_flush.data(), received_buffer->pending_flush.size()); - flat_ptr += received_buffer->pending_flush.size(); - } - m_received_buffers.clear(); - - // For the time being, we cannot stream stuff with content-encoding set to _anything_. - // FIXME: LibCompress exposes a streaming interface, so this can be resolved - auto content_encoding = m_headers.get("Content-Encoding"); - if (content_encoding.has_value()) { - if (auto result = handle_content_encoding(flattened_buffer, content_encoding.value()); !result.is_error()) - flattened_buffer = result.release_value(); - else - return did_fail(Core::NetworkJob::Error::TransmissionFailed); - } - - m_buffered_size = flattened_buffer.size(); - m_received_buffers.append(make(move(flattened_buffer))); - m_can_stream_response = true; - } - - flush_received_buffers(); - if (m_buffered_size != 0) { - // We have to wait for the client to consume all the downloaded data - // before we can actually call `did_finish`. in a normal flow, this should - // never be hit since the client is reading as we are writing, unless there - // are too many concurrent downloads going on. - dbgln_if(JOB_DEBUG, "Flush finished with {} bytes remaining, will try again later", m_buffered_size); - if (!has_timer()) - start_timer(50); - return; - } - - stop_timer(); - m_has_scheduled_finish = true; - auto response = HttpResponse::create(m_code, move(m_headers), m_received_size); - deferred_invoke([this, response = move(response)] { - // If the server responded with "Connection: close", close the connection - // as the server may or may not want to close the socket. Also, if this is - // a legacy HTTP server (1.0 or older), assume close is the default value. - if (auto result = response->headers().get("Connection"sv); result.has_value() ? result->equals_ignoring_ascii_case("close"sv) : m_legacy_connection) - shutdown(ShutdownMode::CloseSocket); - did_finish(response); - }); -} -} diff --git a/Userland/Libraries/LibHTTP/Job.h b/Userland/Libraries/LibHTTP/Job.h deleted file mode 100644 index 15a18d5dbd9..00000000000 --- a/Userland/Libraries/LibHTTP/Job.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (c) 2020-2022, the SerenityOS developers. - * - * SPDX-License-Identifier: BSD-2-Clause - */ - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace HTTP { - -class Job : public Core::NetworkJob { - C_OBJECT(Job); - -public: - explicit Job(HttpRequest&&, Stream&); - virtual ~Job() override = default; - - virtual void start(Core::BufferedSocketBase&) override; - virtual void shutdown(ShutdownMode) override; - - Core::Socket const* socket() const { return m_socket; } - URL::URL url() const { return m_request.url(); } - - HttpResponse* response() { return static_cast(Core::NetworkJob::response()); } - HttpResponse const* response() const { return static_cast(Core::NetworkJob::response()); } - -protected: - void finish_up(); - void on_socket_connected(); - void flush_received_buffers(); - void register_on_ready_to_read(Function); - ErrorOr read_line(size_t); - ErrorOr receive(size_t); - void timer_event(Core::TimerEvent&) override; - - enum class State { - InStatus, - InHeaders, - InBody, - Trailers, - Finished, - }; - - HttpRequest m_request; - State m_state { State::InStatus }; - Core::BufferedSocketBase* m_socket { nullptr }; - bool m_legacy_connection { false }; - int m_code { -1 }; - HTTP::HeaderMap m_headers; - - struct ReceivedBuffer { - ReceivedBuffer(ByteBuffer d) - : data(move(d)) - , pending_flush(data.bytes()) - { - } - - // The entire received buffer. - ByteBuffer data; - - // The bytes we have yet to flush. (This is a slice of `data`) - ReadonlyBytes pending_flush; - }; - - Vector> m_received_buffers; - - size_t m_buffered_size { 0 }; - size_t m_received_size { 0 }; - Optional m_content_length; - Optional m_current_chunk_remaining_size; - Optional m_current_chunk_total_size; - bool m_can_stream_response { true }; - bool m_should_read_chunk_ending_line { false }; - bool m_has_scheduled_finish { false }; -}; - -}