Stowage/ladybird
2
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-12-08 06:09:58 +00:00
Code Issues Projects Releases Packages Wiki Activity
ladybird/Libraries/LibWeb/Loader/ResourceLoader.cpp
Aliaksandr Kalenik 3058274386 LibWeb: Use unbuffered network requests for all Fetch requests 
Previously, unbuffered requests were only available as a special mode
for EventSource. With this change, they are enabled by default, which
means chunks can be read from the stream as soon as they arrive.

This unlocks some interesting possibilities, such as starting to parse
HTML documents before the entire response has been received (that, in
turn, allows us to initiate subresource fetches earlier or begin
executing scripts sooner), or start rendering videos before they are
fully downloaded.

Co-authored-by: Timothy Flynn <trflynn89@pm.me>
2025-11-20 06:29:13 -05:00

649 lines
25 KiB
C++
Raw Blame History

/*
* Copyright (c) 2018-2024, Andreas Kling <andreas@ladybird.org>
* Copyright (c) 2022, Dex♪ <dexes.ttp@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <AK/Debug.h>
#include <LibCore/Directory.h>
#include <LibCore/MimeData.h>
#include <LibCore/Resource.h>
#include <LibGC/Function.h>
#include <LibHTTP/HttpResponse.h>
#include <LibRequests/Request.h>
#include <LibRequests/RequestClient.h>
#include <LibURL/Parser.h>
#include <LibWeb/Cookie/Cookie.h>
#include <LibWeb/Cookie/ParsedCookie.h>
#include <LibWeb/Fetch/Infrastructure/URL.h>
#include <LibWeb/Loader/ContentFilter.h>
#include <LibWeb/Loader/GeneratedPagesLoader.h>
#include <LibWeb/Loader/LoadRequest.h>
#include <LibWeb/Loader/ProxyMappings.h>
#include <LibWeb/Loader/ResourceLoader.h>
#include <LibWeb/Page/Page.h>
#include <LibWeb/Platform/EventLoopPlugin.h>
#include <LibWeb/Platform/Timer.h>
namespace Web {
static RefPtr<ResourceLoader> s_resource_loader;
void ResourceLoader::initialize(GC::Heap& heap, NonnullRefPtr<Requests::RequestClient> request_client)
{
s_resource_loader = adopt_ref(*new ResourceLoader(heap, move(request_client)));
}
ResourceLoader& ResourceLoader::the()
{
if (!s_resource_loader) {
dbgln("Web::ResourceLoader was not initialized");
VERIFY_NOT_REACHED();
}
return *s_resource_loader;
}
ResourceLoader::ResourceLoader(GC::Heap& heap, NonnullRefPtr<Requests::RequestClient> request_client)
: m_heap(heap)
, m_request_client(move(request_client))
, m_user_agent(MUST(String::from_utf8(default_user_agent)))
, m_platform(MUST(String::from_utf8(default_platform)))
, m_preferred_languages({ "en-US"_string })
, m_navigator_compatibility_mode(default_navigator_compatibility_mode)
{
m_request_client->on_request_server_died = [this]() {
m_request_client = nullptr;
};
}
void ResourceLoader::set_client(NonnullRefPtr<Requests::RequestClient> request_client)
{
m_request_client = move(request_client);
m_request_client->on_request_server_died = [this]() {
m_request_client = nullptr;
};
}
void ResourceLoader::prefetch_dns(URL::URL const& url)
{
if (url.scheme().is_one_of("file"sv, "data"sv))
return;
if (ContentFilter::the().is_filtered(url)) {
dbgln("ResourceLoader: Refusing to prefetch DNS for '{}': \033[31;1mURL was filtered\033[0m", url);
return;
}
// FIXME: We could put this request in a queue until the client connection is re-established.
if (m_request_client)
m_request_client->ensure_connection(url, RequestServer::CacheLevel::ResolveOnly);
}
void ResourceLoader::preconnect(URL::URL const& url)
{
if (url.scheme().is_one_of("file"sv, "data"sv))
return;
if (ContentFilter::the().is_filtered(url)) {
dbgln("ResourceLoader: Refusing to pre-connect to '{}': \033[31;1mURL was filtered\033[0m", url);
return;
}
// FIXME: We could put this request in a queue until the client connection is re-established.
if (m_request_client)
m_request_client->ensure_connection(url, RequestServer::CacheLevel::CreateConnection);
}
static ByteString sanitized_url_for_logging(URL::URL const& url)
{
if (url.scheme() == "data"sv)
return "[data URL]"sv;
return url.to_byte_string();
}
static void store_response_cookies(Page& page, URL::URL const& url, ByteString const& set_cookie_entry)
{
auto cookie = Cookie::parse_cookie(url, set_cookie_entry);
if (!cookie.has_value())
return;
page.client().page_did_set_cookie(url, cookie.value(), Cookie::Source::Http); // FIXME: Determine cookie source correctly
}
static HTTP::HeaderMap response_headers_for_file(StringView path, Optional<time_t> const& modified_time)
{
// For file:// and resource:// URLs, we have to guess the MIME type, since there's no HTTP header to tell us what
// it is. We insert a fake Content-Type header here, so that clients can use it to learn the MIME type.
auto mime_type = Core::guess_mime_type_based_on_filename(path);
HTTP::HeaderMap response_headers;
response_headers.set("Access-Control-Allow-Origin"sv, "null"sv);
response_headers.set("Content-Type"sv, mime_type);
if (modified_time.has_value()) {
auto const datetime = AK::UnixDateTime::from_seconds_since_epoch(modified_time.value());
response_headers.set("Last-Modified"sv, datetime.to_byte_string("%a, %d %b %Y %H:%M:%S GMT"sv, AK::UnixDateTime::LocalTime::No));
}
return response_headers;
}
static void log_request_start(LoadRequest const& request)
{
auto url_for_logging = sanitized_url_for_logging(*request.url());
dbgln_if(SPAM_DEBUG, "ResourceLoader: Starting load of: \"{}\"", url_for_logging);
}
static void log_success(LoadRequest const& request)
{
auto url_for_logging = sanitized_url_for_logging(*request.url());
auto load_time_ms = request.load_time().to_milliseconds();
dbgln_if(SPAM_DEBUG, "ResourceLoader: Finished load of: \"{}\", Duration: {}ms", url_for_logging, load_time_ms);
}
template<typename ErrorType>
static void log_failure(LoadRequest const& request, ErrorType const& error)
{
auto url_for_logging = sanitized_url_for_logging(*request.url());
auto load_time_ms = request.load_time().to_milliseconds();
dbgln("ResourceLoader: Failed load of: \"{}\", \033[31;1mError: {}\033[0m, Duration: {}ms", url_for_logging, error, load_time_ms);
}
static void log_filtered_request(LoadRequest const& request)
{
auto url_for_logging = sanitized_url_for_logging(*request.url());
dbgln("ResourceLoader: Filtered request to: \"{}\"", url_for_logging);
}
static bool should_block_request(LoadRequest const& request)
{
auto const& url = request.url().value();
auto is_port_blocked = [](int port) {
static constexpr auto ports = to_array({ 1, 7, 9, 11, 13, 15, 17, 19, 20, 21, 22, 23, 25, 37, 42,
43, 53, 77, 79, 87, 95, 101, 102, 103, 104, 109, 110, 111, 113, 115, 117, 119, 123, 135, 139,
143, 179, 389, 465, 512, 513, 514, 515, 526, 530, 531, 532, 540, 556, 563, 587, 601, 636,
993, 995, 2049, 3659, 4045, 6000, 6379, 6665, 6666, 6667, 6668, 6669 });
return ports.first_index_of(port).has_value();
};
if (is_port_blocked(url.port_or_default())) {
log_failure(request, ByteString::formatted("Port #{} is blocked", url.port_or_default()));
return true;
}
if (ContentFilter::the().is_filtered(url)) {
log_filtered_request(request);
return true;
}
return false;
}
template<typename FileHandler, typename ErrorHandler>
void ResourceLoader::handle_file_load_request(LoadRequest& request, FileHandler on_file, ErrorHandler on_error)
{
auto page = request.page();
if (!page) {
auto const error_message = ByteString("INTERNAL ERROR: No Page for file scheme request"sv);
on_error(error_message);
return;
}
auto const& url = request.url().value();
FileRequest file_request(url.file_path(), [this, request, on_file, on_error, url](ErrorOr<i32> file_or_error) mutable {
--m_pending_loads;
if (on_load_counter_change)
on_load_counter_change();
if (file_or_error.is_error()) {
auto const message = ByteString::formatted("{}", file_or_error.error());
on_error(message);
return;
}
auto const fd = file_or_error.value();
auto maybe_is_valid_directory = Core::Directory::is_valid_directory(fd);
if (!maybe_is_valid_directory.is_error() && maybe_is_valid_directory.value()) {
auto maybe_response = load_file_directory_page(url);
if (maybe_response.is_error()) {
auto const message = ByteString::formatted("{}", maybe_response.error());
on_error(message);
return;
}
FileLoadResult load_result;
load_result.data = maybe_response.value().bytes();
load_result.response_headers.set("Content-Type"sv, "text/html"sv);
on_file(load_result);
return;
}
auto st_or_error = Core::System::fstat(fd);
if (st_or_error.is_error()) {
on_error(ByteString::formatted("{}", st_or_error.error()));
return;
}
auto maybe_file = Core::File::adopt_fd(fd, Core::File::OpenMode::Read);
if (maybe_file.is_error()) {
on_error(ByteString::formatted("{}", maybe_file.error()));
return;
}
auto file = maybe_file.release_value();
auto maybe_data = file->read_until_eof();
if (maybe_data.is_error()) {
on_error(ByteString::formatted("{}", maybe_data.error()));
return;
}
FileLoadResult load_result;
load_result.data = maybe_data.value().bytes();
load_result.response_headers = response_headers_for_file(request.url()->file_path(), st_or_error.value().st_mtime);
on_file(load_result);
});
page->client().request_file(move(file_request));
++m_pending_loads;
if (on_load_counter_change)
on_load_counter_change();
}
template<typename Callback>
void ResourceLoader::handle_about_load_request(LoadRequest const& request, Callback callback)
{
auto const& url = request.url().value();
dbgln_if(SPAM_DEBUG, "Loading about: URL {}", url);
HTTP::HeaderMap response_headers;
response_headers.set("Content-Type"sv, "text/html; charset=UTF-8"sv);
// FIXME: Implement timing info for about requests.
Requests::RequestTimingInfo timing_info {};
auto serialized_path = URL::percent_decode(url.serialize_path());
// About version page
if (serialized_path == "version") {
auto version_page = MUST(load_about_version_page());
callback(version_page.bytes(), timing_info, response_headers);
return;
}
// Other about static HTML pages
auto target_file = ByteString::formatted("{}.html", serialized_path);
auto about_directory = MUST(Core::Resource::load_from_uri("resource://ladybird/about-pages"_string));
if (about_directory->children().contains_slow(target_file.view())) {
auto resource = Core::Resource::load_from_uri(ByteString::formatted("resource://ladybird/about-pages/{}", target_file));
if (!resource.is_error()) {
auto const& buffer = resource.value()->data();
ReadonlyBytes data(buffer.data(), buffer.size());
callback(data, timing_info, response_headers);
return;
}
}
Platform::EventLoopPlugin::the().deferred_invoke(GC::create_function(
m_heap,
[callback, timing_info, response_headers = move(response_headers)]() mutable {
auto buffer = ByteString::empty().to_byte_buffer();
callback(buffer.bytes(), timing_info, response_headers);
}));
}
template<typename ResourceHandler, typename ErrorHandler>
void ResourceLoader::handle_resource_load_request(LoadRequest const& request, ResourceHandler on_resource, ErrorHandler on_error)
{
auto const& url = request.url().value();
auto resource = Core::Resource::load_from_uri(url.serialize());
if (resource.is_error()) {
on_error(ByteString::formatted("{}", resource.error()));
return;
}
auto resource_value = resource.release_value();
// When resource URI is a directory use file directory loader to generate response
if (resource_value->is_directory()) {
auto directory_url = URL::Parser::basic_parse(resource_value->file_url());
VERIFY(directory_url.has_value());
auto maybe_response = load_file_directory_page(directory_url.release_value());
if (maybe_response.is_error()) {
on_error(ByteString::formatted("{}", maybe_response.error()));
return;
}
FileLoadResult load_result;
load_result.data = maybe_response.release_value().bytes();
load_result.response_headers.set("Content-Type"sv, "text/html"sv);
on_resource(load_result);
return;
}
auto const& buffer = resource_value->data();
auto response_headers = response_headers_for_file(url.file_path(), resource_value->modified_time());
// FIXME: Implement timing info for resource requests.
Requests::RequestTimingInfo timing_info {};
FileLoadResult load_result;
load_result.data = buffer;
load_result.response_headers = move(response_headers);
load_result.timing_info = timing_info;
on_resource(load_result);
}
void ResourceLoader::load(LoadRequest& request, GC::Root<SuccessCallback> success_callback, GC::Root<ErrorCallback> error_callback, Optional<u32> timeout, GC::Root<TimeoutCallback> timeout_callback)
{
auto const& url = request.url().value();
log_request_start(request);
request.start_timer();
if (should_block_request(request)) {
error_callback->function()("Request was blocked", {}, {}, {}, {}, {});
return;
}
if (url.scheme() == "about") {
handle_about_load_request(request, [success_callback, request](ReadonlyBytes data, Requests::RequestTimingInfo const& timing_info, HTTP::HeaderMap const& response_headers) {
log_success(request);
success_callback->function()(data, timing_info, response_headers, {}, {});
});
return;
}
if (url.scheme() == "data") {
auto data_url_or_error = Fetch::Infrastructure::process_data_url(url);
if (data_url_or_error.is_error()) {
auto error_message = data_url_or_error.error().string_literal();
log_failure(request, error_message);
error_callback->function()(error_message, {}, {}, {}, {}, {});
return;
}
auto data_url = data_url_or_error.release_value();
dbgln_if(SPAM_DEBUG, "ResourceLoader loading a data URL with mime-type: '{}', payload='{}'",
data_url.mime_type.serialized(),
StringView(data_url.body.bytes()));
HTTP::HeaderMap response_headers;
response_headers.set("Content-Type", data_url.mime_type.serialized().to_byte_string());
log_success(request);
Platform::EventLoopPlugin::the().deferred_invoke(GC::create_function(m_heap, [data = move(data_url.body), response_headers = move(response_headers), success_callback] {
// FIXME: Implement timing info for data requests.
Requests::RequestTimingInfo fixme_implement_timing_info {};
success_callback->function()(data, fixme_implement_timing_info, response_headers, {}, {});
}));
return;
}
if (url.scheme() == "resource") {
handle_resource_load_request(
request,
[success_callback, request](FileLoadResult const& load_result) {
log_success(request);
success_callback->function()(load_result.data, load_result.timing_info, load_result.response_headers, {}, {});
},
[error_callback, request](ByteString const& error_message) {
log_failure(request, error_message);
if (error_callback)
error_callback->function()(error_message, {}, {}, {}, {}, {});
});
return;
}
if (url.scheme() == "file") {
handle_file_load_request(
request,
[success_callback, request](FileLoadResult const& load_result) {
log_success(request);
success_callback->function()(load_result.data, load_result.timing_info, load_result.response_headers, {}, {});
},
[error_callback, request](ByteString const& error_message) {
log_failure(request, error_message);
if (error_callback)
error_callback->function()(error_message, {}, {}, {}, {}, {});
});
return;
}
if (url.scheme() == "http" || url.scheme() == "https") {
auto protocol_request = start_network_request(request);
if (!protocol_request) {
if (error_callback)
error_callback->function()("Failed to start network request"sv, {}, {}, {}, {}, {});
return;
}
if (timeout.has_value() && timeout.value() > 0) {
auto timer = Platform::Timer::create_single_shot(m_heap, timeout.value(), nullptr);
timer->on_timeout = GC::create_function(m_heap, [timer = GC::make_root(timer), protocol_request, timeout_callback] {
(void)timer;
protocol_request->stop();
if (timeout_callback)
timeout_callback->function()();
});
timer->start();
}
auto on_buffered_request_finished = [this, success_callback, error_callback, request, &protocol_request = *protocol_request](auto, auto const& timing_info, auto const& network_error, auto& response_headers, auto status_code, auto const& reason_phrase, ReadonlyBytes payload) mutable {
handle_network_response_headers(request, response_headers);
// NOTE: We finish the network request *after* invoking callbacks, otherwise a nested
// event loop inside a callback may cause this function object to be destroyed
// while we're still calling it.
ScopeGuard cleanup = [&] {
deferred_invoke([this, protocol_request = NonnullRefPtr(protocol_request)] {
finish_network_request(move(protocol_request));
});
};
if (network_error.has_value() || (status_code.has_value() && *status_code >= 400 && *status_code <= 599 && (payload.is_empty() || !request.is_main_resource()))) {
StringBuilder error_builder;
if (network_error.has_value())
error_builder.appendff("{}", Requests::network_error_to_string(*network_error));
else
error_builder.append("Load failed"sv);
if (status_code.has_value()) {
if (*status_code >= 100 && *status_code <= 599)
error_builder.appendff(" (status: {} {})", *status_code, HTTP::HttpResponse::reason_phrase_for_code(*status_code));
else
error_builder.appendff(" (status: {})", *status_code);
}
log_failure(request, error_builder.string_view());
if (error_callback)
error_callback->function()(error_builder.to_byte_string(), timing_info, status_code, reason_phrase, payload, response_headers);
return;
}
log_success(request);
success_callback->function()(payload, timing_info, response_headers, status_code, reason_phrase);
};
protocol_request->set_buffered_request_finished_callback(move(on_buffered_request_finished));
return;
}
auto not_implemented_error = ByteString::formatted("Protocol not implemented: {}", url.scheme());
log_failure(request, not_implemented_error);
if (error_callback) {
error_callback->function()(not_implemented_error, {}, {}, {}, {}, {});
}
}
void ResourceLoader::load_unbuffered(LoadRequest& request, GC::Root<OnHeadersReceived> on_headers_received, GC::Root<OnDataReceived> on_data_received, GC::Root<OnComplete> on_complete)
{
auto const& url = request.url().value();
log_request_start(request);
request.start_timer();
if (should_block_request(request)) {
on_complete->function()(false, {}, "Request was blocked"sv);
return;
}
if (url.scheme() == "about"sv) {
handle_about_load_request(request, [on_headers_received, on_data_received, on_complete, request](ReadonlyBytes data, Requests::RequestTimingInfo const& timing_info, HTTP::HeaderMap const& response_headers) {
log_success(request);
on_headers_received->function()(response_headers, {}, {});
on_data_received->function()(data);
on_complete->function()(true, timing_info, {});
});
return;
}
if (url.scheme() == "resource"sv) {
handle_resource_load_request(
request,
[on_headers_received, on_data_received, on_complete](FileLoadResult const& load_result) {
on_headers_received->function()(load_result.response_headers, {}, {});
on_data_received->function()(load_result.data);
on_complete->function()(true, load_result.timing_info, {});
},
[on_complete](ByteString const& message) {
Requests::RequestTimingInfo fixme_implement_timing_info {};
on_complete->function()(false, fixme_implement_timing_info, StringView(message));
});
return;
}
if (url.scheme() == "file"sv) {
handle_file_load_request(
request,
[request, on_headers_received, on_data_received, on_complete](FileLoadResult const& load_result) {
log_success(request);
on_headers_received->function()(load_result.response_headers, {}, {});
on_data_received->function()(load_result.data);
on_complete->function()(true, load_result.timing_info, {});
},
[on_complete, request](ByteString const& message) {
log_failure(request, message);
on_complete->function()(false, {}, StringView(message));
});
return;
}
if (!url.scheme().is_one_of("http"sv, "https"sv)) {
auto not_implemented_error = ByteString::formatted("Protocol not implemented: {}", url.scheme());
log_failure(request, not_implemented_error);
on_complete->function()(false, {}, not_implemented_error);
return;
}
auto protocol_request = start_network_request(request);
if (!protocol_request) {
on_complete->function()(false, {}, "Failed to start network request"sv);
return;
}
auto protocol_headers_received = [this, on_headers_received, request](auto const& response_headers, auto status_code, auto const& reason_phrase) {
handle_network_response_headers(request, response_headers);
on_headers_received->function()(response_headers, move(status_code), reason_phrase);
};
auto protocol_data_received = [on_data_received](auto data) {
on_data_received->function()(data);
};
auto protocol_complete = [this, on_complete, request, &protocol_request = *protocol_request](u64, Requests::RequestTimingInfo const& timing_info, Optional<Requests::NetworkError> const& network_error) {
finish_network_request(protocol_request);
if (!network_error.has_value()) {
log_success(request);
on_complete->function()(true, timing_info, {});
} else {
log_failure(request, "Request finished with error"sv);
on_complete->function()(false, timing_info, "Request finished with error"sv);
}
};
protocol_request->set_unbuffered_request_callbacks(move(protocol_headers_received), move(protocol_data_received), move(protocol_complete));
}
RefPtr<Requests::Request> ResourceLoader::start_network_request(LoadRequest const& request)
{
auto proxy = ProxyMappings::the().proxy_for_url(request.url().value());
HTTP::HeaderMap headers;
for (auto const& it : request.headers()) {
headers.set(it.key, it.value);
}
if (!headers.contains("User-Agent"))
headers.set("User-Agent", m_user_agent.to_byte_string());
// FIXME: We could put this request in a queue until the client connection is re-established.
if (!m_request_client) {
log_failure(request, "RequestServer is currently unavailable"sv);
return nullptr;
}
auto protocol_request = m_request_client->start_request(request.method(), request.url().value(), headers, request.body(), proxy);
if (!protocol_request) {
log_failure(request, "Failed to initiate load"sv);
return nullptr;
}
protocol_request->on_certificate_requested = []() -> Requests::Request::CertificateAndKey {
return {};
};
++m_pending_loads;
if (on_load_counter_change)
on_load_counter_change();
m_active_requests.set(*protocol_request);
return protocol_request;
}
void ResourceLoader::handle_network_response_headers(LoadRequest const& request, HTTP::HeaderMap const& response_headers)
{
if (!request.page())
return;
if (request.store_set_cookie_headers()) {
// From https://fetch.spec.whatwg.org/#concept-http-network-fetch:
// 15. If includeCredentials is true, then the user agent should parse and store response
// `Set-Cookie` headers given request and response.
for (auto const& [header, value] : response_headers.headers()) {
if (header.equals_ignoring_ascii_case("Set-Cookie"sv)) {
store_response_cookies(*request.page(), request.url().value(), value);
}
}
}
}
void ResourceLoader::finish_network_request(NonnullRefPtr<Requests::Request> protocol_request)
{
--m_pending_loads;
if (on_load_counter_change)
on_load_counter_change();
deferred_invoke([this, protocol_request = move(protocol_request)] {
auto did_remove = m_active_requests.remove(protocol_request);
VERIFY(did_remove);
});
}
}
Reference in a new issue View git blame Copy permalink