Stowage/ladybird
2
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-12-07 21:59:54 +00:00
Code Issues Projects Releases Packages Wiki Activity

LibHTTP+LibWeb: Use LibHTTP's cache implementation in LibWeb

Browse source 
There are a couple of remaining RFC 9111 methods in LibWeb's Fetch, but
these are currently directly tied to the way we store GC-allocated HTTP
response objects. So de-coupling that is left as a future exercise.
This commit is contained in:
Timothy Flynn 2025-11-28 10:39:53 -05:00 • committed by Tim Flynn
parent 21bbbacd07
commit 2453f0bc04
Notes: github-actions[bot] 2025-11-29 13:36:00 +00:00
6 changed files with 43 additions and 257 deletions
Download patch file Download diff file Expand all files Collapse all files

9
Libraries/LibHTTP/Cache/Utilities.cpp
View file

@ -374,6 +374,15 @@ RevalidationAttributes RevalidationAttributes::create(HeaderList const& headers)
return attributes;
}
// https://httpwg.org/specs/rfc9111.html#storing.fields
void store_header_and_trailer_fields(HeaderList& stored_headers, HeaderList const& response_headers)
{
for (auto const& header : response_headers) {
if (!is_header_exempted_from_storage(header.name))
stored_headers.append(header);
}
}
// https://httpwg.org/specs/rfc9111.html#update
void update_header_fields(HeaderList& stored_headers, HeaderList const& updated_headers)
{

5
Libraries/LibHTTP/Cache/Utilities.h
View file

@ -28,8 +28,8 @@ bool is_cacheable(StringView method);
bool is_cacheable(u32 status_code, HeaderList const&);
bool is_header_exempted_from_storage(StringView name);
AK::Duration calculate_freshness_lifetime(u32 status_code, HeaderList const&, AK::Duration current_time_offset_for_testing);
AK::Duration calculate_age(HeaderList const&, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing);
AK::Duration calculate_freshness_lifetime(u32 status_code, HeaderList const&, AK::Duration current_time_offset_for_testing = {});
AK::Duration calculate_age(HeaderList const&, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing = {});
enum class CacheLifetimeStatus {
Fresh,
@ -45,6 +45,7 @@ struct RevalidationAttributes {
Optional<UnixDateTime> last_modified;
};
void store_header_and_trailer_fields(HeaderList&, HeaderList const&);
void update_header_fields(HeaderList&, HeaderList const&);
AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&>, HeaderList const& request_headers);

2
Libraries/LibWeb/DOM/Document.cpp
View file

@ -350,7 +350,7 @@ WebIDL::ExceptionOr<GC::Ref<Document>> Document::create_and_initialize(Type type
DOM::DocumentLoadTimingInfo load_timing_info;
// AD-HOC: The response object no longer has an associated timing info object. For now, we use response's non-standard response time property,
// which represents the time that the time that the response object was created.
auto response_creation_time = navigation_params.response->response_time().nanoseconds() / 1e6;
auto response_creation_time = navigation_params.response->monotonic_response_time().nanoseconds() / 1e6;
load_timing_info.navigation_start_time = HighResolutionTime::coarsen_time(response_creation_time, HTML::relevant_settings_object(*window).cross_origin_isolated_capability() == HTML::CanUseCrossOriginIsolatedAPIs::Yes);
// 9. Let document be a new Document, with

189
Libraries/LibWeb/Fetch/Fetching/Fetching.cpp
View file

@ -12,6 +12,7 @@
#include <AK/Base64.h>
#include <AK/Debug.h>
#include <AK/ScopeGuard.h>
#include <LibHTTP/Cache/Utilities.h>
#include <LibHTTP/Method.h>
#include <LibJS/Runtime/Completion.h>
#include <LibRequests/RequestTimingInfo.h>
@ -1416,7 +1417,7 @@ GC::Ptr<PendingResponse> http_redirect_fetch(JS::Realm& realm, Infrastructure::F
class CachePartition : public RefCounted<CachePartition> {
public:
// https://httpwg.org/specs/rfc9111.html#constructing.responses.from.caches
GC::Ptr<Infrastructure::Response> select_response(JS::Realm& realm, URL::URL const& url, StringView method, HTTP::HeaderList const& headers, Vector<GC::Ptr<Infrastructure::Response>>& initial_set_of_stored_responses) const
GC::Ptr<Infrastructure::Response> select_response(JS::Realm& realm, URL::URL const& url, StringView method, HTTP::HeaderList const& headers) const
{
// When presented with a request, a cache MUST NOT reuse a stored response unless:
@ -1439,8 +1440,6 @@ public:
// FIXME: - the stored response does not contain the no-cache directive (Section 5.2.2.4), unless it is successfully validated (Section 4.3), and
initial_set_of_stored_responses.append(*cached_response);
// FIXME: - the stored response is one of the following:
// + fresh (see Section 4.2), or
// + allowed to be served stale (see Section 4.2.4), or
@ -1452,190 +1451,23 @@ public:
void store_response(JS::Realm& realm, Infrastructure::Request const& http_request, Infrastructure::Response const& response)
{
if (!is_cacheable(http_request, response))
if (!HTTP::is_cacheable(http_request.method()))
return;
if (!HTTP::is_cacheable(response.status(), response.header_list()))
return;
auto cached_response = Infrastructure::Response::create(realm.vm());
store_header_and_trailer_fields(response, *cached_response->header_list());
HTTP::store_header_and_trailer_fields(cached_response->header_list(), response.header_list());
cached_response->set_body(response.body()->clone(realm));
cached_response->set_body_info(response.body_info());
cached_response->set_method(http_request.method());
cached_response->set_status(response.status());
cached_response->url_list().append(http_request.current_url());
m_cache.set(http_request.current_url(), move(cached_response));
}
// https://httpwg.org/specs/rfc9111.html#freshening.responses
void freshen_stored_responses_upon_validation(Infrastructure::Response const& response, Vector<GC::Ptr<Infrastructure::Response>>& initial_set_of_stored_responses)
{
// When a cache receives a 304 (Not Modified) response, it needs to identify stored
// responses that are suitable for updating with the new information provided, and then do so.
// The initial set of stored responses to update are those that could have been
// chosen for that request — i.e., those that meet the requirements in Section 4,
// except the last requirement to be fresh, able to be served stale, or just validated.
for (auto stored_response : initial_set_of_stored_responses) {
// Then, that initial set of stored responses is further filtered by the first match of:
// - FIXME: If the new response contains one or more strong validators (see Section 8.8.1 of [HTTP]),
// then each of those strong validators identifies a selected representation for update.
// All the stored responses in the initial set with one of those same strong validators
// are identified for update.
// If none of the initial set contains at least one of the same strong validators,
// then the cache MUST NOT use the new response to update any stored responses.
// - FIXME: If the new response contains no strong validators but does contain one or more weak validators,
// and those validators correspond to one of the initial set's stored responses,
// then the most recent of those matching stored responses is identified for update.
// - FIXME: If the new response does not include any form of validator (such as where a client generates an
// `If-Modified-Since` request from a source other than the `Last-Modified` response header field),
// and there is only one stored response in the initial set, and that stored response also lacks a validator,
// then that stored response is identified for update.
// For each stored response identified, the cache MUST update its header fields
// with the header fields provided in the 304 (Not Modified) response, as per Section 3.2.
update_stored_header_fields(response, stored_response->header_list());
}
m_cache.set(http_request.current_url(), cached_response);
}
private:
// https://httpwg.org/specs/rfc9111.html#storing.fields
bool is_exempted_for_storage(StringView header_name)
{
// Caches MUST include all received response header fields — including unrecognized ones — when storing a response;
// this assures that new HTTP header fields can be successfully deployed. However, the following exceptions are made:
// - The Connection header field and fields whose names are listed in it are required by Section 7.6.1 of [HTTP]
// to be removed before forwarding the message. This MAY be implemented by doing so before storage.
// - Likewise, some fields' semantics require them to be removed before forwarding the message, and this MAY be
// implemented by doing so before storage; see Section 7.6.1 of [HTTP] for some examples.
// FIXME: - The no-cache (Section 5.2.2.4) and private (Section 5.2.2.7) cache directives can have arguments that
// prevent storage of header fields by all caches and shared caches, respectively.
// FIXME: - Header fields that are specific to the proxy that a cache uses when forwarding a request MUST NOT be stored,
// unless the cache incorporates the identity of the proxy into the cache key.
// Effectively, this is limited to Proxy-Authenticate (Section 11.7.1 of [HTTP]), Proxy-Authentication-Info (Section 11.7.3 of [HTTP]), and Proxy-Authorization (Section 11.7.2 of [HTTP]).
return header_name.is_one_of_ignoring_ascii_case(
"Connection"sv,
"Proxy-Connection"sv,
"Keep-Alive"sv,
"TE"sv,
"Transfer-Encoding"sv,
"Upgrade"sv);
}
// https://httpwg.org/specs/rfc9111.html#update
bool is_exempted_for_updating(StringView header_name)
{
// Caches are required to update a stored response's header fields from another
// (typically newer) response in several situations; for example, see Sections 3.4, 4.3.4, and 4.3.5.
// When doing so, the cache MUST add each header field in the provided response to the stored response,
// replacing field values that are already present, with the following exceptions:
// - Header fields excepted from storage in Section 3.1,
return is_exempted_for_storage(header_name)
// - Header fields that the cache's stored response depends upon, as described below,
|| false
// - Header fields that are automatically processed and removed by the recipient, as described below, and
|| false
// - The Content-Length header field.
|| header_name.equals_ignoring_ascii_case("Content-Length"sv);
// In some cases, caches (especially in user agents) store the results of processing
// the received response, rather than the response itself, and updating header fields
// that affect that processing can result in inconsistent behavior and security issues.
// Caches in this situation MAY omit these header fields from updating stored responses
// on an exceptional basis but SHOULD limit such omission to those fields necessary to
// assure integrity of the stored response.
// For example, a browser might decode the content coding of a response while it is being received,
// creating a disconnect between the data it has stored and the response's original metadata.
// Updating that stored metadata with a different Content-Encoding header field would be problematic.
// Likewise, a browser might store a post-parse HTML tree rather than the content received in the response;
// updating the Content-Type header field would not be workable in this case because any assumptions about
// the format made in parsing would now be invalid.
// Furthermore, some fields are automatically processed and removed by the HTTP implementation,
// such as the Content-Range header field. Implementations MAY automatically omit such header fields from updates,
// even when the processing does not actually occur.
// Note that the Content-* prefix is not a signal that a header field is omitted from update; it is a convention for MIME header fields, not HTTP.
}
// https://httpwg.org/specs/rfc9111.html#update
void update_stored_header_fields(Infrastructure::Response const& response, HTTP::HeaderList& headers)
{
for (auto const& header : *response.header_list()) {
if (!is_exempted_for_updating(header.name))
headers.delete_(header.name);
}
for (auto const& header : *response.header_list()) {
if (!is_exempted_for_updating(header.name))
headers.append(header);
}
}
// https://httpwg.org/specs/rfc9111.html#storing.fields
void store_header_and_trailer_fields(Infrastructure::Response const& response, HTTP::HeaderList& headers)
{
for (auto const& header : *response.header_list()) {
if (!is_exempted_for_storage(header.name))
headers.append(header);
}
}
// https://httpwg.org/specs/rfc9111.html#response.cacheability
static bool is_cacheable(Infrastructure::Request const& request, Infrastructure::Response const& response)
{
// A cache MUST NOT store a response to a request unless:
// - AD-HOC: For now, we simply don't cache responses without a simple ByteBuffer body.
if (!response.body() || !response.body()->source().has<ByteBuffer>())
return false;
// - the request method is understood by the cache;
if (request.method() != "GET"sv && request.method() != "HEAD"sv)
return false;
// - the response status code is final (see Section 15 of [HTTP]);
if (response.status() < 200)
return false;
// - if the response status code is 206 or 304,
// or the must-understand cache directive (see Section 5.2.2.3) is present:
// the cache understands the response status code;
if (response.status() == 206 || response.status() == 304) {
// FIXME: Implement must-understand cache directive
}
// - the no-store cache directive is not present in the response (see Section 5.2.2.5);
if (request.cache_mode() == Infrastructure::Request::CacheMode::NoStore)
return false;
// FIXME: - if the cache is shared: the private response directive is either not present
// or allows a shared cache to store a modified response; see Section 5.2.2.7);
// FIXME: - if the cache is shared: the Authorization header field is not present in the
// request (see Section 11.6.2 of [HTTP]) or a response directive is present
// that explicitly allows shared caching (see Section 3.5); and
// FIXME: - the response contains at least one of the following:
// + a public response directive (see Section 5.2.2.9);
// + a private response directive, if the cache is not shared (see Section 5.2.2.7);
// + an Expires header field (see Section 5.3);
// + a max-age response directive (see Section 5.2.2.1);
// + if the cache is shared: an s-maxage response directive (see Section 5.2.2.10);
// + a cache extension that allows it to be cached (see Section 5.2.3); or
// + a status code that is defined as heuristically cacheable (see Section 4.2.2).
return true;
}
HashMap<URL::URL, GC::Root<Infrastructure::Response>> m_cache;
};
@ -1702,7 +1534,6 @@ GC::Ref<PendingResponse> http_network_or_cache_fetch(JS::Realm& realm, Infrastru
// 5. Let storedResponse be null.
GC::Ptr<Infrastructure::Response> stored_response;
GC::RootVector<GC::Ptr<Infrastructure::Response>> initial_set_of_stored_responses(realm.heap());
// 6. Let httpCache be null.
// (Typeless until we actually implement it, needed for checks below)
@ -1971,7 +1802,7 @@ GC::Ref<PendingResponse> http_network_or_cache_fetch(JS::Realm& realm, Infrastru
// validation, as per the "Constructing Responses from Caches" chapter of HTTP Caching [HTTP-CACHING],
// if any.
// NOTE: As mandated by HTTP, this still takes the `Vary` header into account.
stored_response = http_cache->select_response(realm, http_request->current_url(), http_request->method(), *http_request->header_list(), initial_set_of_stored_responses);
stored_response = http_cache->select_response(realm, http_request->current_url(), http_request->method(), *http_request->header_list());
// 2. If storedResponse is non-null, then:
if (stored_response) {
@ -2056,7 +1887,7 @@ GC::Ref<PendingResponse> http_network_or_cache_fetch(JS::Realm& realm, Infrastru
auto returned_pending_response = PendingResponse::create(vm, request);
pending_forward_response->when_loaded([&realm, &vm, &fetch_params, request, response, stored_response, initial_set_of_stored_responses, http_request, returned_pending_response, is_authentication_fetch, is_new_connection_fetch, revalidating_flag, include_credentials, response_was_null = !response, http_cache](GC::Ref<Infrastructure::Response> resolved_forward_response) mutable {
pending_forward_response->when_loaded([&realm, &vm, &fetch_params, request, response, stored_response, http_request, returned_pending_response, is_authentication_fetch, is_new_connection_fetch, revalidating_flag, include_credentials, response_was_null = !response, http_cache](GC::Ref<Infrastructure::Response> resolved_forward_response) mutable {
dbgln_if(WEB_FETCH_DEBUG, "Fetch: Running 'HTTP-network-or-cache fetch' pending_forward_response load callback");
if (response_was_null) {
auto forward_response = resolved_forward_response;
@ -2079,7 +1910,7 @@ GC::Ref<PendingResponse> http_network_or_cache_fetch(JS::Realm& realm, Infrastru
// 1. Update storedResponse’s header list using forwardResponse’s header list, as per the "Freshening
// Stored Responses upon Validation" chapter of HTTP Caching.
// NOTE: This updates the stored response in cache as well.
http_cache->freshen_stored_responses_upon_validation(*forward_response, initial_set_of_stored_responses);
HTTP::update_header_fields(stored_response->header_list(), forward_response->header_list());
// 2. Set response to storedResponse.
response = stored_response;

81
Libraries/LibWeb/Fetch/Infrastructure/HTTP/Responses.cpp
View file

@ -7,6 +7,7 @@
#include <AK/Debug.h>
#include <AK/TypeCasts.h>
#include <LibGC/Heap.h>
#include <LibHTTP/Cache/Utilities.h>
#include <LibJS/Runtime/Completion.h>
#include <LibJS/Runtime/VM.h>
#include <LibWeb/Bindings/MainThreadVM.h>
@ -32,7 +33,8 @@ GC::Ref<Response> Response::create(JS::VM& vm)
Response::Response(NonnullRefPtr<HTTP::HeaderList> header_list)
: m_header_list(move(header_list))
, m_response_time(MonotonicTime::now())
, m_response_time(UnixDateTime::now())
, m_monotonic_response_time(MonotonicTime::now())
{
}
@ -242,82 +244,25 @@ bool Response::is_stale() const
return !is_fresh() && !is_stale_while_revalidate();
}
// https://httpwg.org/specs/rfc9111.html#age.calculations
u64 Response::current_age() const
AK::Duration Response::current_age() const
{
// The term "age_value" denotes the value of the Age header field (Section 5.1), in a form appropriate for arithmetic operation; or 0, if not available.
Optional<AK::Duration> age;
if (auto const age_header = header_list()->get("Age"sv); age_header.has_value()) {
if (auto converted_age = age_header->to_number<u64>(); converted_age.has_value())
age = AK::Duration::from_seconds(converted_age.value());
}
auto const age_value = age.value_or(AK::Duration::from_seconds(0));
// The term "date_value" denotes the value of the Date header field, in a form appropriate for arithmetic operations. See Section 6.6.1 of [HTTP] for the definition of the Date header field and for requirements regarding responses without it.
// FIXME: Do we have a parser for HTTP-date?
auto const date_value = MonotonicTime::now() - AK::Duration::from_seconds(5);
// The term "now" means the current value of this implementation's clock (Section 5.6.7 of [HTTP]).
auto const now = MonotonicTime::now();
// The value of the clock at the time of the request that resulted in the stored response.
// FIXME: Let's get the correct time.
auto const request_time = MonotonicTime::now() - AK::Duration::from_seconds(5);
auto const request_time = UnixDateTime::now() - AK::Duration::from_seconds(5);
// The value of the clock at the time the response was received.
auto const response_time = m_response_time;
auto const apparent_age = max(0, (response_time - date_value).to_seconds());
auto const response_delay = response_time - request_time;
auto const corrected_age_value = age_value + response_delay;
auto const corrected_initial_age = max(apparent_age, corrected_age_value.to_seconds());
auto const resident_time = (now - response_time).to_seconds();
return corrected_initial_age + resident_time;
return HTTP::calculate_age(m_header_list, request_time, m_response_time);
}
// https://httpwg.org/specs/rfc9111.html#calculating.freshness.lifetime
u64 Response::freshness_lifetime() const
AK::Duration Response::freshness_lifetime() const
{
auto const elem = header_list()->get_decode_and_split("Cache-Control"sv);
if (!elem.has_value())
return 0;
// FIXME: If the cache is shared and the s-maxage response directive (Section 5.2.2.10) is present, use its value
// If the max-age response directive (Section 5.2.2.1) is present, use its value, or
for (auto const& directive : *elem) {
if (directive.starts_with_bytes("max-age"sv)) {
auto equal_offset = directive.find_byte_offset('=');
if (!equal_offset.has_value()) {
dbgln("Bogus directive: '{}'", directive);
continue;
}
auto const value_string = directive.bytes_as_string_view().substring_view(equal_offset.value() + 1);
auto maybe_value = value_string.to_number<u64>();
if (!maybe_value.has_value()) {
dbgln("Bogus directive: '{}'", directive);
continue;
}
return maybe_value.value();
}
}
// FIXME: If the Expires response header field (Section 5.3) is present, use its value minus the value of the Date response header field (using the time the message was received if it is not present, as per Section 6.6.1 of [HTTP]), or
// FIXME: Otherwise, no explicit expiration time is present in the response. A heuristic freshness lifetime might be applicable; see Section 4.2.2.
return 0;
return HTTP::calculate_freshness_lifetime(m_status, m_header_list);
}
// https://httpwg.org/specs/rfc5861.html#n-the-stale-while-revalidate-cache-control-extension
u64 Response::stale_while_revalidate_lifetime() const
AK::Duration Response::stale_while_revalidate_lifetime() const
{
auto const elem = header_list()->get_decode_and_split("Cache-Control"sv);
if (!elem.has_value())
return 0;
return {};
for (auto const& directive : *elem) {
if (directive.starts_with_bytes("stale-while-revalidate"sv)) {
@ -327,16 +272,16 @@ u64 Response::stale_while_revalidate_lifetime() const
continue;
}
auto const value_string = directive.bytes_as_string_view().substring_view(equal_offset.value() + 1);
auto maybe_value = value_string.to_number<u64>();
auto maybe_value = value_string.to_number<i64>();
if (!maybe_value.has_value()) {
dbgln("Bogus directive: '{}'", directive);
continue;
}
return maybe_value.value();
return AK::Duration::from_seconds(*maybe_value);
}
}
return 0;
return {};
}
// Non-standard

14
Libraries/LibWeb/Fetch/Infrastructure/HTTP/Responses.h
View file

@ -127,7 +127,7 @@ public:
// Non-standard
[[nodiscard]] Optional<String> const& network_error_message() const { return m_network_error_message; }
MonotonicTime response_time() const { return m_response_time; }
MonotonicTime monotonic_response_time() const { return m_monotonic_response_time; }
protected:
explicit Response(NonnullRefPtr<HTTP::HeaderList>);
@ -135,6 +135,10 @@ protected:
virtual void visit_edges(JS::Cell::Visitor&) override;
private:
AK::Duration current_age() const;
AK::Duration freshness_lifetime() const;
AK::Duration stale_while_revalidate_lifetime() const;
// https://fetch.spec.whatwg.org/#concept-response-type
// A response has an associated type which is "basic", "cors", "default", "error", "opaque", or "opaqueredirect". Unless stated otherwise, it is "default".
Type m_type { Type::Default };
@ -194,14 +198,10 @@ private:
// A response has an associated redirect taint ("same-origin", "same-site", or "cross-site"), which is initially "same-origin".
RedirectTaint m_redirect_taint { RedirectTaint::SameOrigin };
// FIXME: is the type correct?
u64 current_age() const;
u64 freshness_lifetime() const;
u64 stale_while_revalidate_lifetime() const;
// Non-standard
ByteString m_method;
MonotonicTime m_response_time;
UnixDateTime m_response_time;
MonotonicTime m_monotonic_response_time;
Optional<String> m_network_error_message;