mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2025-12-07 21:59:54 +00:00
LibHTTP+RequestServer: Move the HTTP cache implementation to LibHTTP
We currently have two ongoing implementations of RFC 9111, HTTP caching. In order to consolidate these, this patch moves the implementation from RequestServer to LibHTTP for re-use within LibWeb.
This commit is contained in:
Timothy Flynn
Tim Flynn
parent
1f8a42c367
commit
21bbbacd07
Notes:
github-actions[bot]
2025-11-29 13:36:06 +00:00
Author: https://github.com/trflynn89
Commit: 21bbbacd07
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/6967
18 changed files with 240 additions and 203 deletions
|
|
@ -1,4 +1,8 @@
|
|||
set(SOURCES
|
||||
Cache/CacheEntry.cpp
|
||||
Cache/CacheIndex.cpp
|
||||
Cache/DiskCache.cpp
|
||||
Cache/Utilities.cpp
|
||||
Header.cpp
|
||||
HeaderList.cpp
|
||||
HTTP.cpp
|
||||
|
|
@ -7,4 +11,4 @@ set(SOURCES
|
|||
)
|
||||
|
||||
ladybird_lib(LibHTTP http)
|
||||
target_link_libraries(LibHTTP PRIVATE LibCompress LibCore LibIPC LibRegex LibTextCodec LibTLS LibURL)
|
||||
target_link_libraries(LibHTTP PRIVATE LibCompress LibCore LibCrypto LibDatabase LibFileSystem LibIPC LibRegex LibTextCodec LibTLS LibURL)
|
||||
|
|
|
|||
|
|
@ -6,15 +6,14 @@
|
|||
|
||||
#include <AK/HashFunctions.h>
|
||||
#include <AK/ScopeGuard.h>
|
||||
#include <LibCore/Notifier.h>
|
||||
#include <LibCore/System.h>
|
||||
#include <LibFileSystem/FileSystem.h>
|
||||
#include <RequestServer/Cache/CacheEntry.h>
|
||||
#include <RequestServer/Cache/CacheIndex.h>
|
||||
#include <RequestServer/Cache/DiskCache.h>
|
||||
#include <RequestServer/Cache/Utilities.h>
|
||||
#include <LibHTTP/Cache/CacheEntry.h>
|
||||
#include <LibHTTP/Cache/CacheIndex.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
ErrorOr<CacheHeader> CacheHeader::read_from_stream(Stream& stream)
|
||||
{
|
||||
|
|
@ -117,7 +116,7 @@ CacheEntryWriter::CacheEntryWriter(DiskCache& disk_cache, CacheIndex& index, u64
|
|||
{
|
||||
}
|
||||
|
||||
ErrorOr<void> CacheEntryWriter::write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HTTP::HeaderList const& response_headers)
|
||||
ErrorOr<void> CacheEntryWriter::write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HeaderList const& response_headers)
|
||||
{
|
||||
if (m_marked_for_deletion) {
|
||||
close_and_destroy_cache_entry();
|
||||
|
|
@ -183,7 +182,7 @@ ErrorOr<void> CacheEntryWriter::write_data(ReadonlyBytes data)
|
|||
return {};
|
||||
}
|
||||
|
||||
ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HTTP::HeaderList> response_headers)
|
||||
ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HeaderList> response_headers)
|
||||
{
|
||||
ScopeGuard guard { [&]() { close_and_destroy_cache_entry(); } };
|
||||
|
||||
|
|
@ -205,7 +204,7 @@ ErrorOr<void> CacheEntryWriter::flush(NonnullRefPtr<HTTP::HeaderList> response_h
|
|||
return {};
|
||||
}
|
||||
|
||||
ErrorOr<NonnullOwnPtr<CacheEntryReader>> CacheEntryReader::create(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, NonnullRefPtr<HTTP::HeaderList> response_headers, u64 data_size)
|
||||
ErrorOr<NonnullOwnPtr<CacheEntryReader>> CacheEntryReader::create(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, NonnullRefPtr<HeaderList> response_headers, u64 data_size)
|
||||
{
|
||||
auto path = path_for_cache_key(disk_cache.cache_directory(), cache_key);
|
||||
|
||||
|
|
@ -253,7 +252,7 @@ ErrorOr<NonnullOwnPtr<CacheEntryReader>> CacheEntryReader::create(DiskCache& dis
|
|||
return adopt_own(*new CacheEntryReader { disk_cache, index, cache_key, move(url), move(path), move(file), fd, cache_header, move(reason_phrase), move(response_headers), data_offset, data_size });
|
||||
}
|
||||
|
||||
CacheEntryReader::CacheEntryReader(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, LexicalPath path, NonnullOwnPtr<Core::File> file, int fd, CacheHeader cache_header, Optional<String> reason_phrase, NonnullRefPtr<HTTP::HeaderList> response_headers, u64 data_offset, u64 data_size)
|
||||
CacheEntryReader::CacheEntryReader(DiskCache& disk_cache, CacheIndex& index, u64 cache_key, String url, LexicalPath path, NonnullOwnPtr<Core::File> file, int fd, CacheHeader cache_header, Optional<String> reason_phrase, NonnullRefPtr<HeaderList> response_headers, u64 data_offset, u64 data_size)
|
||||
: CacheEntry(disk_cache, index, cache_key, move(url), move(path), cache_header)
|
||||
, m_file(move(file))
|
||||
, m_fd(fd)
|
||||
|
|
@ -264,7 +263,7 @@ CacheEntryReader::CacheEntryReader(DiskCache& disk_cache, CacheIndex& index, u64
|
|||
{
|
||||
}
|
||||
|
||||
void CacheEntryReader::revalidation_succeeded(HTTP::HeaderList const& response_headers)
|
||||
void CacheEntryReader::revalidation_succeeded(HeaderList const& response_headers)
|
||||
{
|
||||
dbgln("\033[34;1mCache revalidation succeeded for\033[0m {}", m_url);
|
||||
|
||||
|
|
@ -13,11 +13,12 @@
|
|||
#include <AK/Time.h>
|
||||
#include <AK/Types.h>
|
||||
#include <LibCore/File.h>
|
||||
#include <LibCore/Notifier.h>
|
||||
#include <LibHTTP/Cache/Version.h>
|
||||
#include <LibHTTP/Forward.h>
|
||||
#include <LibHTTP/HeaderList.h>
|
||||
#include <RequestServer/Cache/Version.h>
|
||||
#include <RequestServer/Forward.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
struct CacheHeader {
|
||||
static ErrorOr<CacheHeader> read_from_stream(Stream&);
|
||||
|
|
@ -87,9 +88,9 @@ public:
|
|||
static ErrorOr<NonnullOwnPtr<CacheEntryWriter>> create(DiskCache&, CacheIndex&, u64 cache_key, String url, UnixDateTime request_time, AK::Duration current_time_offset_for_testing);
|
||||
virtual ~CacheEntryWriter() override = default;
|
||||
|
||||
ErrorOr<void> write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HTTP::HeaderList const&);
|
||||
ErrorOr<void> write_status_and_reason(u32 status_code, Optional<String> reason_phrase, HeaderList const&);
|
||||
ErrorOr<void> write_data(ReadonlyBytes);
|
||||
ErrorOr<void> flush(NonnullRefPtr<HTTP::HeaderList>);
|
||||
ErrorOr<void> flush(NonnullRefPtr<HeaderList>);
|
||||
|
||||
private:
|
||||
CacheEntryWriter(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::OutputBufferedFile>, CacheHeader, UnixDateTime request_time, AK::Duration current_time_offset_for_testing);
|
||||
|
|
@ -104,24 +105,24 @@ private:
|
|||
|
||||
class CacheEntryReader : public CacheEntry {
|
||||
public:
|
||||
static ErrorOr<NonnullOwnPtr<CacheEntryReader>> create(DiskCache&, CacheIndex&, u64 cache_key, NonnullRefPtr<HTTP::HeaderList>, u64 data_size);
|
||||
static ErrorOr<NonnullOwnPtr<CacheEntryReader>> create(DiskCache&, CacheIndex&, u64 cache_key, NonnullRefPtr<HeaderList>, u64 data_size);
|
||||
virtual ~CacheEntryReader() override = default;
|
||||
|
||||
bool must_revalidate() const { return m_must_revalidate; }
|
||||
void set_must_revalidate() { m_must_revalidate = true; }
|
||||
|
||||
void revalidation_succeeded(HTTP::HeaderList const&);
|
||||
void revalidation_succeeded(HeaderList const&);
|
||||
void revalidation_failed();
|
||||
|
||||
void pipe_to(int pipe_fd, Function<void(u64 bytes_piped)> on_complete, Function<void(u64 bytes_piped)> on_error);
|
||||
|
||||
u32 status_code() const { return m_cache_header.status_code; }
|
||||
Optional<String> const& reason_phrase() const { return m_reason_phrase; }
|
||||
HTTP::HeaderList& response_headers() { return m_response_headers; }
|
||||
HTTP::HeaderList const& response_headers() const { return m_response_headers; }
|
||||
HeaderList& response_headers() { return m_response_headers; }
|
||||
HeaderList const& response_headers() const { return m_response_headers; }
|
||||
|
||||
private:
|
||||
CacheEntryReader(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::File>, int fd, CacheHeader, Optional<String> reason_phrase, NonnullRefPtr<HTTP::HeaderList>, u64 data_offset, u64 data_size);
|
||||
CacheEntryReader(DiskCache&, CacheIndex&, u64 cache_key, String url, LexicalPath, NonnullOwnPtr<Core::File>, int fd, CacheHeader, Optional<String> reason_phrase, NonnullRefPtr<HeaderList>, u64 data_offset, u64 data_size);
|
||||
|
||||
void pipe_without_blocking();
|
||||
void pipe_complete();
|
||||
|
|
@ -140,7 +141,7 @@ private:
|
|||
u64 m_bytes_piped { 0 };
|
||||
|
||||
Optional<String> m_reason_phrase;
|
||||
NonnullRefPtr<HTTP::HeaderList> m_response_headers;
|
||||
NonnullRefPtr<HeaderList> m_response_headers;
|
||||
|
||||
bool m_must_revalidate { false };
|
||||
|
||||
|
|
@ -5,15 +5,15 @@
|
|||
*/
|
||||
|
||||
#include <AK/StringBuilder.h>
|
||||
#include <RequestServer/Cache/CacheIndex.h>
|
||||
#include <RequestServer/Cache/Utilities.h>
|
||||
#include <RequestServer/Cache/Version.h>
|
||||
#include <LibHTTP/Cache/CacheIndex.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibHTTP/Cache/Version.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
static constexpr u32 CACHE_METADATA_KEY = 12389u;
|
||||
|
||||
static ByteString serialize_headers(HTTP::HeaderList const& headers)
|
||||
static ByteString serialize_headers(HeaderList const& headers)
|
||||
{
|
||||
StringBuilder builder;
|
||||
|
||||
|
|
@ -27,9 +27,9 @@ static ByteString serialize_headers(HTTP::HeaderList const& headers)
|
|||
return builder.to_byte_string();
|
||||
}
|
||||
|
||||
static NonnullRefPtr<HTTP::HeaderList> deserialize_headers(StringView serialized_headers)
|
||||
static NonnullRefPtr<HeaderList> deserialize_headers(StringView serialized_headers)
|
||||
{
|
||||
auto headers = HTTP::HeaderList::create();
|
||||
auto headers = HeaderList::create();
|
||||
|
||||
serialized_headers.for_each_split_view('\n', SplitBehavior::Nothing, [&](StringView serialized_header) {
|
||||
auto index = serialized_header.find(':');
|
||||
|
|
@ -110,7 +110,7 @@ CacheIndex::CacheIndex(Database::Database& database, Statements statements)
|
|||
{
|
||||
}
|
||||
|
||||
void CacheIndex::create_entry(u64 cache_key, String url, NonnullRefPtr<HTTP::HeaderList> response_headers, u64 data_size, UnixDateTime request_time, UnixDateTime response_time)
|
||||
void CacheIndex::create_entry(u64 cache_key, String url, NonnullRefPtr<HeaderList> response_headers, u64 data_size, UnixDateTime request_time, UnixDateTime response_time)
|
||||
{
|
||||
auto now = UnixDateTime::now();
|
||||
|
||||
|
|
@ -133,22 +133,22 @@ void CacheIndex::create_entry(u64 cache_key, String url, NonnullRefPtr<HTTP::Hea
|
|||
.last_access_time = now,
|
||||
};
|
||||
|
||||
m_database.execute_statement(m_statements.insert_entry, {}, entry.cache_key, entry.url, serialize_headers(entry.response_headers), entry.data_size, entry.request_time, entry.response_time, entry.last_access_time);
|
||||
m_database->execute_statement(m_statements.insert_entry, {}, entry.cache_key, entry.url, serialize_headers(entry.response_headers), entry.data_size, entry.request_time, entry.response_time, entry.last_access_time);
|
||||
m_entries.set(cache_key, move(entry));
|
||||
}
|
||||
|
||||
void CacheIndex::remove_entry(u64 cache_key)
|
||||
{
|
||||
m_database.execute_statement(m_statements.remove_entry, {}, cache_key);
|
||||
m_database->execute_statement(m_statements.remove_entry, {}, cache_key);
|
||||
m_entries.remove(cache_key);
|
||||
}
|
||||
|
||||
void CacheIndex::remove_entries_accessed_since(UnixDateTime since, Function<void(u64 cache_key)> on_entry_removed)
|
||||
{
|
||||
m_database.execute_statement(
|
||||
m_database->execute_statement(
|
||||
m_statements.remove_entries_accessed_since,
|
||||
[&](auto statement_id) {
|
||||
auto cache_key = m_database.result_column<u64>(statement_id, 0);
|
||||
auto cache_key = m_database->result_column<u64>(statement_id, 0);
|
||||
m_entries.remove(cache_key);
|
||||
|
||||
on_entry_removed(cache_key);
|
||||
|
|
@ -156,13 +156,13 @@ void CacheIndex::remove_entries_accessed_since(UnixDateTime since, Function<void
|
|||
since);
|
||||
}
|
||||
|
||||
void CacheIndex::update_response_headers(u64 cache_key, NonnullRefPtr<HTTP::HeaderList> response_headers)
|
||||
void CacheIndex::update_response_headers(u64 cache_key, NonnullRefPtr<HeaderList> response_headers)
|
||||
{
|
||||
auto entry = m_entries.get(cache_key);
|
||||
if (!entry.has_value())
|
||||
return;
|
||||
|
||||
m_database.execute_statement(m_statements.update_response_headers, {}, serialize_headers(response_headers), cache_key);
|
||||
m_database->execute_statement(m_statements.update_response_headers, {}, serialize_headers(response_headers), cache_key);
|
||||
entry->response_headers = move(response_headers);
|
||||
}
|
||||
|
||||
|
|
@ -174,7 +174,7 @@ void CacheIndex::update_last_access_time(u64 cache_key)
|
|||
|
||||
auto now = UnixDateTime::now();
|
||||
|
||||
m_database.execute_statement(m_statements.update_last_access_time, {}, now, cache_key);
|
||||
m_database->execute_statement(m_statements.update_last_access_time, {}, now, cache_key);
|
||||
entry->last_access_time = now;
|
||||
}
|
||||
|
||||
|
|
@ -183,17 +183,17 @@ Optional<CacheIndex::Entry&> CacheIndex::find_entry(u64 cache_key)
|
|||
if (auto entry = m_entries.get(cache_key); entry.has_value())
|
||||
return entry;
|
||||
|
||||
m_database.execute_statement(
|
||||
m_database->execute_statement(
|
||||
m_statements.select_entry, [&](auto statement_id) {
|
||||
int column = 0;
|
||||
|
||||
auto cache_key = m_database.result_column<u64>(statement_id, column++);
|
||||
auto url = m_database.result_column<String>(statement_id, column++);
|
||||
auto response_headers = m_database.result_column<ByteString>(statement_id, column++);
|
||||
auto data_size = m_database.result_column<u64>(statement_id, column++);
|
||||
auto request_time = m_database.result_column<UnixDateTime>(statement_id, column++);
|
||||
auto response_time = m_database.result_column<UnixDateTime>(statement_id, column++);
|
||||
auto last_access_time = m_database.result_column<UnixDateTime>(statement_id, column++);
|
||||
auto cache_key = m_database->result_column<u64>(statement_id, column++);
|
||||
auto url = m_database->result_column<String>(statement_id, column++);
|
||||
auto response_headers = m_database->result_column<ByteString>(statement_id, column++);
|
||||
auto data_size = m_database->result_column<u64>(statement_id, column++);
|
||||
auto request_time = m_database->result_column<UnixDateTime>(statement_id, column++);
|
||||
auto response_time = m_database->result_column<UnixDateTime>(statement_id, column++);
|
||||
auto last_access_time = m_database->result_column<UnixDateTime>(statement_id, column++);
|
||||
|
||||
Entry entry { cache_key, move(url), deserialize_headers(response_headers), data_size, request_time, response_time, last_access_time };
|
||||
m_entries.set(cache_key, move(entry));
|
||||
|
|
@ -203,18 +203,18 @@ Optional<CacheIndex::Entry&> CacheIndex::find_entry(u64 cache_key)
|
|||
return m_entries.get(cache_key);
|
||||
}
|
||||
|
||||
Requests::CacheSizes CacheIndex::estimate_cache_size_accessed_since(UnixDateTime since) const
|
||||
Requests::CacheSizes CacheIndex::estimate_cache_size_accessed_since(UnixDateTime since)
|
||||
{
|
||||
Requests::CacheSizes sizes;
|
||||
|
||||
m_database.execute_statement(
|
||||
m_database->execute_statement(
|
||||
m_statements.estimate_cache_size_accessed_since,
|
||||
[&](auto statement_id) { sizes.since_requested_time = m_database.result_column<u64>(statement_id, 0); },
|
||||
[&](auto statement_id) { sizes.since_requested_time = m_database->result_column<u64>(statement_id, 0); },
|
||||
since);
|
||||
|
||||
m_database.execute_statement(
|
||||
m_database->execute_statement(
|
||||
m_statements.estimate_cache_size_accessed_since,
|
||||
[&](auto statement_id) { sizes.total = m_database.result_column<u64>(statement_id, 0); },
|
||||
[&](auto statement_id) { sizes.total = m_database->result_column<u64>(statement_id, 0); },
|
||||
UnixDateTime::earliest());
|
||||
|
||||
return sizes;
|
||||
|
|
@ -8,13 +8,14 @@
|
|||
|
||||
#include <AK/Error.h>
|
||||
#include <AK/HashMap.h>
|
||||
#include <AK/NonnullRawPtr.h>
|
||||
#include <AK/Time.h>
|
||||
#include <AK/Types.h>
|
||||
#include <LibDatabase/Database.h>
|
||||
#include <LibHTTP/HeaderList.h>
|
||||
#include <LibRequests/CacheSizes.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
// The cache index is a SQL database containing metadata about each cache entry. An entry in the index is created once
|
||||
// the entire cache entry has been successfully written to disk.
|
||||
|
|
@ -23,7 +24,7 @@ class CacheIndex {
|
|||
u64 cache_key { 0 };
|
||||
|
||||
String url;
|
||||
NonnullRefPtr<HTTP::HeaderList> response_headers;
|
||||
NonnullRefPtr<HeaderList> response_headers;
|
||||
u64 data_size { 0 };
|
||||
|
||||
UnixDateTime request_time;
|
||||
|
|
@ -34,16 +35,16 @@ class CacheIndex {
|
|||
public:
|
||||
static ErrorOr<CacheIndex> create(Database::Database&);
|
||||
|
||||
void create_entry(u64 cache_key, String url, NonnullRefPtr<HTTP::HeaderList>, u64 data_size, UnixDateTime request_time, UnixDateTime response_time);
|
||||
void create_entry(u64 cache_key, String url, NonnullRefPtr<HeaderList>, u64 data_size, UnixDateTime request_time, UnixDateTime response_time);
|
||||
void remove_entry(u64 cache_key);
|
||||
void remove_entries_accessed_since(UnixDateTime, Function<void(u64 cache_key)> on_entry_removed);
|
||||
|
||||
Optional<Entry&> find_entry(u64 cache_key);
|
||||
|
||||
void update_response_headers(u64 cache_key, NonnullRefPtr<HTTP::HeaderList>);
|
||||
void update_response_headers(u64 cache_key, NonnullRefPtr<HeaderList>);
|
||||
void update_last_access_time(u64 cache_key);
|
||||
|
||||
Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since) const;
|
||||
Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since);
|
||||
|
||||
private:
|
||||
struct Statements {
|
||||
|
|
@ -58,7 +59,7 @@ private:
|
|||
|
||||
CacheIndex(Database::Database&, Statements);
|
||||
|
||||
Database::Database& m_database;
|
||||
NonnullRawPtr<Database::Database> m_database;
|
||||
Statements m_statements;
|
||||
|
||||
HashMap<u32, Entry> m_entries;
|
||||
35
Libraries/LibHTTP/Cache/CacheRequest.h
Normal file
35
Libraries/LibHTTP/Cache/CacheRequest.h
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
/*
|
||||
* Copyright (c) 2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Badge.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/Weakable.h>
|
||||
#include <LibHTTP/Forward.h>
|
||||
|
||||
namespace HTTP {
|
||||
|
||||
class CacheRequest : public Weakable<CacheRequest> {
|
||||
public:
|
||||
virtual ~CacheRequest() = default;
|
||||
|
||||
virtual void notify_request_unblocked(Badge<DiskCache>) = 0;
|
||||
|
||||
protected:
|
||||
enum class CacheStatus : u8 {
|
||||
Unknown,
|
||||
NotCached,
|
||||
WrittenToCache,
|
||||
ReadFromCache,
|
||||
};
|
||||
|
||||
Optional<CacheEntryReader&> m_cache_entry_reader;
|
||||
Optional<CacheEntryWriter&> m_cache_entry_writer;
|
||||
CacheStatus m_cache_status { CacheStatus::Unknown };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -4,14 +4,15 @@
|
|||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <LibCore/EventLoop.h>
|
||||
#include <LibCore/StandardPaths.h>
|
||||
#include <LibFileSystem/FileSystem.h>
|
||||
#include <LibHTTP/Cache/CacheRequest.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibURL/URL.h>
|
||||
#include <RequestServer/Cache/DiskCache.h>
|
||||
#include <RequestServer/Cache/Utilities.h>
|
||||
#include <RequestServer/Request.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
static constexpr auto INDEX_DATABASE = "INDEX"sv;
|
||||
|
||||
|
|
@ -37,29 +38,37 @@ DiskCache::DiskCache(Mode mode, NonnullRefPtr<Database::Database> database, Lexi
|
|||
remove_entries_accessed_since(UnixDateTime::earliest());
|
||||
}
|
||||
|
||||
Variant<Optional<CacheEntryWriter&>, DiskCache::CacheHasOpenEntry> DiskCache::create_entry(Request& request)
|
||||
DiskCache::DiskCache(DiskCache&&) = default;
|
||||
DiskCache& DiskCache::operator=(DiskCache&&) = default;
|
||||
|
||||
DiskCache::~DiskCache() = default;
|
||||
|
||||
Variant<Optional<CacheEntryWriter&>, DiskCache::CacheHasOpenEntry> DiskCache::create_entry(CacheRequest& request, URL::URL const& url, StringView method, HeaderList const& request_headers, UnixDateTime request_start_time)
|
||||
{
|
||||
if (!is_cacheable(request.method()))
|
||||
if (!is_cacheable(method))
|
||||
return Optional<CacheEntryWriter&> {};
|
||||
|
||||
if (m_mode == Mode::Testing) {
|
||||
if (!request.request_headers().contains(TEST_CACHE_ENABLED_HEADER))
|
||||
if (!request_headers.contains(TEST_CACHE_ENABLED_HEADER))
|
||||
return Optional<CacheEntryWriter&> {};
|
||||
}
|
||||
|
||||
auto serialized_url = serialize_url_for_cache_storage(request.url());
|
||||
auto cache_key = create_cache_key(serialized_url, request.method());
|
||||
auto serialized_url = serialize_url_for_cache_storage(url);
|
||||
auto cache_key = create_cache_key(serialized_url, method);
|
||||
|
||||
if (check_if_cache_has_open_entry(request, cache_key, CheckReaderEntries::Yes))
|
||||
if (check_if_cache_has_open_entry(request, cache_key, url, CheckReaderEntries::Yes))
|
||||
return CacheHasOpenEntry {};
|
||||
|
||||
auto cache_entry = CacheEntryWriter::create(*this, m_index, cache_key, move(serialized_url), request.request_start_time(), request.current_time_offset_for_testing());
|
||||
auto current_time_offset_for_testing = compute_current_time_offset_for_testing(*this, request_headers);
|
||||
request_start_time += current_time_offset_for_testing;
|
||||
|
||||
auto cache_entry = CacheEntryWriter::create(*this, m_index, cache_key, move(serialized_url), request_start_time, current_time_offset_for_testing);
|
||||
if (cache_entry.is_error()) {
|
||||
dbgln("\033[31;1mUnable to create cache entry for\033[0m {}: {}", request.url(), cache_entry.error());
|
||||
dbgln("\033[31;1mUnable to create cache entry for\033[0m {}: {}", url, cache_entry.error());
|
||||
return Optional<CacheEntryWriter&> {};
|
||||
}
|
||||
|
||||
dbgln("\033[32;1mCreated disk cache entry for\033[0m {}", request.url());
|
||||
dbgln("\033[32;1mCreated disk cache entry for\033[0m {}", url);
|
||||
|
||||
auto* cache_entry_pointer = cache_entry.value().ptr();
|
||||
m_open_cache_entries.ensure(cache_key).append(cache_entry.release_value());
|
||||
|
|
@ -67,52 +76,54 @@ Variant<Optional<CacheEntryWriter&>, DiskCache::CacheHasOpenEntry> DiskCache::cr
|
|||
return Optional<CacheEntryWriter&> { *cache_entry_pointer };
|
||||
}
|
||||
|
||||
Variant<Optional<CacheEntryReader&>, DiskCache::CacheHasOpenEntry> DiskCache::open_entry(Request& request)
|
||||
Variant<Optional<CacheEntryReader&>, DiskCache::CacheHasOpenEntry> DiskCache::open_entry(CacheRequest& request, URL::URL const& url, StringView method, HeaderList const& request_headers)
|
||||
{
|
||||
if (!is_cacheable(request.method()))
|
||||
if (!is_cacheable(method))
|
||||
return Optional<CacheEntryReader&> {};
|
||||
|
||||
auto serialized_url = serialize_url_for_cache_storage(request.url());
|
||||
auto cache_key = create_cache_key(serialized_url, request.method());
|
||||
auto serialized_url = serialize_url_for_cache_storage(url);
|
||||
auto cache_key = create_cache_key(serialized_url, method);
|
||||
|
||||
if (check_if_cache_has_open_entry(request, cache_key, CheckReaderEntries::No))
|
||||
if (check_if_cache_has_open_entry(request, cache_key, url, CheckReaderEntries::No))
|
||||
return CacheHasOpenEntry {};
|
||||
|
||||
auto index_entry = m_index.find_entry(cache_key);
|
||||
if (!index_entry.has_value()) {
|
||||
dbgln("\033[35;1mNo disk cache entry for\033[0m {}", request.url());
|
||||
dbgln("\033[35;1mNo disk cache entry for\033[0m {}", url);
|
||||
return Optional<CacheEntryReader&> {};
|
||||
}
|
||||
|
||||
auto cache_entry = CacheEntryReader::create(*this, m_index, cache_key, index_entry->response_headers, index_entry->data_size);
|
||||
if (cache_entry.is_error()) {
|
||||
dbgln("\033[31;1mUnable to open cache entry for\033[0m {}: {}", request.url(), cache_entry.error());
|
||||
dbgln("\033[31;1mUnable to open cache entry for\033[0m {}: {}", url, cache_entry.error());
|
||||
m_index.remove_entry(cache_key);
|
||||
|
||||
return Optional<CacheEntryReader&> {};
|
||||
}
|
||||
|
||||
auto current_time_offset_for_testing = compute_current_time_offset_for_testing(*this, request_headers);
|
||||
|
||||
auto const& response_headers = cache_entry.value()->response_headers();
|
||||
auto freshness_lifetime = calculate_freshness_lifetime(cache_entry.value()->status_code(), response_headers, request.current_time_offset_for_testing());
|
||||
auto current_age = calculate_age(response_headers, index_entry->request_time, index_entry->response_time, request.current_time_offset_for_testing());
|
||||
auto freshness_lifetime = calculate_freshness_lifetime(cache_entry.value()->status_code(), response_headers, current_time_offset_for_testing);
|
||||
auto current_age = calculate_age(response_headers, index_entry->request_time, index_entry->response_time, current_time_offset_for_testing);
|
||||
|
||||
switch (cache_lifetime_status(response_headers, freshness_lifetime, current_age)) {
|
||||
case CacheLifetimeStatus::Fresh:
|
||||
dbgln("\033[32;1mOpened disk cache entry for\033[0m {} (lifetime={}s age={}s) ({} bytes)", request.url(), freshness_lifetime.to_seconds(), current_age.to_seconds(), index_entry->data_size);
|
||||
dbgln("\033[32;1mOpened disk cache entry for\033[0m {} (lifetime={}s age={}s) ({} bytes)", url, freshness_lifetime.to_seconds(), current_age.to_seconds(), index_entry->data_size);
|
||||
break;
|
||||
|
||||
case CacheLifetimeStatus::Expired:
|
||||
dbgln("\033[33;1mCache entry expired for\033[0m {} (lifetime={}s age={}s)", request.url(), freshness_lifetime.to_seconds(), current_age.to_seconds());
|
||||
dbgln("\033[33;1mCache entry expired for\033[0m {} (lifetime={}s age={}s)", url, freshness_lifetime.to_seconds(), current_age.to_seconds());
|
||||
cache_entry.value()->remove();
|
||||
|
||||
return Optional<CacheEntryReader&> {};
|
||||
|
||||
case CacheLifetimeStatus::MustRevalidate:
|
||||
// We will hold an exclusive lock on the cache entry for revalidation requests.
|
||||
if (check_if_cache_has_open_entry(request, cache_key, CheckReaderEntries::Yes))
|
||||
if (check_if_cache_has_open_entry(request, cache_key, url, CheckReaderEntries::Yes))
|
||||
return Optional<CacheEntryReader&> {};
|
||||
|
||||
dbgln("\033[36;1mMust revalidate disk cache entry for\033[0m {} (lifetime={}s age={}s)", request.url(), freshness_lifetime.to_seconds(), current_age.to_seconds());
|
||||
dbgln("\033[36;1mMust revalidate disk cache entry for\033[0m {} (lifetime={}s age={}s)", url, freshness_lifetime.to_seconds(), current_age.to_seconds());
|
||||
cache_entry.value()->set_must_revalidate();
|
||||
break;
|
||||
}
|
||||
|
|
@ -123,7 +134,7 @@ Variant<Optional<CacheEntryReader&>, DiskCache::CacheHasOpenEntry> DiskCache::op
|
|||
return Optional<CacheEntryReader&> { *cache_entry_pointer };
|
||||
}
|
||||
|
||||
bool DiskCache::check_if_cache_has_open_entry(Request& request, u64 cache_key, CheckReaderEntries check_reader_entries)
|
||||
bool DiskCache::check_if_cache_has_open_entry(CacheRequest& request, u64 cache_key, URL::URL const& url, CheckReaderEntries check_reader_entries)
|
||||
{
|
||||
auto open_entries = m_open_cache_entries.get(cache_key);
|
||||
if (!open_entries.has_value())
|
||||
|
|
@ -131,7 +142,7 @@ bool DiskCache::check_if_cache_has_open_entry(Request& request, u64 cache_key, C
|
|||
|
||||
for (auto const& open_entry : *open_entries) {
|
||||
if (is<CacheEntryWriter>(*open_entry)) {
|
||||
dbgln("\033[36;1mDeferring disk cache entry for\033[0m {} (waiting for existing writer)", request.url());
|
||||
dbgln("\033[36;1mDeferring disk cache entry for\033[0m {} (waiting for existing writer)", url);
|
||||
m_requests_waiting_completion.ensure(cache_key).append(request);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -139,7 +150,7 @@ bool DiskCache::check_if_cache_has_open_entry(Request& request, u64 cache_key, C
|
|||
// We allow concurrent readers unless another reader is open for revalidation. That reader will issue the network
|
||||
// request, which may then result in the cache entry being updated or deleted.
|
||||
if (check_reader_entries == CheckReaderEntries::Yes || as<CacheEntryReader>(*open_entry).must_revalidate()) {
|
||||
dbgln("\033[36;1mDeferring disk cache entry for\033[0m {} (waiting for existing reader)", request.url());
|
||||
dbgln("\033[36;1mDeferring disk cache entry for\033[0m {} (waiting for existing reader)", url);
|
||||
m_requests_waiting_completion.ensure(cache_key).append(request);
|
||||
return true;
|
||||
}
|
||||
|
|
@ -148,7 +159,7 @@ bool DiskCache::check_if_cache_has_open_entry(Request& request, u64 cache_key, C
|
|||
return false;
|
||||
}
|
||||
|
||||
Requests::CacheSizes DiskCache::estimate_cache_size_accessed_since(UnixDateTime since) const
|
||||
Requests::CacheSizes DiskCache::estimate_cache_size_accessed_since(UnixDateTime since)
|
||||
{
|
||||
return m_index.estimate_cache_size_accessed_since(since);
|
||||
}
|
||||
|
|
@ -14,11 +14,11 @@
|
|||
#include <AK/Types.h>
|
||||
#include <AK/WeakPtr.h>
|
||||
#include <LibDatabase/Database.h>
|
||||
#include <LibHTTP/Cache/CacheEntry.h>
|
||||
#include <LibHTTP/Cache/CacheIndex.h>
|
||||
#include <LibURL/Forward.h>
|
||||
#include <RequestServer/Cache/CacheEntry.h>
|
||||
#include <RequestServer/Cache/CacheIndex.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
class DiskCache {
|
||||
public:
|
||||
|
|
@ -31,13 +31,18 @@ public:
|
|||
};
|
||||
static ErrorOr<DiskCache> create(Mode);
|
||||
|
||||
DiskCache(DiskCache&&);
|
||||
DiskCache& operator=(DiskCache&&);
|
||||
|
||||
~DiskCache();
|
||||
|
||||
Mode mode() const { return m_mode; }
|
||||
|
||||
struct CacheHasOpenEntry { };
|
||||
Variant<Optional<CacheEntryWriter&>, CacheHasOpenEntry> create_entry(Request&);
|
||||
Variant<Optional<CacheEntryReader&>, CacheHasOpenEntry> open_entry(Request&);
|
||||
Variant<Optional<CacheEntryWriter&>, CacheHasOpenEntry> create_entry(CacheRequest&, URL::URL const&, StringView method, HeaderList const& request_headers, UnixDateTime request_start_time);
|
||||
Variant<Optional<CacheEntryReader&>, CacheHasOpenEntry> open_entry(CacheRequest&, URL::URL const&, StringView method, HeaderList const& request_headers);
|
||||
|
||||
Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since) const;
|
||||
Requests::CacheSizes estimate_cache_size_accessed_since(UnixDateTime since);
|
||||
void remove_entries_accessed_since(UnixDateTime since);
|
||||
|
||||
LexicalPath const& cache_directory() { return m_cache_directory; }
|
||||
|
|
@ -51,14 +56,14 @@ private:
|
|||
No,
|
||||
Yes,
|
||||
};
|
||||
bool check_if_cache_has_open_entry(Request&, u64 cache_key, CheckReaderEntries);
|
||||
bool check_if_cache_has_open_entry(CacheRequest&, u64 cache_key, URL::URL const&, CheckReaderEntries);
|
||||
|
||||
Mode m_mode;
|
||||
|
||||
NonnullRefPtr<Database::Database> m_database;
|
||||
|
||||
HashMap<u64, Vector<NonnullOwnPtr<CacheEntry>, 1>> m_open_cache_entries;
|
||||
HashMap<u64, Vector<WeakPtr<Request>, 1>> m_requests_waiting_completion;
|
||||
HashMap<u64, Vector<WeakPtr<CacheRequest>, 1>> m_requests_waiting_completion;
|
||||
|
||||
LexicalPath m_cache_directory;
|
||||
CacheIndex m_index;
|
||||
|
|
@ -5,11 +5,11 @@
|
|||
*/
|
||||
|
||||
#include <LibCrypto/Hash/SHA1.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibURL/URL.h>
|
||||
#include <RequestServer/Cache/DiskCache.h>
|
||||
#include <RequestServer/Cache/Utilities.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
static Optional<StringView> extract_cache_control_directive(StringView cache_control, StringView directive)
|
||||
{
|
||||
|
|
@ -111,7 +111,7 @@ static bool is_heuristically_cacheable_status(u32 status_code)
|
|||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#response.cacheability
|
||||
bool is_cacheable(u32 status_code, HTTP::HeaderList const& headers)
|
||||
bool is_cacheable(u32 status_code, HeaderList const& headers)
|
||||
{
|
||||
// A cache MUST NOT store a response to a request unless:
|
||||
|
||||
|
|
@ -213,7 +213,7 @@ bool is_header_exempted_from_storage(StringView name)
|
|||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#heuristic.freshness
|
||||
static AK::Duration calculate_heuristic_freshness_lifetime(HTTP::HeaderList const& headers, AK::Duration current_time_offset_for_testing)
|
||||
static AK::Duration calculate_heuristic_freshness_lifetime(HeaderList const& headers, AK::Duration current_time_offset_for_testing)
|
||||
{
|
||||
// Since origin servers do not always provide explicit expiration times, a cache MAY assign a heuristic expiration
|
||||
// time when an explicit time is not specified, employing algorithms that use other field values (such as the
|
||||
|
|
@ -245,7 +245,7 @@ static AK::Duration calculate_heuristic_freshness_lifetime(HTTP::HeaderList cons
|
|||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#calculating.freshness.lifetime
|
||||
AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderList const& headers, AK::Duration current_time_offset_for_testing)
|
||||
AK::Duration calculate_freshness_lifetime(u32 status_code, HeaderList const& headers, AK::Duration current_time_offset_for_testing)
|
||||
{
|
||||
// A cache can calculate the freshness lifetime (denoted as freshness_lifetime) of a response by evaluating the
|
||||
// following rules and using the first match:
|
||||
|
|
@ -296,7 +296,7 @@ AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderList cons
|
|||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#age.calculations
|
||||
AK::Duration calculate_age(HTTP::HeaderList const& headers, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing)
|
||||
AK::Duration calculate_age(HeaderList const& headers, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing)
|
||||
{
|
||||
// The term "age_value" denotes the value of the Age header field (Section 5.1), in a form appropriate for arithmetic
|
||||
// operation; or 0, if not available.
|
||||
|
|
@ -328,7 +328,7 @@ AK::Duration calculate_age(HTTP::HeaderList const& headers, UnixDateTime request
|
|||
return current_age;
|
||||
}
|
||||
|
||||
CacheLifetimeStatus cache_lifetime_status(HTTP::HeaderList const& headers, AK::Duration freshness_lifetime, AK::Duration current_age)
|
||||
CacheLifetimeStatus cache_lifetime_status(HeaderList const& headers, AK::Duration freshness_lifetime, AK::Duration current_age)
|
||||
{
|
||||
auto revalidation_status = [&]() {
|
||||
// In order to revalidate a cache entry, we must have one of these headers to attach to the revalidation request.
|
||||
|
|
@ -365,7 +365,7 @@ CacheLifetimeStatus cache_lifetime_status(HTTP::HeaderList const& headers, AK::D
|
|||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#validation.sent
|
||||
RevalidationAttributes RevalidationAttributes::create(HTTP::HeaderList const& headers)
|
||||
RevalidationAttributes RevalidationAttributes::create(HeaderList const& headers)
|
||||
{
|
||||
RevalidationAttributes attributes;
|
||||
attributes.etag = headers.get("ETag"sv).map([](auto const& etag) { return etag; });
|
||||
|
|
@ -375,7 +375,7 @@ RevalidationAttributes RevalidationAttributes::create(HTTP::HeaderList const& he
|
|||
}
|
||||
|
||||
// https://httpwg.org/specs/rfc9111.html#update
|
||||
void update_header_fields(HTTP::HeaderList& stored_headers, HTTP::HeaderList const& updated_headers)
|
||||
void update_header_fields(HeaderList& stored_headers, HeaderList const& updated_headers)
|
||||
{
|
||||
// Caches are required to update a stored response's header fields from another (typically newer) response in
|
||||
// several situations; for example, see Sections 3.4, 4.3.4, and 4.3.5.
|
||||
|
|
@ -408,7 +408,7 @@ void update_header_fields(HTTP::HeaderList& stored_headers, HTTP::HeaderList con
|
|||
}
|
||||
}
|
||||
|
||||
AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&> disk_cache, HTTP::HeaderList const& request_headers)
|
||||
AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&> disk_cache, HeaderList const& request_headers)
|
||||
{
|
||||
if (disk_cache.has_value() && disk_cache->mode() == DiskCache::Mode::Testing) {
|
||||
if (auto header = request_headers.get(TEST_CACHE_REQUEST_TIME_OFFSET); header.has_value()) {
|
||||
|
|
@ -10,11 +10,11 @@
|
|||
#include <AK/StringView.h>
|
||||
#include <AK/Time.h>
|
||||
#include <AK/Types.h>
|
||||
#include <LibHTTP/Forward.h>
|
||||
#include <LibHTTP/HeaderList.h>
|
||||
#include <LibURL/Forward.h>
|
||||
#include <RequestServer/Forward.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
constexpr inline auto TEST_CACHE_ENABLED_HEADER = "X-Ladybird-Enable-Disk-Cache"sv;
|
||||
constexpr inline auto TEST_CACHE_STATUS_HEADER = "X-Ladybird-Disk-Cache-Status"sv;
|
||||
|
|
@ -25,28 +25,28 @@ u64 create_cache_key(StringView url, StringView method);
|
|||
LexicalPath path_for_cache_key(LexicalPath const& cache_directory, u64 cache_key);
|
||||
|
||||
bool is_cacheable(StringView method);
|
||||
bool is_cacheable(u32 status_code, HTTP::HeaderList const&);
|
||||
bool is_cacheable(u32 status_code, HeaderList const&);
|
||||
bool is_header_exempted_from_storage(StringView name);
|
||||
|
||||
AK::Duration calculate_freshness_lifetime(u32 status_code, HTTP::HeaderList const&, AK::Duration current_time_offset_for_testing);
|
||||
AK::Duration calculate_age(HTTP::HeaderList const&, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing);
|
||||
AK::Duration calculate_freshness_lifetime(u32 status_code, HeaderList const&, AK::Duration current_time_offset_for_testing);
|
||||
AK::Duration calculate_age(HeaderList const&, UnixDateTime request_time, UnixDateTime response_time, AK::Duration current_time_offset_for_testing);
|
||||
|
||||
enum class CacheLifetimeStatus {
|
||||
Fresh,
|
||||
Expired,
|
||||
MustRevalidate,
|
||||
};
|
||||
CacheLifetimeStatus cache_lifetime_status(HTTP::HeaderList const&, AK::Duration freshness_lifetime, AK::Duration current_age);
|
||||
CacheLifetimeStatus cache_lifetime_status(HeaderList const&, AK::Duration freshness_lifetime, AK::Duration current_age);
|
||||
|
||||
struct RevalidationAttributes {
|
||||
static RevalidationAttributes create(HTTP::HeaderList const&);
|
||||
static RevalidationAttributes create(HeaderList const&);
|
||||
|
||||
Optional<ByteString> etag;
|
||||
Optional<UnixDateTime> last_modified;
|
||||
};
|
||||
|
||||
void update_header_fields(HTTP::HeaderList&, HTTP::HeaderList const&);
|
||||
void update_header_fields(HeaderList&, HeaderList const&);
|
||||
|
||||
AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&>, HTTP::HeaderList const& request_headers);
|
||||
AK::Duration compute_current_time_offset_for_testing(Optional<DiskCache&>, HeaderList const& request_headers);
|
||||
|
||||
}
|
||||
|
|
@ -8,7 +8,7 @@
|
|||
|
||||
#include <AK/Types.h>
|
||||
|
||||
namespace RequestServer {
|
||||
namespace HTTP {
|
||||
|
||||
// Increment this version when a breaking change is made to the cache index or cache entry formats.
|
||||
static constexpr inline u32 CACHE_VERSION = 4u;
|
||||
|
|
@ -8,6 +8,12 @@
|
|||
|
||||
namespace HTTP {
|
||||
|
||||
class CacheEntry;
|
||||
class CacheEntryReader;
|
||||
class CacheEntryWriter;
|
||||
class CacheIndex;
|
||||
class CacheRequest;
|
||||
class DiskCache;
|
||||
class HeaderList;
|
||||
class HttpRequest;
|
||||
class HttpResponse;
|
||||
|
|
|
|||
|
|
@ -3,10 +3,6 @@ set(CMAKE_AUTORCC OFF)
|
|||
set(CMAKE_AUTOUIC OFF)
|
||||
|
||||
set(SOURCES
|
||||
Cache/CacheEntry.cpp
|
||||
Cache/CacheIndex.cpp
|
||||
Cache/DiskCache.cpp
|
||||
Cache/Utilities.cpp
|
||||
ConnectionFromClient.cpp
|
||||
CURL.cpp
|
||||
Request.cpp
|
||||
|
|
@ -41,7 +37,7 @@ target_include_directories(requestserverservice PRIVATE ${CMAKE_CURRENT_BINARY_D
|
|||
target_include_directories(requestserverservice PRIVATE ${LADYBIRD_SOURCE_DIR}/Services/)
|
||||
|
||||
target_link_libraries(RequestServer PRIVATE requestserverservice)
|
||||
target_link_libraries(requestserverservice PUBLIC LibCore LibDatabase LibDNS LibCrypto LibFileSystem LibHTTP LibIPC LibMain LibRequests LibTLS LibWebSocket LibURL LibTextCodec LibThreading CURL::libcurl)
|
||||
target_link_libraries(requestserverservice PUBLIC LibCore LibDNS LibHTTP LibIPC LibMain LibRequests LibTLS LibWebSocket LibURL LibTextCodec CURL::libcurl)
|
||||
target_link_libraries(requestserverservice PRIVATE OpenSSL::Crypto OpenSSL::SSL)
|
||||
|
||||
if (WIN32)
|
||||
|
|
|
|||
|
|
@ -10,11 +10,11 @@
|
|||
#include <LibCore/Proxy.h>
|
||||
#include <LibCore/Socket.h>
|
||||
#include <LibCore/StandardPaths.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibRequests/WebSocket.h>
|
||||
#include <LibWebSocket/ConnectionInfo.h>
|
||||
#include <LibWebSocket/Message.h>
|
||||
#include <RequestServer/CURL.h>
|
||||
#include <RequestServer/Cache/DiskCache.h>
|
||||
#include <RequestServer/ConnectionFromClient.h>
|
||||
#include <RequestServer/Request.h>
|
||||
#include <RequestServer/Resolver.h>
|
||||
|
|
@ -25,7 +25,7 @@ namespace RequestServer {
|
|||
static HashMap<int, RefPtr<ConnectionFromClient>> s_connections;
|
||||
static IDAllocator s_client_ids;
|
||||
|
||||
Optional<DiskCache> g_disk_cache;
|
||||
Optional<HTTP::DiskCache> g_disk_cache;
|
||||
|
||||
ConnectionFromClient::ConnectionFromClient(NonnullOwnPtr<IPC::Transport> transport)
|
||||
: IPC::ConnectionFromClient<RequestClientEndpoint, RequestServerEndpoint>(*this, move(transport), s_client_ids.allocate())
|
||||
|
|
|
|||
|
|
@ -8,12 +8,7 @@
|
|||
|
||||
namespace RequestServer {
|
||||
|
||||
class CacheEntry;
|
||||
class CacheEntryReader;
|
||||
class CacheEntryWriter;
|
||||
class CacheIndex;
|
||||
class ConnectionFromClient;
|
||||
class DiskCache;
|
||||
class Request;
|
||||
class RequestPipe;
|
||||
|
||||
|
|
|
|||
|
|
@ -5,13 +5,12 @@
|
|||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/Enumerate.h>
|
||||
#include <AK/GenericShorthands.h>
|
||||
#include <LibCore/Notifier.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibHTTP/Cache/Utilities.h>
|
||||
#include <LibTextCodec/Decoder.h>
|
||||
#include <RequestServer/CURL.h>
|
||||
#include <RequestServer/Cache/DiskCache.h>
|
||||
#include <RequestServer/Cache/Utilities.h>
|
||||
#include <RequestServer/ConnectionFromClient.h>
|
||||
#include <RequestServer/Request.h>
|
||||
#include <RequestServer/Resolver.h>
|
||||
|
|
@ -22,7 +21,7 @@ static long s_connect_timeout_seconds = 90L;
|
|||
|
||||
NonnullOwnPtr<Request> Request::fetch(
|
||||
i32 request_id,
|
||||
Optional<DiskCache&> disk_cache,
|
||||
Optional<HTTP::DiskCache&> disk_cache,
|
||||
ConnectionFromClient& client,
|
||||
void* curl_multi,
|
||||
Resolver& resolver,
|
||||
|
|
@ -63,7 +62,7 @@ NonnullOwnPtr<Request> Request::connect(
|
|||
|
||||
Request::Request(
|
||||
i32 request_id,
|
||||
Optional<DiskCache&> disk_cache,
|
||||
Optional<HTTP::DiskCache&> disk_cache,
|
||||
ConnectionFromClient& client,
|
||||
void* curl_multi,
|
||||
Resolver& resolver,
|
||||
|
|
@ -86,9 +85,7 @@ Request::Request(
|
|||
, m_alt_svc_cache_path(move(alt_svc_cache_path))
|
||||
, m_proxy_data(proxy_data)
|
||||
, m_response_headers(HTTP::HeaderList::create())
|
||||
, m_current_time_offset_for_testing(compute_current_time_offset_for_testing(m_disk_cache, m_request_headers))
|
||||
{
|
||||
m_request_start_time += m_current_time_offset_for_testing;
|
||||
}
|
||||
|
||||
Request::Request(
|
||||
|
|
@ -105,9 +102,7 @@ Request::Request(
|
|||
, m_url(move(url))
|
||||
, m_request_headers(HTTP::HeaderList::create())
|
||||
, m_response_headers(HTTP::HeaderList::create())
|
||||
, m_current_time_offset_for_testing(compute_current_time_offset_for_testing(m_disk_cache, m_request_headers))
|
||||
{
|
||||
m_request_start_time += m_current_time_offset_for_testing;
|
||||
}
|
||||
|
||||
Request::~Request()
|
||||
|
|
@ -129,7 +124,7 @@ Request::~Request()
|
|||
(void)m_cache_entry_writer->flush(m_response_headers);
|
||||
}
|
||||
|
||||
void Request::notify_request_unblocked(Badge<DiskCache>)
|
||||
void Request::notify_request_unblocked(Badge<HTTP::DiskCache>)
|
||||
{
|
||||
// FIXME: We may want a timer to limit how long we are waiting for a request before proceeding with a network
|
||||
// request that skips the disk cache.
|
||||
|
|
@ -196,38 +191,40 @@ void Request::process()
|
|||
void Request::handle_initial_state()
|
||||
{
|
||||
if (m_disk_cache.has_value()) {
|
||||
m_disk_cache->open_entry(*this).visit(
|
||||
[&](Optional<CacheEntryReader&> cache_entry_reader) {
|
||||
m_cache_entry_reader = cache_entry_reader;
|
||||
m_disk_cache->open_entry(*this, m_url, m_method, m_request_headers)
|
||||
.visit(
|
||||
[&](Optional<HTTP::CacheEntryReader&> cache_entry_reader) {
|
||||
m_cache_entry_reader = cache_entry_reader;
|
||||
|
||||
if (m_cache_entry_reader.has_value()) {
|
||||
if (m_cache_entry_reader->must_revalidate())
|
||||
transition_to_state(State::DNSLookup);
|
||||
else
|
||||
transition_to_state(State::ReadCache);
|
||||
}
|
||||
},
|
||||
[&](DiskCache::CacheHasOpenEntry) {
|
||||
// If an existing entry is open for writing, we must wait for it to complete.
|
||||
transition_to_state(State::WaitForCache);
|
||||
});
|
||||
if (m_cache_entry_reader.has_value()) {
|
||||
if (m_cache_entry_reader->must_revalidate())
|
||||
transition_to_state(State::DNSLookup);
|
||||
else
|
||||
transition_to_state(State::ReadCache);
|
||||
}
|
||||
},
|
||||
[&](HTTP::DiskCache::CacheHasOpenEntry) {
|
||||
// If an existing entry is open for writing, we must wait for it to complete.
|
||||
transition_to_state(State::WaitForCache);
|
||||
});
|
||||
|
||||
if (m_state != State::Init)
|
||||
return;
|
||||
|
||||
m_disk_cache->create_entry(*this).visit(
|
||||
[&](Optional<CacheEntryWriter&> cache_entry_writer) {
|
||||
m_cache_entry_writer = cache_entry_writer;
|
||||
m_disk_cache->create_entry(*this, m_url, m_method, m_request_headers, m_request_start_time)
|
||||
.visit(
|
||||
[&](Optional<HTTP::CacheEntryWriter&> cache_entry_writer) {
|
||||
m_cache_entry_writer = cache_entry_writer;
|
||||
|
||||
if (!m_cache_entry_writer.has_value())
|
||||
m_cache_status = CacheStatus::NotCached;
|
||||
},
|
||||
[&](DiskCache::CacheHasOpenEntry) {
|
||||
// If an existing entry is open for reading or writing, we must wait for it to complete. An entry being
|
||||
// open for reading is a rare case, but may occur if a cached response expired between the existing
|
||||
// entry's cache validation and the attempted reader validation when this request was created.
|
||||
transition_to_state(State::WaitForCache);
|
||||
});
|
||||
if (!m_cache_entry_writer.has_value())
|
||||
m_cache_status = CacheStatus::NotCached;
|
||||
},
|
||||
[&](HTTP::DiskCache::CacheHasOpenEntry) {
|
||||
// If an existing entry is open for reading or writing, we must wait for it to complete. An entry being
|
||||
// open for reading is a rare case, but may occur if a cached response expired between the existing
|
||||
// entry's cache validation and the attempted reader validation when this request was created.
|
||||
transition_to_state(State::WaitForCache);
|
||||
});
|
||||
|
||||
if (m_state != State::Init)
|
||||
return;
|
||||
|
|
@ -385,7 +382,7 @@ void Request::handle_fetch_state()
|
|||
}
|
||||
|
||||
if (is_revalidation_request) {
|
||||
auto revalidation_attributes = RevalidationAttributes::create(m_cache_entry_reader->response_headers());
|
||||
auto revalidation_attributes = HTTP::RevalidationAttributes::create(m_cache_entry_reader->response_headers());
|
||||
VERIFY(revalidation_attributes.etag.has_value() || revalidation_attributes.last_modified.has_value());
|
||||
|
||||
if (revalidation_attributes.etag.has_value()) {
|
||||
|
|
@ -512,14 +509,15 @@ size_t Request::on_data_received(void* buffer, size_t size, size_t nmemb, void*
|
|||
if (request.revalidation_failed().is_error())
|
||||
return CURL_WRITEFUNC_ERROR;
|
||||
|
||||
request.m_disk_cache->create_entry(request).visit(
|
||||
[&](Optional<CacheEntryWriter&> cache_entry_writer) {
|
||||
request.m_cache_entry_writer = cache_entry_writer;
|
||||
},
|
||||
[&](DiskCache::CacheHasOpenEntry) {
|
||||
// This should not be reachable, as cache revalidation holds an exclusive lock on the cache entry.
|
||||
VERIFY_NOT_REACHED();
|
||||
});
|
||||
request.m_disk_cache->create_entry(request, request.m_url, request.m_method, request.m_request_headers, request.m_request_start_time)
|
||||
.visit(
|
||||
[&](Optional<HTTP::CacheEntryWriter&> cache_entry_writer) {
|
||||
request.m_cache_entry_writer = cache_entry_writer;
|
||||
},
|
||||
[&](HTTP::DiskCache::CacheHasOpenEntry) {
|
||||
// This should not be reachable, as cache revalidation holds an exclusive lock on the cache entry.
|
||||
VERIFY_NOT_REACHED();
|
||||
});
|
||||
}
|
||||
|
||||
request.transfer_headers_to_client_if_needed();
|
||||
|
|
@ -574,18 +572,18 @@ void Request::transfer_headers_to_client_if_needed()
|
|||
}
|
||||
}
|
||||
|
||||
if (m_disk_cache.has_value() && m_disk_cache->mode() == DiskCache::Mode::Testing) {
|
||||
if (m_disk_cache.has_value() && m_disk_cache->mode() == HTTP::DiskCache::Mode::Testing) {
|
||||
switch (m_cache_status) {
|
||||
case CacheStatus::Unknown:
|
||||
break;
|
||||
case CacheStatus::NotCached:
|
||||
m_response_headers->set({ TEST_CACHE_STATUS_HEADER, "not-cached"sv });
|
||||
m_response_headers->set({ HTTP::TEST_CACHE_STATUS_HEADER, "not-cached"sv });
|
||||
break;
|
||||
case CacheStatus::WrittenToCache:
|
||||
m_response_headers->set({ TEST_CACHE_STATUS_HEADER, "written-to-cache"sv });
|
||||
m_response_headers->set({ HTTP::TEST_CACHE_STATUS_HEADER, "written-to-cache"sv });
|
||||
break;
|
||||
case CacheStatus::ReadFromCache:
|
||||
m_response_headers->set({ TEST_CACHE_STATUS_HEADER, "read-from-cache"sv });
|
||||
m_response_headers->set({ HTTP::TEST_CACHE_STATUS_HEADER, "read-from-cache"sv });
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,9 +11,9 @@
|
|||
#include <AK/MemoryStream.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/Time.h>
|
||||
#include <AK/Weakable.h>
|
||||
#include <LibCore/Proxy.h>
|
||||
#include <LibDNS/Resolver.h>
|
||||
#include <LibHTTP/Cache/CacheRequest.h>
|
||||
#include <LibHTTP/HeaderList.h>
|
||||
#include <LibRequests/NetworkError.h>
|
||||
#include <LibRequests/RequestTimingInfo.h>
|
||||
|
|
@ -26,11 +26,11 @@ struct curl_slist;
|
|||
|
||||
namespace RequestServer {
|
||||
|
||||
class Request : public Weakable<Request> {
|
||||
class Request : public HTTP::CacheRequest {
|
||||
public:
|
||||
static NonnullOwnPtr<Request> fetch(
|
||||
i32 request_id,
|
||||
Optional<DiskCache&> disk_cache,
|
||||
Optional<HTTP::DiskCache&> disk_cache,
|
||||
ConnectionFromClient& client,
|
||||
void* curl_multi,
|
||||
Resolver& resolver,
|
||||
|
|
@ -49,15 +49,14 @@ public:
|
|||
URL::URL url,
|
||||
CacheLevel cache_level);
|
||||
|
||||
~Request();
|
||||
virtual ~Request() override;
|
||||
|
||||
URL::URL const& url() const { return m_url; }
|
||||
ByteString const& method() const { return m_method; }
|
||||
HTTP::HeaderList const& request_headers() const { return m_request_headers; }
|
||||
UnixDateTime request_start_time() const { return m_request_start_time; }
|
||||
AK::Duration current_time_offset_for_testing() const { return m_current_time_offset_for_testing; }
|
||||
|
||||
void notify_request_unblocked(Badge<DiskCache>);
|
||||
virtual void notify_request_unblocked(Badge<HTTP::DiskCache>) override;
|
||||
void notify_fetch_complete(Badge<ConnectionFromClient>, int result_code);
|
||||
|
||||
private:
|
||||
|
|
@ -77,16 +76,9 @@ private:
|
|||
Error, // Any error occured during the request's lifetime.
|
||||
};
|
||||
|
||||
enum class CacheStatus : u8 {
|
||||
Unknown,
|
||||
NotCached,
|
||||
WrittenToCache,
|
||||
ReadFromCache,
|
||||
};
|
||||
|
||||
Request(
|
||||
i32 request_id,
|
||||
Optional<DiskCache&> disk_cache,
|
||||
Optional<HTTP::DiskCache&> disk_cache,
|
||||
ConnectionFromClient& client,
|
||||
void* curl_multi,
|
||||
Resolver& resolver,
|
||||
|
|
@ -132,7 +124,7 @@ private:
|
|||
Type m_type { Type::Fetch };
|
||||
State m_state { State::Init };
|
||||
|
||||
Optional<DiskCache&> m_disk_cache;
|
||||
Optional<HTTP::DiskCache&> m_disk_cache;
|
||||
ConnectionFromClient& m_client;
|
||||
|
||||
void* m_curl_multi_handle { nullptr };
|
||||
|
|
@ -164,13 +156,7 @@ private:
|
|||
Optional<RequestPipe> m_client_request_pipe;
|
||||
size_t m_bytes_transferred_to_client { 0 };
|
||||
|
||||
Optional<CacheEntryReader&> m_cache_entry_reader;
|
||||
Optional<CacheEntryWriter&> m_cache_entry_writer;
|
||||
CacheStatus m_cache_status { CacheStatus::Unknown };
|
||||
|
||||
Optional<Requests::NetworkError> m_network_error;
|
||||
|
||||
AK::Duration m_current_time_offset_for_testing;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@
|
|||
#include <LibCore/ArgsParser.h>
|
||||
#include <LibCore/EventLoop.h>
|
||||
#include <LibCore/Process.h>
|
||||
#include <LibHTTP/Cache/DiskCache.h>
|
||||
#include <LibIPC/SingleServer.h>
|
||||
#include <LibMain/Main.h>
|
||||
#include <RequestServer/Cache/DiskCache.h>
|
||||
#include <RequestServer/ConnectionFromClient.h>
|
||||
#include <RequestServer/Resolver.h>
|
||||
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
|
||||
namespace RequestServer {
|
||||
|
||||
extern Optional<DiskCache> g_disk_cache;
|
||||
extern Optional<HTTP::DiskCache> g_disk_cache;
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -60,10 +60,10 @@ ErrorOr<int> ladybird_main(Main::Arguments arguments)
|
|||
|
||||
if (http_disk_cache_mode.is_one_of("enabled"sv, "testing"sv)) {
|
||||
auto mode = http_disk_cache_mode == "enabled"sv
|
||||
? RequestServer::DiskCache::Mode::Normal
|
||||
: RequestServer::DiskCache::Mode::Testing;
|
||||
? HTTP::DiskCache::Mode::Normal
|
||||
: HTTP::DiskCache::Mode::Testing;
|
||||
|
||||
if (auto cache = RequestServer::DiskCache::create(mode); cache.is_error())
|
||||
if (auto cache = HTTP::DiskCache::create(mode); cache.is_error())
|
||||
warnln("Unable to create disk cache: {}", cache.error());
|
||||
else
|
||||
RequestServer::g_disk_cache = cache.release_value();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue