mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2025-12-07 21:59:54 +00:00
LibWebView+Tests+UI: Migrate headless-browser to test-web
Now that headless mode is built into the main Ladybird executable, the headless-browser's only purpose is to run tests. So let's move it to the testing directory and rename it to test-web (a la test-js / test-wasm).
This commit is contained in:
Timothy Flynn
Tim Flynn
parent
c011dc766f
commit
9a5b31ccd1
Notes:
github-actions[bot]
2025-06-10 16:05:57 +00:00
Author: https://github.com/trflynn89
Commit: 9a5b31ccd1
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/5026
21 changed files with 166 additions and 189 deletions
|
|
@ -104,17 +104,14 @@ else()
|
|||
)
|
||||
endif()
|
||||
|
||||
add_subdirectory(Headless)
|
||||
|
||||
set(ladybird_helper_processes ImageDecoder RequestServer WebContent WebWorker)
|
||||
|
||||
add_dependencies(ladybird ${ladybird_helper_processes})
|
||||
add_dependencies(headless-browser ${ladybird_helper_processes} ladybird_build_resource_files)
|
||||
add_dependencies(WebDriver ladybird)
|
||||
|
||||
set_helper_process_properties(${ladybird_helper_processes})
|
||||
if (APPLE)
|
||||
set_helper_process_properties(headless-browser WebDriver)
|
||||
set_helper_process_properties(WebDriver)
|
||||
endif()
|
||||
|
||||
if(NOT CMAKE_SKIP_INSTALL_RULES)
|
||||
|
|
|
|||
|
|
@ -1,113 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <LibCore/ArgsParser.h>
|
||||
#include <LibCore/Environment.h>
|
||||
#include <LibCore/System.h>
|
||||
#include <LibWebView/Utilities.h>
|
||||
#include <UI/Headless/Application.h>
|
||||
#include <UI/Headless/Fixture.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
Application::Application(Badge<WebView::Application>, Main::Arguments&)
|
||||
: resources_folder(WebView::s_ladybird_resource_root)
|
||||
, test_concurrency(Core::System::hardware_concurrency())
|
||||
, python_executable_path("python3")
|
||||
|
||||
{
|
||||
}
|
||||
|
||||
Application::~Application()
|
||||
{
|
||||
for (auto& fixture : Fixture::all())
|
||||
fixture->teardown();
|
||||
}
|
||||
|
||||
void Application::create_platform_arguments(Core::ArgsParser& args_parser)
|
||||
{
|
||||
args_parser.add_option(test_concurrency, "Maximum number of tests to run at once", "test-concurrency", 'j', "jobs");
|
||||
args_parser.add_option(python_executable_path, "Path to python3", "python-executable", 'P', "path");
|
||||
args_parser.add_option(test_globs, "Only run tests matching the given glob", "filter", 'f', "glob");
|
||||
args_parser.add_option(test_dry_run, "List the tests that would be run, without running them", "dry-run");
|
||||
args_parser.add_option(dump_failed_ref_tests, "Dump screenshots of failing ref tests", "dump-failed-ref-tests", 'D');
|
||||
args_parser.add_option(dump_gc_graph, "Dump GC graph", "dump-gc-graph", 'G');
|
||||
args_parser.add_option(resources_folder, "Path of the base resources folder (defaults to /res)", "resources", 'r', "resources-root-path");
|
||||
args_parser.add_option(rebaseline, "Rebaseline any executed layout or text tests", "rebaseline");
|
||||
args_parser.add_option(per_test_timeout_in_seconds, "Per-test timeout (default: 30)", "per-test-timeout", 't', "seconds");
|
||||
|
||||
args_parser.add_option(Core::ArgsParser::Option {
|
||||
.argument_mode = Core::ArgsParser::OptionArgumentMode::Optional,
|
||||
.help_string = "Run tests. If a path is provided, tests are loaded from that path. Otherwise, LADYBIRD_SOURCE_DIR must be set.",
|
||||
.long_name = "run-tests",
|
||||
.short_name = 'R',
|
||||
.value_name = "test-root-path",
|
||||
.accept_value = [&](StringView value) -> ErrorOr<bool> {
|
||||
if (!value.is_empty()) {
|
||||
test_root_path = value;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (auto ladybird_source_dir = Core::Environment::get("LADYBIRD_SOURCE_DIR"sv); ladybird_source_dir.has_value()) {
|
||||
test_root_path = LexicalPath::join(*ladybird_source_dir, "Tests"sv, "LibWeb"sv).string();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
},
|
||||
});
|
||||
|
||||
args_parser.add_option(Core::ArgsParser::Option {
|
||||
.argument_mode = Core::ArgsParser::OptionArgumentMode::Optional,
|
||||
.help_string = "Log extra information about test results (use multiple times for more information)",
|
||||
.long_name = "verbose",
|
||||
.short_name = 'v',
|
||||
.accept_value = [&](StringView value) -> ErrorOr<bool> {
|
||||
if (value.is_empty() && verbosity < NumericLimits<u8>::max()) {
|
||||
++verbosity;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
void Application::create_platform_options(WebView::BrowserOptions& browser_options, WebView::WebContentOptions& web_content_options)
|
||||
{
|
||||
browser_options.headless_mode = WebView::HeadlessMode::Test;
|
||||
web_content_options.is_layout_test_mode = WebView::IsLayoutTestMode::Yes;
|
||||
|
||||
// Allow window.open() to succeed for tests.
|
||||
browser_options.allow_popups = WebView::AllowPopups::Yes;
|
||||
|
||||
// Ensure consistent font rendering between operating systems.
|
||||
web_content_options.force_fontconfig = WebView::ForceFontconfig::Yes;
|
||||
|
||||
// Ensure tests are resilient to minor changes to the viewport scrollbar.
|
||||
web_content_options.paint_viewport_scrollbars = WebView::PaintViewportScrollbars::No;
|
||||
|
||||
if (dump_gc_graph) {
|
||||
// Force all tests to run in serial if we are interested in the GC graph.
|
||||
test_concurrency = 1;
|
||||
}
|
||||
}
|
||||
|
||||
ErrorOr<void> Application::launch_test_fixtures()
|
||||
{
|
||||
Fixture::initialize_fixtures();
|
||||
|
||||
// FIXME: Add option to only run specific fixtures from command line by name
|
||||
// And an option to not run any fixtures at all
|
||||
for (auto const& fixture : Fixture::all()) {
|
||||
if (auto result = fixture->setup(web_content_options()); result.is_error())
|
||||
return result;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/ByteString.h>
|
||||
#include <AK/Error.h>
|
||||
#include <AK/Vector.h>
|
||||
#include <LibWebView/Application.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
class Application : public WebView::Application {
|
||||
WEB_VIEW_APPLICATION(Application)
|
||||
|
||||
public:
|
||||
~Application();
|
||||
|
||||
static Application& the()
|
||||
{
|
||||
return static_cast<Application&>(WebView::Application::the());
|
||||
}
|
||||
|
||||
virtual void create_platform_arguments(Core::ArgsParser&) override;
|
||||
virtual void create_platform_options(WebView::BrowserOptions&, WebView::WebContentOptions&) override;
|
||||
|
||||
ErrorOr<void> launch_test_fixtures();
|
||||
|
||||
static constexpr u8 VERBOSITY_LEVEL_LOG_TEST_DURATION = 1;
|
||||
static constexpr u8 VERBOSITY_LEVEL_LOG_SLOWEST_TESTS = 2;
|
||||
static constexpr u8 VERBOSITY_LEVEL_LOG_SKIPPED_TESTS = 3;
|
||||
|
||||
ByteString resources_folder;
|
||||
bool dump_failed_ref_tests { false };
|
||||
bool dump_gc_graph { false };
|
||||
size_t test_concurrency { 1 };
|
||||
ByteString python_executable_path;
|
||||
ByteString test_root_path;
|
||||
Vector<ByteString> test_globs;
|
||||
bool test_dry_run { false };
|
||||
bool rebaseline { false };
|
||||
u8 verbosity { 0 };
|
||||
int per_test_timeout_in_seconds { 30 };
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
set(SOURCES
|
||||
Application.cpp
|
||||
Fixture.cpp
|
||||
HeadlessWebView.cpp
|
||||
Test.cpp
|
||||
main.cpp
|
||||
)
|
||||
|
||||
add_executable(headless-browser ${SOURCES})
|
||||
target_include_directories(headless-browser PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
target_include_directories(headless-browser PRIVATE ${LADYBIRD_SOURCE_DIR})
|
||||
target_link_libraries(headless-browser PRIVATE ${LADYBIRD_LIBS} LibDiff)
|
||||
|
||||
if (BUILD_TESTING)
|
||||
find_package(Python3 REQUIRED)
|
||||
add_test(
|
||||
NAME LibWeb
|
||||
COMMAND $<TARGET_FILE:headless-browser> --run-tests ${LADYBIRD_SOURCE_DIR}/Tests/LibWeb --python-executable ${Python3_EXECUTABLE} --dump-failed-ref-tests --per-test-timeout 120 --verbose
|
||||
)
|
||||
endif()
|
||||
|
|
@ -1,124 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024, Andrew Kaster <andrew@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/ByteBuffer.h>
|
||||
#include <AK/LexicalPath.h>
|
||||
#include <LibCore/Process.h>
|
||||
#include <LibCore/StandardPaths.h>
|
||||
#include <UI/Headless/Application.h>
|
||||
#include <UI/Headless/Fixture.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
static ByteString s_fixtures_path;
|
||||
|
||||
// Key function for Fixture
|
||||
Fixture::~Fixture() = default;
|
||||
|
||||
Optional<Fixture&> Fixture::lookup(StringView name)
|
||||
{
|
||||
for (auto const& fixture : all()) {
|
||||
if (fixture->name() == name)
|
||||
return *fixture;
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
Vector<NonnullOwnPtr<Fixture>>& Fixture::all()
|
||||
{
|
||||
static Vector<NonnullOwnPtr<Fixture>> fixtures;
|
||||
return fixtures;
|
||||
}
|
||||
|
||||
class HttpEchoServerFixture final : public Fixture {
|
||||
public:
|
||||
virtual ErrorOr<void> setup(WebView::WebContentOptions&) override;
|
||||
virtual void teardown_impl() override;
|
||||
virtual StringView name() const override { return "HttpEchoServer"sv; }
|
||||
virtual bool is_running() const override { return m_process.has_value(); }
|
||||
|
||||
private:
|
||||
ByteString m_script_path { "http-test-server.py" };
|
||||
Optional<Core::Process> m_process;
|
||||
};
|
||||
|
||||
#ifdef AK_OS_WINDOWS
|
||||
// FIXME: Implement Ladybird::HttpEchoServerFixture::setup on Windows
|
||||
ErrorOr<void> HttpEchoServerFixture::setup(WebView::WebContentOptions&)
|
||||
{
|
||||
VERIFY(0 && "Ladybird::HttpEchoServerFixture::setup is not implemented");
|
||||
}
|
||||
// FIXME: Implement Ladybird::HttpEchoServerFixture::teardown_impl on Windows
|
||||
void HttpEchoServerFixture::teardown_impl()
|
||||
{
|
||||
VERIFY(0 && "Ladybird::HttpEchoServerFixture::teardown_impl is not implemented");
|
||||
}
|
||||
#else
|
||||
ErrorOr<void> HttpEchoServerFixture::setup(WebView::WebContentOptions& web_content_options)
|
||||
{
|
||||
auto const script_path = LexicalPath::join(s_fixtures_path, m_script_path);
|
||||
auto const arguments = Vector { script_path.string(), "--directory", Ladybird::Application::the().test_root_path };
|
||||
|
||||
// FIXME: Pick a more reasonable log path that is more observable
|
||||
auto const log_path = LexicalPath::join(Core::StandardPaths::tempfile_directory(), "http-test-server.log"sv).string();
|
||||
|
||||
auto stdout_fds = TRY(Core::System::pipe2(0));
|
||||
|
||||
auto const process_options = Core::ProcessSpawnOptions {
|
||||
.executable = Ladybird::Application::the().python_executable_path,
|
||||
.search_for_executable_in_path = true,
|
||||
.arguments = arguments,
|
||||
.file_actions = {
|
||||
Core::FileAction::OpenFile { ByteString::formatted("{}.stderr", log_path), Core::File::OpenMode::Write, STDERR_FILENO },
|
||||
Core::FileAction::DupFd { stdout_fds[1], STDOUT_FILENO } }
|
||||
};
|
||||
|
||||
m_process = TRY(Core::Process::spawn(process_options));
|
||||
|
||||
TRY(Core::System::close(stdout_fds[1]));
|
||||
|
||||
auto const stdout_file = MUST(Core::File::adopt_fd(stdout_fds[0], Core::File::OpenMode::Read));
|
||||
|
||||
auto buffer = MUST(ByteBuffer::create_uninitialized(5));
|
||||
TRY(stdout_file->read_some(buffer));
|
||||
|
||||
auto const raw_output = ByteString { buffer, AK::ShouldChomp::NoChomp };
|
||||
|
||||
if (auto const maybe_port = raw_output.to_number<u16>(); maybe_port.has_value())
|
||||
web_content_options.echo_server_port = maybe_port.value();
|
||||
else
|
||||
warnln("Failed to read echo server port from buffer: '{}'", raw_output);
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
void HttpEchoServerFixture::teardown_impl()
|
||||
{
|
||||
VERIFY(m_process.has_value());
|
||||
|
||||
auto script_path = LexicalPath::join(s_fixtures_path, m_script_path);
|
||||
|
||||
if (auto kill_or_error = Core::System::kill(m_process->pid(), SIGINT); kill_or_error.is_error()) {
|
||||
if (kill_or_error.error().code() != ESRCH) {
|
||||
warnln("Failed to kill HTTP echo server, error: {}", kill_or_error.error());
|
||||
} else if (auto termination_or_error = m_process->wait_for_termination(); termination_or_error.is_error()) {
|
||||
warnln("Failed to terminate HTTP echo server, error: {}", termination_or_error.error());
|
||||
}
|
||||
}
|
||||
|
||||
m_process = {};
|
||||
}
|
||||
#endif
|
||||
|
||||
void Fixture::initialize_fixtures()
|
||||
{
|
||||
s_fixtures_path = LexicalPath::join(Ladybird::Application::the().test_root_path, "Fixtures"sv).string();
|
||||
|
||||
auto& registry = all();
|
||||
registry.append(make<HttpEchoServerFixture>());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024, Andrew Kaster <andrew@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Error.h>
|
||||
#include <AK/NonnullOwnPtr.h>
|
||||
#include <AK/Optional.h>
|
||||
#include <AK/StringView.h>
|
||||
#include <AK/Vector.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
class Fixture {
|
||||
public:
|
||||
virtual ~Fixture();
|
||||
|
||||
virtual ErrorOr<void> setup(WebView::WebContentOptions&) = 0;
|
||||
|
||||
void teardown()
|
||||
{
|
||||
if (is_running())
|
||||
teardown_impl();
|
||||
}
|
||||
|
||||
virtual StringView name() const = 0;
|
||||
virtual bool is_running() const { return false; }
|
||||
|
||||
static void initialize_fixtures();
|
||||
static Optional<Fixture&> lookup(StringView name);
|
||||
static Vector<NonnullOwnPtr<Fixture>>& all();
|
||||
|
||||
protected:
|
||||
virtual void teardown_impl() = 0;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <LibGfx/Bitmap.h>
|
||||
#include <LibGfx/ShareableBitmap.h>
|
||||
#include <UI/Headless/HeadlessWebView.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
NonnullOwnPtr<HeadlessWebView> HeadlessWebView::create(Core::AnonymousBuffer theme, Web::DevicePixelSize window_size)
|
||||
{
|
||||
auto view = adopt_own(*new HeadlessWebView(move(theme), window_size));
|
||||
view->initialize_client(CreateNewClient::Yes);
|
||||
|
||||
return view;
|
||||
}
|
||||
|
||||
HeadlessWebView::HeadlessWebView(Core::AnonymousBuffer theme, Web::DevicePixelSize viewport_size)
|
||||
: WebView::HeadlessWebView(move(theme), viewport_size)
|
||||
, m_test_promise(TestPromise::construct())
|
||||
{
|
||||
}
|
||||
|
||||
void HeadlessWebView::clear_content_filters()
|
||||
{
|
||||
client().async_set_content_filters(m_client_state.page_index, {});
|
||||
}
|
||||
|
||||
NonnullRefPtr<Core::Promise<RefPtr<Gfx::Bitmap const>>> HeadlessWebView::take_screenshot()
|
||||
{
|
||||
VERIFY(!m_pending_screenshot);
|
||||
|
||||
m_pending_screenshot = Core::Promise<RefPtr<Gfx::Bitmap const>>::construct();
|
||||
client().async_take_document_screenshot(0);
|
||||
|
||||
return *m_pending_screenshot;
|
||||
}
|
||||
|
||||
void HeadlessWebView::did_receive_screenshot(Badge<WebView::WebContentClient>, Gfx::ShareableBitmap const& screenshot)
|
||||
{
|
||||
VERIFY(m_pending_screenshot);
|
||||
|
||||
auto pending_screenshot = move(m_pending_screenshot);
|
||||
pending_screenshot->resolve(screenshot.bitmap());
|
||||
}
|
||||
|
||||
void HeadlessWebView::on_test_complete(TestCompletion completion)
|
||||
{
|
||||
m_pending_screenshot.clear();
|
||||
m_pending_dialog = Web::Page::PendingDialog::None;
|
||||
m_pending_prompt_text.clear();
|
||||
|
||||
m_test_promise->resolve(move(completion));
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,41 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Badge.h>
|
||||
#include <AK/RefPtr.h>
|
||||
#include <LibCore/Forward.h>
|
||||
#include <LibCore/Promise.h>
|
||||
#include <LibGfx/Forward.h>
|
||||
#include <LibWeb/PixelUnits.h>
|
||||
#include <LibWebView/HeadlessWebView.h>
|
||||
#include <UI/Headless/Test.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
class HeadlessWebView final : public WebView::HeadlessWebView {
|
||||
public:
|
||||
static NonnullOwnPtr<HeadlessWebView> create(Core::AnonymousBuffer theme, Web::DevicePixelSize window_size);
|
||||
|
||||
void clear_content_filters();
|
||||
|
||||
NonnullRefPtr<Core::Promise<RefPtr<Gfx::Bitmap const>>> take_screenshot();
|
||||
|
||||
TestPromise& test_promise() { return *m_test_promise; }
|
||||
void on_test_complete(TestCompletion);
|
||||
|
||||
private:
|
||||
HeadlessWebView(Core::AnonymousBuffer theme, Web::DevicePixelSize viewport_size);
|
||||
|
||||
virtual void did_receive_screenshot(Badge<WebView::WebContentClient>, Gfx::ShareableBitmap const& screenshot) override;
|
||||
|
||||
RefPtr<Core::Promise<RefPtr<Gfx::Bitmap const>>> m_pending_screenshot;
|
||||
|
||||
NonnullRefPtr<TestPromise> m_test_promise;
|
||||
};
|
||||
|
||||
}
|
||||
|
|
@ -1,687 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024-2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/ByteBuffer.h>
|
||||
#include <AK/ByteString.h>
|
||||
#include <AK/Enumerate.h>
|
||||
#include <AK/LexicalPath.h>
|
||||
#include <AK/QuickSort.h>
|
||||
#include <AK/Vector.h>
|
||||
#include <LibCore/ConfigFile.h>
|
||||
#include <LibCore/DirIterator.h>
|
||||
#include <LibCore/Directory.h>
|
||||
#include <LibCore/EventLoop.h>
|
||||
#include <LibCore/File.h>
|
||||
#include <LibCore/Timer.h>
|
||||
#include <LibDiff/Format.h>
|
||||
#include <LibDiff/Generator.h>
|
||||
#include <LibFileSystem/FileSystem.h>
|
||||
#include <LibGfx/Bitmap.h>
|
||||
#include <LibGfx/ImageFormats/PNGWriter.h>
|
||||
#include <LibURL/URL.h>
|
||||
#include <LibWeb/HTML/SelectedFile.h>
|
||||
#include <UI/Headless/Application.h>
|
||||
#include <UI/Headless/HeadlessWebView.h>
|
||||
#include <UI/Headless/Test.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
static Vector<ByteString> s_skipped_tests;
|
||||
|
||||
static ErrorOr<void> load_test_config(StringView test_root_path)
|
||||
{
|
||||
auto config_path = LexicalPath::join(test_root_path, "TestConfig.ini"sv);
|
||||
auto config_or_error = Core::ConfigFile::open(config_path.string());
|
||||
|
||||
if (config_or_error.is_error()) {
|
||||
if (config_or_error.error().code() == ENOENT)
|
||||
return {};
|
||||
warnln("Unable to open test config {}", config_path);
|
||||
return config_or_error.release_error();
|
||||
}
|
||||
|
||||
auto config = config_or_error.release_value();
|
||||
|
||||
for (auto const& group : config->groups()) {
|
||||
if (group == "Skipped"sv) {
|
||||
for (auto& key : config->keys(group))
|
||||
s_skipped_tests.append(TRY(FileSystem::real_path(LexicalPath::join(test_root_path, key).string())));
|
||||
} else {
|
||||
warnln("Unknown group '{}' in config {}", group, config_path);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static bool is_valid_test_name(StringView test_name)
|
||||
{
|
||||
auto valid_test_file_suffixes = { ".htm"sv, ".html"sv, ".svg"sv, ".xhtml"sv, ".xht"sv };
|
||||
return AK::any_of(valid_test_file_suffixes, [&](auto suffix) { return test_name.ends_with(suffix); });
|
||||
}
|
||||
|
||||
static ErrorOr<void> collect_dump_tests(Application const& app, Vector<Test>& tests, StringView path, StringView trail, TestMode mode)
|
||||
{
|
||||
Core::DirIterator it(ByteString::formatted("{}/input/{}", path, trail), Core::DirIterator::Flags::SkipDots);
|
||||
|
||||
while (it.has_next()) {
|
||||
auto name = it.next_path();
|
||||
auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/input/{}/{}", path, trail, name)));
|
||||
|
||||
if (FileSystem::is_directory(input_path)) {
|
||||
TRY(collect_dump_tests(app, tests, path, ByteString::formatted("{}/{}", trail, name), mode));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!is_valid_test_name(name))
|
||||
continue;
|
||||
|
||||
auto expectation_path = ByteString::formatted("{}/expected/{}/{}.txt", path, trail, LexicalPath::title(name));
|
||||
auto relative_path = LexicalPath::relative_path(input_path, app.test_root_path).release_value();
|
||||
tests.append({ mode, input_path, move(expectation_path), move(relative_path) });
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static ErrorOr<void> collect_ref_tests(Application const& app, Vector<Test>& tests, StringView path, StringView trail)
|
||||
{
|
||||
Core::DirIterator it(ByteString::formatted("{}/input/{}", path, trail), Core::DirIterator::Flags::SkipDots);
|
||||
while (it.has_next()) {
|
||||
auto name = it.next_path();
|
||||
auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/input/{}/{}", path, trail, name)));
|
||||
|
||||
if (FileSystem::is_directory(input_path)) {
|
||||
TRY(collect_ref_tests(app, tests, path, ByteString::formatted("{}/{}", trail, name)));
|
||||
continue;
|
||||
}
|
||||
|
||||
auto relative_path = LexicalPath::relative_path(input_path, app.test_root_path).release_value();
|
||||
tests.append({ TestMode::Ref, input_path, {}, move(relative_path) });
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static ErrorOr<void> collect_crash_tests(Application const& app, Vector<Test>& tests, StringView path, StringView trail)
|
||||
{
|
||||
Core::DirIterator it(ByteString::formatted("{}/{}", path, trail), Core::DirIterator::Flags::SkipDots);
|
||||
while (it.has_next()) {
|
||||
auto name = it.next_path();
|
||||
auto input_path = TRY(FileSystem::real_path(ByteString::formatted("{}/{}/{}", path, trail, name)));
|
||||
|
||||
if (FileSystem::is_directory(input_path)) {
|
||||
TRY(collect_crash_tests(app, tests, path, ByteString::formatted("{}/{}", trail, name)));
|
||||
continue;
|
||||
}
|
||||
if (!is_valid_test_name(name))
|
||||
continue;
|
||||
|
||||
auto relative_path = LexicalPath::relative_path(input_path, app.test_root_path).release_value();
|
||||
tests.append({ TestMode::Crash, input_path, {}, move(relative_path) });
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static void clear_test_callbacks(HeadlessWebView& view)
|
||||
{
|
||||
view.on_load_finish = {};
|
||||
view.on_test_finish = {};
|
||||
view.on_web_content_crashed = {};
|
||||
}
|
||||
|
||||
void run_dump_test(HeadlessWebView& view, Test& test, URL::URL const& url, int timeout_in_milliseconds)
|
||||
{
|
||||
auto timer = Core::Timer::create_single_shot(timeout_in_milliseconds, [&view, &test]() {
|
||||
view.on_load_finish = {};
|
||||
view.on_test_finish = {};
|
||||
view.on_set_test_timeout = {};
|
||||
view.reset_zoom();
|
||||
|
||||
view.on_test_complete({ test, TestResult::Timeout });
|
||||
});
|
||||
|
||||
auto handle_completed_test = [&test, url]() -> ErrorOr<TestResult> {
|
||||
if (test.expectation_path.is_empty()) {
|
||||
if (test.mode != TestMode::Crash)
|
||||
outln("{}", test.text);
|
||||
return TestResult::Pass;
|
||||
}
|
||||
|
||||
auto open_expectation_file = [&](auto mode) {
|
||||
auto expectation_file_or_error = Core::File::open(test.expectation_path, mode);
|
||||
if (expectation_file_or_error.is_error())
|
||||
warnln("Failed opening '{}': {}", test.expectation_path, expectation_file_or_error.error());
|
||||
|
||||
return expectation_file_or_error;
|
||||
};
|
||||
|
||||
ByteBuffer expectation;
|
||||
|
||||
if (auto expectation_file = open_expectation_file(Core::File::OpenMode::Read); !expectation_file.is_error()) {
|
||||
expectation = TRY(expectation_file.value()->read_until_eof());
|
||||
|
||||
auto result_trimmed = StringView { test.text }.trim("\n"sv, TrimMode::Right);
|
||||
auto expectation_trimmed = StringView { expectation }.trim("\n"sv, TrimMode::Right);
|
||||
|
||||
if (result_trimmed == expectation_trimmed)
|
||||
return TestResult::Pass;
|
||||
} else if (!Application::the().rebaseline) {
|
||||
return expectation_file.release_error();
|
||||
}
|
||||
|
||||
if (Application::the().rebaseline) {
|
||||
TRY(Core::Directory::create(LexicalPath { test.expectation_path }.parent().string(), Core::Directory::CreateDirectories::Yes));
|
||||
|
||||
auto expectation_file = TRY(open_expectation_file(Core::File::OpenMode::Write));
|
||||
TRY(expectation_file->write_until_depleted(test.text));
|
||||
|
||||
return TestResult::Pass;
|
||||
}
|
||||
|
||||
auto const color_output = TRY(Core::System::isatty(STDOUT_FILENO)) ? Diff::ColorOutput::Yes : Diff::ColorOutput::No;
|
||||
|
||||
if (color_output == Diff::ColorOutput::Yes)
|
||||
outln("\n\033[33;1mTest failed\033[0m: {}", url);
|
||||
else
|
||||
outln("\nTest failed: {}", url);
|
||||
|
||||
auto hunks = TRY(Diff::from_text(expectation, test.text, 3));
|
||||
auto out = TRY(Core::File::standard_output());
|
||||
|
||||
TRY(Diff::write_unified_header(test.expectation_path, test.expectation_path, *out));
|
||||
for (auto const& hunk : hunks)
|
||||
TRY(Diff::write_unified(hunk, *out, color_output));
|
||||
|
||||
return TestResult::Fail;
|
||||
};
|
||||
|
||||
auto on_test_complete = [&view, &test, timer, handle_completed_test]() {
|
||||
view.reset_zoom();
|
||||
clear_test_callbacks(view);
|
||||
timer->stop();
|
||||
|
||||
if (auto result = handle_completed_test(); result.is_error())
|
||||
view.on_test_complete({ test, TestResult::Fail });
|
||||
else
|
||||
view.on_test_complete({ test, result.value() });
|
||||
};
|
||||
|
||||
view.on_web_content_crashed = [&view, &test, timer]() {
|
||||
clear_test_callbacks(view);
|
||||
timer->stop();
|
||||
|
||||
view.on_test_complete({ test, TestResult::Crashed });
|
||||
};
|
||||
|
||||
if (test.mode == TestMode::Layout) {
|
||||
view.on_load_finish = [&view, &test, url, on_test_complete = move(on_test_complete)](auto const& loaded_url) {
|
||||
// We don't want subframe loads to trigger the test finish.
|
||||
if (!url.equals(loaded_url, URL::ExcludeFragment::Yes))
|
||||
return;
|
||||
|
||||
// NOTE: We take a screenshot here to force the lazy layout of SVG-as-image documents to happen.
|
||||
// It also causes a lot more code to run, which is good for finding bugs. :^)
|
||||
view.take_screenshot()->when_resolved([&view, &test, on_test_complete = move(on_test_complete)](auto) {
|
||||
auto promise = view.request_internal_page_info(WebView::PageInfoType::LayoutTree | WebView::PageInfoType::PaintTree);
|
||||
|
||||
promise->when_resolved([&test, on_test_complete = move(on_test_complete)](auto const& text) {
|
||||
test.text = text;
|
||||
on_test_complete();
|
||||
});
|
||||
});
|
||||
};
|
||||
} else if (test.mode == TestMode::Text) {
|
||||
view.on_load_finish = [&view, &test, on_test_complete, url](auto const& loaded_url) {
|
||||
// We don't want subframe loads to trigger the test finish.
|
||||
if (!url.equals(loaded_url, URL::ExcludeFragment::Yes))
|
||||
return;
|
||||
|
||||
test.did_finish_loading = true;
|
||||
|
||||
if (test.expectation_path.is_empty()) {
|
||||
auto promise = view.request_internal_page_info(WebView::PageInfoType::Text);
|
||||
|
||||
promise->when_resolved([&test, on_test_complete = move(on_test_complete)](auto const& text) {
|
||||
test.text = text;
|
||||
on_test_complete();
|
||||
});
|
||||
} else if (test.did_finish_test) {
|
||||
on_test_complete();
|
||||
}
|
||||
};
|
||||
|
||||
view.on_test_finish = [&test, on_test_complete](auto const& text) {
|
||||
test.text = text;
|
||||
test.did_finish_test = true;
|
||||
|
||||
if (test.did_finish_loading)
|
||||
on_test_complete();
|
||||
};
|
||||
} else if (test.mode == TestMode::Crash) {
|
||||
view.on_load_finish = [on_test_complete = move(on_test_complete), url](auto const& loaded_url) {
|
||||
// We don't want subframe loads to trigger the test finish.
|
||||
if (!url.equals(loaded_url, URL::ExcludeFragment::Yes))
|
||||
return;
|
||||
on_test_complete();
|
||||
};
|
||||
}
|
||||
|
||||
view.on_set_test_timeout = [timer, timeout_in_milliseconds](double milliseconds) {
|
||||
if (milliseconds <= timeout_in_milliseconds)
|
||||
return;
|
||||
timer->stop();
|
||||
timer->start(milliseconds);
|
||||
};
|
||||
|
||||
view.on_set_browser_zoom = [&view](double factor) {
|
||||
view.set_zoom(factor);
|
||||
};
|
||||
|
||||
view.load(url);
|
||||
timer->start();
|
||||
}
|
||||
|
||||
static String wait_for_reftest_completion = R"(
|
||||
function hasReftestWaitClass() {
|
||||
return document.documentElement.classList.contains('reftest-wait');
|
||||
}
|
||||
|
||||
if (!hasReftestWaitClass()) {
|
||||
internals.signalTestIsDone("PASS");
|
||||
} else {
|
||||
const observer = new MutationObserver(() => {
|
||||
if (!hasReftestWaitClass()) {
|
||||
internals.signalTestIsDone("PASS");
|
||||
}
|
||||
});
|
||||
|
||||
observer.observe(document.documentElement, {
|
||||
attributes: true,
|
||||
attributeFilter: ['class'],
|
||||
});
|
||||
}
|
||||
)"_string;
|
||||
|
||||
static void run_ref_test(HeadlessWebView& view, Test& test, URL::URL const& url, int timeout_in_milliseconds)
|
||||
{
|
||||
auto timer = Core::Timer::create_single_shot(timeout_in_milliseconds, [&view, &test]() {
|
||||
view.on_load_finish = {};
|
||||
view.on_test_finish = {};
|
||||
view.on_set_test_timeout = {};
|
||||
view.reset_zoom();
|
||||
|
||||
view.on_test_complete({ test, TestResult::Timeout });
|
||||
});
|
||||
|
||||
auto handle_completed_test = [&test, url]() -> ErrorOr<TestResult> {
|
||||
VERIFY(test.ref_test_expectation_type.has_value());
|
||||
auto should_match = test.ref_test_expectation_type == RefTestExpectationType::Match;
|
||||
auto screenshot_matches = test.actual_screenshot->visually_equals(*test.expectation_screenshot);
|
||||
if (should_match == screenshot_matches)
|
||||
return TestResult::Pass;
|
||||
|
||||
if (Application::the().dump_failed_ref_tests) {
|
||||
warnln("\033[33;1mRef test {} failed; dumping screenshots\033[0m", test.relative_path);
|
||||
|
||||
auto dump_screenshot = [&](Gfx::Bitmap const& bitmap, StringView path) -> ErrorOr<void> {
|
||||
auto screenshot_file = TRY(Core::File::open(path, Core::File::OpenMode::Write));
|
||||
auto encoded_data = TRY(Gfx::PNGWriter::encode(bitmap));
|
||||
TRY(screenshot_file->write_until_depleted(encoded_data));
|
||||
|
||||
outln("\033[33;1mDumped {}\033[0m", TRY(FileSystem::real_path(path)));
|
||||
return {};
|
||||
};
|
||||
|
||||
TRY(Core::Directory::create("test-dumps"sv, Core::Directory::CreateDirectories::Yes));
|
||||
|
||||
auto title = LexicalPath::title(URL::percent_decode(url.serialize_path()));
|
||||
TRY(dump_screenshot(*test.actual_screenshot, ByteString::formatted("test-dumps/{}.png", title)));
|
||||
TRY(dump_screenshot(*test.expectation_screenshot, ByteString::formatted("test-dumps/{}-ref.png", title)));
|
||||
}
|
||||
|
||||
return TestResult::Fail;
|
||||
};
|
||||
|
||||
auto on_test_complete = [&view, &test, timer, handle_completed_test]() {
|
||||
clear_test_callbacks(view);
|
||||
timer->stop();
|
||||
|
||||
if (auto result = handle_completed_test(); result.is_error())
|
||||
view.on_test_complete({ test, TestResult::Fail });
|
||||
else
|
||||
view.on_test_complete({ test, result.value() });
|
||||
};
|
||||
|
||||
view.on_web_content_crashed = [&view, &test, timer]() {
|
||||
clear_test_callbacks(view);
|
||||
timer->stop();
|
||||
|
||||
view.on_test_complete({ test, TestResult::Crashed });
|
||||
};
|
||||
|
||||
view.on_load_finish = [&view](auto const&) {
|
||||
view.run_javascript(wait_for_reftest_completion);
|
||||
};
|
||||
|
||||
view.on_test_finish = [&view, &test, on_test_complete = move(on_test_complete)](auto const&) {
|
||||
if (test.actual_screenshot) {
|
||||
if (view.url().query().has_value() && view.url().query()->equals_ignoring_ascii_case("mismatch"sv)) {
|
||||
test.ref_test_expectation_type = RefTestExpectationType::Mismatch;
|
||||
} else {
|
||||
test.ref_test_expectation_type = RefTestExpectationType::Match;
|
||||
}
|
||||
view.take_screenshot()->when_resolved([&view, &test, on_test_complete = move(on_test_complete)](RefPtr<Gfx::Bitmap const> screenshot) {
|
||||
test.expectation_screenshot = move(screenshot);
|
||||
view.reset_zoom();
|
||||
on_test_complete();
|
||||
});
|
||||
} else {
|
||||
view.take_screenshot()->when_resolved([&view, &test](RefPtr<Gfx::Bitmap const> screenshot) {
|
||||
test.actual_screenshot = move(screenshot);
|
||||
view.reset_zoom();
|
||||
view.debug_request("load-reference-page");
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
view.on_set_test_timeout = [timer, timeout_in_milliseconds](double milliseconds) {
|
||||
if (milliseconds <= timeout_in_milliseconds)
|
||||
return;
|
||||
timer->stop();
|
||||
timer->start(milliseconds);
|
||||
};
|
||||
|
||||
view.on_set_browser_zoom = [&view](double factor) {
|
||||
view.set_zoom(factor);
|
||||
};
|
||||
|
||||
view.load(url);
|
||||
timer->start();
|
||||
}
|
||||
|
||||
static void run_test(HeadlessWebView& view, Test& test, Application& app)
|
||||
{
|
||||
// Clear the current document.
|
||||
// FIXME: Implement a debug-request to do this more thoroughly.
|
||||
auto promise = Core::Promise<Empty>::construct();
|
||||
|
||||
view.on_load_finish = [promise](auto const& url) {
|
||||
if (!url.equals(URL::about_blank()))
|
||||
return;
|
||||
|
||||
Core::deferred_invoke([promise]() {
|
||||
promise->resolve({});
|
||||
});
|
||||
};
|
||||
|
||||
view.on_test_finish = {};
|
||||
|
||||
promise->when_resolved([&view, &test, &app](auto) {
|
||||
auto url = URL::create_with_file_scheme(MUST(FileSystem::real_path(test.input_path))).release_value();
|
||||
|
||||
switch (test.mode) {
|
||||
case TestMode::Crash:
|
||||
case TestMode::Text:
|
||||
case TestMode::Layout:
|
||||
run_dump_test(view, test, url, app.per_test_timeout_in_seconds * 1000);
|
||||
return;
|
||||
case TestMode::Ref:
|
||||
run_ref_test(view, test, url, app.per_test_timeout_in_seconds * 1000);
|
||||
return;
|
||||
}
|
||||
|
||||
VERIFY_NOT_REACHED();
|
||||
});
|
||||
|
||||
view.load(URL::about_blank());
|
||||
}
|
||||
|
||||
static void set_ui_callbacks_for_tests(HeadlessWebView& view)
|
||||
{
|
||||
view.on_request_file_picker = [&](auto const& accepted_file_types, auto allow_multiple_files) {
|
||||
// Create some dummy files for tests.
|
||||
Vector<Web::HTML::SelectedFile> selected_files;
|
||||
|
||||
bool add_txt_files = accepted_file_types.filters.is_empty();
|
||||
bool add_cpp_files = false;
|
||||
|
||||
for (auto const& filter : accepted_file_types.filters) {
|
||||
filter.visit(
|
||||
[](Web::HTML::FileFilter::FileType) {},
|
||||
[&](Web::HTML::FileFilter::MimeType const& mime_type) {
|
||||
if (mime_type.value == "text/plain"sv)
|
||||
add_txt_files = true;
|
||||
},
|
||||
[&](Web::HTML::FileFilter::Extension const& extension) {
|
||||
if (extension.value == "cpp"sv)
|
||||
add_cpp_files = true;
|
||||
});
|
||||
}
|
||||
|
||||
if (add_txt_files) {
|
||||
selected_files.empend("file1"sv, MUST(ByteBuffer::copy("Contents for file1"sv.bytes())));
|
||||
|
||||
if (allow_multiple_files == Web::HTML::AllowMultipleFiles::Yes) {
|
||||
selected_files.empend("file2"sv, MUST(ByteBuffer::copy("Contents for file2"sv.bytes())));
|
||||
selected_files.empend("file3"sv, MUST(ByteBuffer::copy("Contents for file3"sv.bytes())));
|
||||
selected_files.empend("file4"sv, MUST(ByteBuffer::copy("Contents for file4"sv.bytes())));
|
||||
}
|
||||
}
|
||||
|
||||
if (add_cpp_files) {
|
||||
selected_files.empend("file1.cpp"sv, MUST(ByteBuffer::copy("int main() {{ return 1; }}"sv.bytes())));
|
||||
|
||||
if (allow_multiple_files == Web::HTML::AllowMultipleFiles::Yes) {
|
||||
selected_files.empend("file2.cpp"sv, MUST(ByteBuffer::copy("int main() {{ return 2; }}"sv.bytes())));
|
||||
}
|
||||
}
|
||||
|
||||
view.file_picker_closed(move(selected_files));
|
||||
};
|
||||
|
||||
view.on_request_alert = [&](auto const&) {
|
||||
// For tests, just close the alert right away to unblock JS execution.
|
||||
view.alert_closed();
|
||||
};
|
||||
}
|
||||
|
||||
ErrorOr<int> run_tests(Core::AnonymousBuffer const& theme, Web::DevicePixelSize window_size)
|
||||
{
|
||||
auto& app = Application::the();
|
||||
TRY(load_test_config(app.test_root_path));
|
||||
|
||||
Vector<Test> tests;
|
||||
|
||||
for (auto& glob : app.test_globs)
|
||||
glob = ByteString::formatted("*{}*", glob);
|
||||
if (app.test_globs.is_empty())
|
||||
app.test_globs.append("*"sv);
|
||||
|
||||
TRY(collect_dump_tests(app, tests, ByteString::formatted("{}/Layout", app.test_root_path), "."sv, TestMode::Layout));
|
||||
TRY(collect_dump_tests(app, tests, ByteString::formatted("{}/Text", app.test_root_path), "."sv, TestMode::Text));
|
||||
TRY(collect_ref_tests(app, tests, ByteString::formatted("{}/Ref", app.test_root_path), "."sv));
|
||||
TRY(collect_crash_tests(app, tests, ByteString::formatted("{}/Crash", app.test_root_path), "."sv));
|
||||
#if defined(AK_OS_LINUX) && ARCH(X86_64)
|
||||
TRY(collect_ref_tests(app, tests, ByteString::formatted("{}/Screenshot", app.test_root_path), "."sv));
|
||||
#endif
|
||||
|
||||
tests.remove_all_matching([&](auto const& test) {
|
||||
static constexpr Array support_file_patterns {
|
||||
"*/wpt-import/*/support/*"sv,
|
||||
"*/wpt-import/*/resources/*"sv,
|
||||
"*/wpt-import/common/*"sv,
|
||||
};
|
||||
bool is_support_file = any_of(support_file_patterns, [&](auto pattern) { return test.input_path.matches(pattern); });
|
||||
bool match_glob = any_of(app.test_globs, [&](auto const& glob) { return test.relative_path.matches(glob, CaseSensitivity::CaseSensitive); });
|
||||
return is_support_file || !match_glob;
|
||||
});
|
||||
|
||||
if (app.test_dry_run) {
|
||||
outln("Found {} tests...", tests.size());
|
||||
|
||||
for (auto const& [i, test] : enumerate(tests))
|
||||
outln("{}/{}: {}", i + 1, tests.size(), test.relative_path);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (tests.is_empty()) {
|
||||
if (app.test_globs.is_empty())
|
||||
return Error::from_string_literal("No tests found");
|
||||
return Error::from_string_literal("No tests found matching filter");
|
||||
}
|
||||
|
||||
auto concurrency = min(app.test_concurrency, tests.size());
|
||||
size_t loaded_web_views = 0;
|
||||
|
||||
Vector<NonnullOwnPtr<HeadlessWebView>> views;
|
||||
views.ensure_capacity(concurrency);
|
||||
|
||||
for (size_t i = 0; i < concurrency; ++i) {
|
||||
auto view = HeadlessWebView::create(theme, window_size);
|
||||
view->on_load_finish = [&](auto const&) { ++loaded_web_views; };
|
||||
|
||||
views.unchecked_append(move(view));
|
||||
}
|
||||
|
||||
// We need to wait for the initial about:blank load to complete before starting the tests, otherwise we may load the
|
||||
// test URL before the about:blank load completes. WebContent currently cannot handle this, and will drop the test URL.
|
||||
Core::EventLoop::current().spin_until([&]() {
|
||||
return loaded_web_views == concurrency;
|
||||
});
|
||||
|
||||
size_t pass_count = 0;
|
||||
size_t fail_count = 0;
|
||||
size_t timeout_count = 0;
|
||||
size_t crashed_count = 0;
|
||||
size_t skipped_count = 0;
|
||||
|
||||
// Keep clearing and reusing the same line if stdout is a TTY.
|
||||
bool log_on_one_line = app.verbosity < Application::VERBOSITY_LEVEL_LOG_TEST_DURATION && TRY(Core::System::isatty(STDOUT_FILENO));
|
||||
outln("Running {} tests...", tests.size());
|
||||
|
||||
auto all_tests_complete = Core::Promise<Empty>::construct();
|
||||
auto tests_remaining = tests.size();
|
||||
auto current_test = 0uz;
|
||||
|
||||
Vector<TestCompletion> non_passing_tests;
|
||||
|
||||
for (auto& view : views) {
|
||||
set_ui_callbacks_for_tests(*view);
|
||||
view->clear_content_filters();
|
||||
|
||||
auto run_next_test = [&]() {
|
||||
auto index = current_test++;
|
||||
if (index >= tests.size())
|
||||
return;
|
||||
|
||||
auto& test = tests[index];
|
||||
test.start_time = UnixDateTime::now();
|
||||
test.index = index + 1;
|
||||
|
||||
if (log_on_one_line) {
|
||||
out("\33[2K\r{}/{}: {}", test.index, tests.size(), test.relative_path);
|
||||
(void)fflush(stdout);
|
||||
} else {
|
||||
outln("{}/{}: Start {}", test.index, tests.size(), test.relative_path);
|
||||
}
|
||||
|
||||
Core::deferred_invoke([&]() mutable {
|
||||
if (s_skipped_tests.contains_slow(test.input_path))
|
||||
view->on_test_complete({ test, TestResult::Skipped });
|
||||
else
|
||||
run_test(*view, test, app);
|
||||
});
|
||||
};
|
||||
|
||||
view->test_promise().when_resolved([&, run_next_test](auto result) {
|
||||
result.test.end_time = UnixDateTime::now();
|
||||
|
||||
if (app.verbosity >= Application::VERBOSITY_LEVEL_LOG_TEST_DURATION) {
|
||||
auto duration = result.test.end_time - result.test.start_time;
|
||||
outln("{}/{}: Finish {}: {}ms", result.test.index, tests.size(), result.test.relative_path, duration.to_milliseconds());
|
||||
}
|
||||
|
||||
switch (result.result) {
|
||||
case TestResult::Pass:
|
||||
++pass_count;
|
||||
break;
|
||||
case TestResult::Fail:
|
||||
++fail_count;
|
||||
break;
|
||||
case TestResult::Timeout:
|
||||
++timeout_count;
|
||||
break;
|
||||
case TestResult::Crashed:
|
||||
++crashed_count;
|
||||
break;
|
||||
case TestResult::Skipped:
|
||||
++skipped_count;
|
||||
break;
|
||||
}
|
||||
|
||||
if (result.result != TestResult::Pass)
|
||||
non_passing_tests.append(move(result));
|
||||
|
||||
if (--tests_remaining == 0)
|
||||
all_tests_complete->resolve({});
|
||||
else
|
||||
run_next_test();
|
||||
});
|
||||
|
||||
Core::deferred_invoke([run_next_test]() {
|
||||
run_next_test();
|
||||
});
|
||||
}
|
||||
|
||||
MUST(all_tests_complete->await());
|
||||
|
||||
if (log_on_one_line)
|
||||
outln("\33[2K\rDone!");
|
||||
|
||||
outln("==========================================================");
|
||||
outln("Pass: {}, Fail: {}, Skipped: {}, Timeout: {}, Crashed: {}", pass_count, fail_count, skipped_count, timeout_count, crashed_count);
|
||||
outln("==========================================================");
|
||||
|
||||
for (auto const& non_passing_test : non_passing_tests) {
|
||||
if (non_passing_test.result == TestResult::Skipped && app.verbosity < Application::VERBOSITY_LEVEL_LOG_SKIPPED_TESTS)
|
||||
continue;
|
||||
|
||||
outln("{}: {}", test_result_to_string(non_passing_test.result), non_passing_test.test.relative_path);
|
||||
}
|
||||
|
||||
if (app.verbosity >= Application::VERBOSITY_LEVEL_LOG_SLOWEST_TESTS) {
|
||||
auto tests_to_print = min(10uz, tests.size());
|
||||
outln("\nSlowest {} tests:", tests_to_print);
|
||||
|
||||
quick_sort(tests, [&](auto const& lhs, auto const& rhs) {
|
||||
auto lhs_duration = lhs.end_time - lhs.start_time;
|
||||
auto rhs_duration = rhs.end_time - rhs.start_time;
|
||||
return lhs_duration > rhs_duration;
|
||||
});
|
||||
|
||||
for (auto const& test : tests.span().trim(tests_to_print)) {
|
||||
auto duration = test.end_time - test.start_time;
|
||||
|
||||
outln("{}: {}ms", test.relative_path, duration.to_milliseconds());
|
||||
}
|
||||
}
|
||||
|
||||
if (app.dump_gc_graph) {
|
||||
for (auto& view : views) {
|
||||
if (auto path = view->dump_gc_graph(); path.is_error())
|
||||
warnln("Failed to dump GC graph: {}", path.error());
|
||||
else
|
||||
outln("GC graph dumped to {}", path.value());
|
||||
}
|
||||
}
|
||||
|
||||
return fail_count + timeout_count + crashed_count;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2024, Tim Flynn <trflynn89@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Assertions.h>
|
||||
#include <AK/ByteString.h>
|
||||
#include <AK/Error.h>
|
||||
#include <AK/RefPtr.h>
|
||||
#include <AK/String.h>
|
||||
#include <AK/StringView.h>
|
||||
#include <AK/Time.h>
|
||||
#include <LibCore/Forward.h>
|
||||
#include <LibCore/Promise.h>
|
||||
#include <LibGfx/Forward.h>
|
||||
#include <LibURL/Forward.h>
|
||||
#include <LibWeb/PixelUnits.h>
|
||||
|
||||
namespace Ladybird {
|
||||
|
||||
class HeadlessWebView;
|
||||
|
||||
enum class TestMode {
|
||||
Layout,
|
||||
Text,
|
||||
Ref,
|
||||
Crash,
|
||||
};
|
||||
|
||||
enum class TestResult {
|
||||
Pass,
|
||||
Fail,
|
||||
Skipped,
|
||||
Timeout,
|
||||
Crashed,
|
||||
};
|
||||
|
||||
enum class RefTestExpectationType {
|
||||
Match,
|
||||
Mismatch,
|
||||
};
|
||||
|
||||
static constexpr StringView test_result_to_string(TestResult result)
|
||||
{
|
||||
switch (result) {
|
||||
case TestResult::Pass:
|
||||
return "Pass"sv;
|
||||
case TestResult::Fail:
|
||||
return "Fail"sv;
|
||||
case TestResult::Skipped:
|
||||
return "Skipped"sv;
|
||||
case TestResult::Timeout:
|
||||
return "Timeout"sv;
|
||||
case TestResult::Crashed:
|
||||
return "Crashed"sv;
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
||||
struct Test {
|
||||
TestMode mode;
|
||||
|
||||
ByteString input_path {};
|
||||
ByteString expectation_path {};
|
||||
ByteString relative_path {};
|
||||
|
||||
UnixDateTime start_time {};
|
||||
UnixDateTime end_time {};
|
||||
size_t index { 0 };
|
||||
|
||||
String text {};
|
||||
bool did_finish_test { false };
|
||||
bool did_finish_loading { false };
|
||||
|
||||
Optional<RefTestExpectationType> ref_test_expectation_type {};
|
||||
|
||||
RefPtr<Gfx::Bitmap const> actual_screenshot {};
|
||||
RefPtr<Gfx::Bitmap const> expectation_screenshot {};
|
||||
};
|
||||
|
||||
struct TestCompletion {
|
||||
Test& test;
|
||||
TestResult result;
|
||||
};
|
||||
|
||||
using TestPromise = Core::Promise<TestCompletion>;
|
||||
|
||||
ErrorOr<int> run_tests(Core::AnonymousBuffer const& theme, Web::DevicePixelSize window_size);
|
||||
void run_dump_test(HeadlessWebView&, Test&, URL::URL const&, int timeout_in_milliseconds);
|
||||
|
||||
}
|
||||
|
|
@ -1,40 +0,0 @@
|
|||
/*
|
||||
* Copyright (c) 2022, Dex♪ <dexes.ttp@gmail.com>
|
||||
* Copyright (c) 2023-2025, Tim Flynn <trflynn89@ladybird.org>
|
||||
* Copyright (c) 2023, Andreas Kling <andreas@ladybird.org>
|
||||
* Copyright (c) 2023-2024, Sam Atkins <sam@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <AK/LexicalPath.h>
|
||||
#include <AK/String.h>
|
||||
#include <LibCore/ResourceImplementationFile.h>
|
||||
#include <LibFileSystem/FileSystem.h>
|
||||
#include <LibGfx/SystemTheme.h>
|
||||
#include <LibWebView/Utilities.h>
|
||||
#include <UI/Headless/Application.h>
|
||||
#include <UI/Headless/Test.h>
|
||||
|
||||
ErrorOr<int> serenity_main(Main::Arguments arguments)
|
||||
{
|
||||
WebView::platform_init();
|
||||
|
||||
auto app = Ladybird::Application::create(arguments);
|
||||
TRY(app->launch_services());
|
||||
|
||||
Core::ResourceImplementation::install(make<Core::ResourceImplementationFile>(MUST(String::from_byte_string(app->resources_folder))));
|
||||
|
||||
auto theme_path = LexicalPath::join(app->resources_folder, "themes"sv, "Default.ini"sv);
|
||||
auto theme = TRY(Gfx::load_system_theme(theme_path.string()));
|
||||
|
||||
auto const& browser_options = Ladybird::Application::browser_options();
|
||||
Web::DevicePixelSize window_size { browser_options.window_width, browser_options.window_height };
|
||||
|
||||
VERIFY(!app->test_root_path.is_empty());
|
||||
|
||||
app->test_root_path = LexicalPath::absolute_path(TRY(FileSystem::current_working_directory()), app->test_root_path);
|
||||
TRY(app->launch_test_fixtures());
|
||||
|
||||
return Ladybird::run_tests(theme, window_size);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue