mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2026-06-18 07:43:37 +00:00
f4960d9d7d
Keep decoded CSS text separate from tokenizer byte input. CSSOM and already-decoded stylesheet text preserve code point preprocessing, so a lone surrogate maps to one replacement character instead of being re-decoded as malformed UTF-8 bytes. Decode tokenizer byte input with the requested encoding unless that encoding is UTF-8 and the byte stream is strictly valid UTF-8. Keep the fast path by constructing the decoded string without validating twice after strict validation succeeds. Preserve UTF-8 decoder behavior on the byte fast path by stripping an initial UTF-8 BOM and rejecting encoded surrogate bytes. Invalid UTF-8 still goes through the decoder. Add tokenizer coverage for both the C++ and Rust backends across decoded text, UTF-8 aliases, BOM-prefixed input, invalid UTF-8, and non-UTF requested encodings.
113 lines
2.9 KiB
C++
113 lines
2.9 KiB
C++
/*
|
|
* Copyright (c) 2020-2021, the SerenityOS developers.
|
|
* Copyright (c) 2021-2023, Sam Atkins <atkinssj@serenityos.org>
|
|
*
|
|
* SPDX-License-Identifier: BSD-2-Clause
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include <AK/Optional.h>
|
|
#include <AK/StringView.h>
|
|
#include <AK/Types.h>
|
|
#include <AK/Utf8View.h>
|
|
#include <LibWeb/CSS/Parser/Token.h>
|
|
#include <LibWeb/Export.h>
|
|
#include <LibWeb/Forward.h>
|
|
|
|
namespace Web::CSS::Parser {
|
|
|
|
enum class TokenizerInput {
|
|
DecodedText,
|
|
EncodedBytes,
|
|
};
|
|
|
|
class U32Twin {
|
|
public:
|
|
void set(size_t index, u32 value)
|
|
{
|
|
if (index == 0)
|
|
first = value;
|
|
if (index == 1)
|
|
second = value;
|
|
}
|
|
|
|
u32 first {};
|
|
u32 second {};
|
|
};
|
|
|
|
class U32Triplet {
|
|
public:
|
|
void set(size_t index, u32 value)
|
|
{
|
|
if (index == 0)
|
|
first = value;
|
|
if (index == 1)
|
|
second = value;
|
|
if (index == 2)
|
|
third = value;
|
|
}
|
|
|
|
U32Twin to_twin_12()
|
|
{
|
|
return { first, second };
|
|
}
|
|
|
|
U32Twin to_twin_23()
|
|
{
|
|
return { second, third };
|
|
}
|
|
|
|
u32 first {};
|
|
u32 second {};
|
|
u32 third {};
|
|
};
|
|
|
|
class WEB_API Tokenizer {
|
|
public:
|
|
static Vector<Token> tokenize(StringView input, StringView encoding, TokenizerInput = TokenizerInput::DecodedText);
|
|
|
|
[[nodiscard]] static Token create_eof_token();
|
|
|
|
private:
|
|
explicit Tokenizer(String decoded_input);
|
|
|
|
[[nodiscard]] Vector<Token> tokenize();
|
|
|
|
size_t current_byte_offset() const;
|
|
String input_since(size_t offset) const;
|
|
|
|
[[nodiscard]] u32 next_code_point();
|
|
[[nodiscard]] u32 peek_code_point(size_t offset = 0) const;
|
|
[[nodiscard]] U32Twin peek_twin() const;
|
|
[[nodiscard]] U32Triplet peek_triplet() const;
|
|
|
|
[[nodiscard]] U32Twin start_of_input_stream_twin();
|
|
[[nodiscard]] U32Triplet start_of_input_stream_triplet();
|
|
|
|
[[nodiscard]] Token consume_a_token();
|
|
[[nodiscard]] Token consume_string_token(u32 ending_code_point);
|
|
[[nodiscard]] Token consume_a_numeric_token();
|
|
[[nodiscard]] Token consume_an_ident_like_token();
|
|
[[nodiscard]] Number consume_a_number();
|
|
[[nodiscard]] double convert_a_string_to_a_number(StringView);
|
|
[[nodiscard]] FlyString consume_an_ident_sequence();
|
|
[[nodiscard]] u32 consume_escaped_code_point();
|
|
[[nodiscard]] Token consume_a_url_token(size_t start_byte_offset);
|
|
void consume_the_remnants_of_a_bad_url();
|
|
void consume_comments();
|
|
void consume_as_much_whitespace_as_possible();
|
|
void reconsume_current_input_code_point();
|
|
[[nodiscard]] static bool is_valid_escape_sequence(U32Twin);
|
|
[[nodiscard]] static bool would_start_an_ident_sequence(U32Triplet);
|
|
[[nodiscard]] static bool would_start_a_number(U32Triplet);
|
|
|
|
String m_decoded_input;
|
|
Utf8View m_utf8_view;
|
|
AK::Utf8CodePointIterator m_utf8_iterator;
|
|
AK::Utf8CodePointIterator m_prev_utf8_iterator;
|
|
Token::Position m_position;
|
|
Token::Position m_prev_position;
|
|
};
|
|
|
|
}
|