Stowage/godot
2
0
Fork 0
mirror of https://github.com/godotengine/godot.git synced 2025-10-19 16:03:29 +00:00
Code Issues Projects Releases Packages Wiki Activity

[Complex Test Layouts] Change String to use UTF-32 encoding on all platforms.

Browse source
This commit is contained in:
bruvzg 2020-07-27 13:43:20 +03:00
parent 0864f12f0d
commit 80b8eff6aa
No known key found for this signature in database
GPG key ID: FCED35F1CECE0D3A
94 changed files with 4889 additions and 1686 deletions
Download patch file Download diff file Expand all files Collapse all files

46
modules/gdscript/gdscript_tokenizer.cpp
View file

@ -222,7 +222,7 @@ String GDScriptTokenizer::get_token_name(Token::Type p_token_type) {
void GDScriptTokenizer::set_source_code(const String &p_source_code) {
source = p_source_code;
if (source.empty()) {
_source = L"";
_source = U"";
} else {
_source = source.ptr();
}
@ -263,7 +263,7 @@ bool GDScriptTokenizer::is_past_cursor() const {
return true;
}
CharType GDScriptTokenizer::_advance() {
char32_t GDScriptTokenizer::_advance() {
if (unlikely(_is_at_end())) {
return '\0';
}
@ -282,15 +282,15 @@ CharType GDScriptTokenizer::_advance() {
return _peek(-1);
}
void GDScriptTokenizer::push_paren(CharType p_char) {
void GDScriptTokenizer::push_paren(char32_t p_char) {
paren_stack.push_back(p_char);
}
bool GDScriptTokenizer::pop_paren(CharType p_expected) {
bool GDScriptTokenizer::pop_paren(char32_t p_expected) {
if (paren_stack.empty()) {
return false;
}
CharType actual = paren_stack.back()->get();
char32_t actual = paren_stack.back()->get();
paren_stack.pop_back();
return actual == p_expected;
@ -302,19 +302,19 @@ GDScriptTokenizer::Token GDScriptTokenizer::pop_error() {
return error;
}
static bool _is_alphanumeric(CharType c) {
static bool _is_alphanumeric(char32_t c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}
static bool _is_digit(CharType c) {
static bool _is_digit(char32_t c) {
return (c >= '0' && c <= '9');
}
static bool _is_hex_digit(CharType c) {
static bool _is_hex_digit(char32_t c) {
return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
}
static bool _is_binary_digit(CharType c) {
static bool _is_binary_digit(char32_t c) {
return (c == '0' || c == '1');
}
@ -404,7 +404,7 @@ void GDScriptTokenizer::push_error(const Token &p_error) {
error_stack.push_back(p_error);
}
GDScriptTokenizer::Token GDScriptTokenizer::make_paren_error(CharType p_paren) {
GDScriptTokenizer::Token GDScriptTokenizer::make_paren_error(char32_t p_paren) {
if (paren_stack.empty()) {
return make_error(vformat("Closing \"%c\" doesn't have an opening counterpart.", p_paren));
}
@ -413,8 +413,8 @@ GDScriptTokenizer::Token GDScriptTokenizer::make_paren_error(CharType p_paren) {
return error;
}
GDScriptTokenizer::Token GDScriptTokenizer::check_vcs_marker(CharType p_test, Token::Type p_double_type) {
const CharType *next = _current + 1;
GDScriptTokenizer::Token GDScriptTokenizer::check_vcs_marker(char32_t p_test, Token::Type p_double_type) {
const char32_t *next = _current + 1;
int chars = 2; // Two already matched.
// Test before consuming characters, since we don't want to consume more than needed.
@ -602,7 +602,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::number() {
bool has_decimal = false;
bool has_exponent = false;
bool has_error = false;
bool (*digit_check_func)(CharType) = _is_digit;
bool (*digit_check_func)(char32_t) = _is_digit;
if (_peek(-1) == '.') {
has_decimal = true;
@ -762,7 +762,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
_advance();
}
CharType quote_char = _peek(-1);
char32_t quote_char = _peek(-1);
if (_peek() == quote_char && _peek(1) == quote_char) {
is_multiline = true;
@ -779,7 +779,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
return make_error("Unterminated string.");
}
CharType ch = _peek();
char32_t ch = _peek();
if (ch == '\\') {
// Escape pattern.
@ -789,13 +789,13 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
}
// Grab escape character.
CharType code = _peek();
char32_t code = _peek();
_advance();
if (_is_at_end()) {
return make_error("Unterminated string.");
}
CharType escaped = 0;
char32_t escaped = 0;
bool valid_escape = true;
switch (code) {
@ -836,8 +836,8 @@ GDScriptTokenizer::Token GDScriptTokenizer::string() {
return make_error("Unterminated string.");
}
CharType digit = _peek();
CharType value = 0;
char32_t digit = _peek();
char32_t value = 0;
if (digit >= '0' && digit <= '9') {
value = digit - '0';
} else if (digit >= 'a' && digit <= 'f') {
@ -940,7 +940,7 @@ void GDScriptTokenizer::check_indent() {
}
for (;;) {
CharType current_indent_char = _peek();
char32_t current_indent_char = _peek();
int indent_count = 0;
if (current_indent_char != ' ' && current_indent_char != '\t' && current_indent_char != '\r' && current_indent_char != '\n' && current_indent_char != '#') {
@ -970,7 +970,7 @@ void GDScriptTokenizer::check_indent() {
// Check indent level.
bool mixed = false;
while (!_is_at_end()) {
CharType space = _peek();
char32_t space = _peek();
if (space == '\t') {
// Consider individual tab columns.
column += tab_size - 1;
@ -1103,7 +1103,7 @@ void GDScriptTokenizer::_skip_whitespace() {
}
for (;;) {
CharType c = _peek();
char32_t c = _peek();
switch (c) {
case ' ':
_advance();
@ -1192,7 +1192,7 @@ GDScriptTokenizer::Token GDScriptTokenizer::scan() {
return make_token(Token::TK_EOF);
}
const CharType c = _advance();
const char32_t c = _advance();
if (c == '\\') {
// Line continuation with backslash.