| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | /*
 | 
					
						
							|  |  |  |  * Copyright (c) 2021, Tim Flynn <trflynn89@pm.me> | 
					
						
							|  |  |  |  * | 
					
						
							|  |  |  |  * SPDX-License-Identifier: BSD-2-Clause | 
					
						
							|  |  |  |  */ | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | #include <AK/AllOf.h>
 | 
					
						
							|  |  |  | #include <AK/CharacterTypes.h>
 | 
					
						
							|  |  |  | #include <AK/GenericLexer.h>
 | 
					
						
							|  |  |  | #include <AK/QuickSort.h>
 | 
					
						
							|  |  |  | #include <AK/StringBuilder.h>
 | 
					
						
							|  |  |  | #include <LibUnicode/Locale.h>
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-24 22:17:08 -04:00
										 |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  | #    include <LibUnicode/UnicodeLocale.h>
 | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | namespace Unicode { | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | bool is_unicode_language_subtag(StringView subtag) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // unicode_language_subtag = alpha{2,3} | alpha{5,8}
 | 
					
						
							|  |  |  |     if ((subtag.length() < 2) || (subtag.length() == 4) || (subtag.length() > 8)) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return all_of(subtag, is_ascii_alpha); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | bool is_unicode_script_subtag(StringView subtag) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // unicode_script_subtag = alpha{4}
 | 
					
						
							|  |  |  |     if (subtag.length() != 4) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return all_of(subtag, is_ascii_alpha); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | bool is_unicode_region_subtag(StringView subtag) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // unicode_region_subtag = (alpha{2} | digit{3})
 | 
					
						
							|  |  |  |     if (subtag.length() == 2) | 
					
						
							|  |  |  |         return all_of(subtag, is_ascii_alpha); | 
					
						
							|  |  |  |     if (subtag.length() == 3) | 
					
						
							|  |  |  |         return all_of(subtag, is_ascii_digit); | 
					
						
							|  |  |  |     return false; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | bool is_unicode_variant_subtag(StringView subtag) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3})
 | 
					
						
							|  |  |  |     if ((subtag.length() >= 5) && (subtag.length() <= 8)) | 
					
						
							|  |  |  |         return all_of(subtag, is_ascii_alphanumeric); | 
					
						
							|  |  |  |     if (subtag.length() == 4) | 
					
						
							|  |  |  |         return is_ascii_digit(subtag[0]) && all_of(subtag.substring_view(1), is_ascii_alphanumeric); | 
					
						
							|  |  |  |     return false; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  | static bool is_key(StringView key) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // key = alphanum alpha
 | 
					
						
							|  |  |  |     if (key.length() != 2) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return is_ascii_alphanumeric(key[0]) && is_ascii_alpha(key[1]); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | static bool is_single_type(StringView type) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // type = alphanum{3,8} (sep alphanum{3,8})*
 | 
					
						
							|  |  |  |     // Note: Consecutive types are not handled here, that is left to the caller.
 | 
					
						
							|  |  |  |     if ((type.length() < 3) || (type.length() > 8)) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return all_of(type, is_ascii_alphanumeric); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | static bool is_attribute(StringView type) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // attribute = alphanum{3,8}
 | 
					
						
							|  |  |  |     if ((type.length() < 3) || (type.length() > 8)) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return all_of(type, is_ascii_alphanumeric); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 17:11:48 -04:00
										 |  |  | static bool is_transformed_key(StringView key) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // tkey = alpha digit
 | 
					
						
							|  |  |  |     if (key.length() != 2) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return is_ascii_alpha(key[0]) && is_ascii_digit(key[1]); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | static bool is_single_transformed_value(StringView value) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // tvalue = (sep alphanum{3,8})+
 | 
					
						
							|  |  |  |     // Note: Consecutive values are not handled here, that is left to the caller.
 | 
					
						
							|  |  |  |     if ((value.length() < 3) || (value.length() > 8)) | 
					
						
							|  |  |  |         return false; | 
					
						
							|  |  |  |     return all_of(value, is_ascii_alphanumeric); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  | static Optional<StringView> consume_next_segment(GenericLexer& lexer, bool with_separator = true) | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  | { | 
					
						
							|  |  |  |     constexpr auto is_separator = is_any_of("-_"sv); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (with_separator) { | 
					
						
							|  |  |  |         if (!lexer.next_is(is_separator)) | 
					
						
							|  |  |  |             return {}; | 
					
						
							|  |  |  |         lexer.ignore(); | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto segment = lexer.consume_until(is_separator); | 
					
						
							|  |  |  |     if (segment.is_empty()) { | 
					
						
							|  |  |  |         lexer.retreat(with_separator); | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     return segment; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | static Optional<LanguageID> parse_unicode_language_id(GenericLexer& lexer) | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#Unicode_language_identifier
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // unicode_language_id = "root"
 | 
					
						
							|  |  |  |     //     OR
 | 
					
						
							|  |  |  |     // unicode_language_id = ((unicode_language_subtag (sep unicode_script_subtag)?) | unicode_script_subtag)
 | 
					
						
							|  |  |  |     //                       (sep unicode_region_subtag)?
 | 
					
						
							|  |  |  |     //                       (sep unicode_variant_subtag)*
 | 
					
						
							|  |  |  |     LanguageID language_id {}; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     if (lexer.consume_specific("root"sv)) { | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  |         language_id.is_root = true; | 
					
						
							|  |  |  |         return language_id; | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     enum class ParseState { | 
					
						
							|  |  |  |         ParsingLanguageOrScript, | 
					
						
							|  |  |  |         ParsingScript, | 
					
						
							|  |  |  |         ParsingRegion, | 
					
						
							|  |  |  |         ParsingVariant, | 
					
						
							|  |  |  |         Done, | 
					
						
							|  |  |  |     }; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     auto state = ParseState::ParsingLanguageOrScript; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     while (!lexer.is_eof() && (state != ParseState::Done)) { | 
					
						
							|  |  |  |         auto segment = consume_next_segment(lexer, state != ParseState::ParsingLanguageOrScript); | 
					
						
							|  |  |  |         if (!segment.has_value()) | 
					
						
							|  |  |  |             return {}; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |         switch (state) { | 
					
						
							|  |  |  |         case ParseState::ParsingLanguageOrScript: | 
					
						
							|  |  |  |             if (is_unicode_language_subtag(*segment)) { | 
					
						
							|  |  |  |                 state = ParseState::ParsingScript; | 
					
						
							|  |  |  |                 language_id.language = *segment; | 
					
						
							|  |  |  |             } else if (is_unicode_script_subtag(*segment)) { | 
					
						
							|  |  |  |                 state = ParseState::ParsingRegion; | 
					
						
							|  |  |  |                 language_id.script = *segment; | 
					
						
							|  |  |  |             } else { | 
					
						
							|  |  |  |                 return {}; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         case ParseState::ParsingScript: | 
					
						
							|  |  |  |             if (is_unicode_script_subtag(*segment)) { | 
					
						
							|  |  |  |                 state = ParseState::ParsingRegion; | 
					
						
							|  |  |  |                 language_id.script = *segment; | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             state = ParseState::ParsingRegion; | 
					
						
							|  |  |  |             [[fallthrough]]; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         case ParseState::ParsingRegion: | 
					
						
							|  |  |  |             if (is_unicode_region_subtag(*segment)) { | 
					
						
							|  |  |  |                 state = ParseState::ParsingVariant; | 
					
						
							|  |  |  |                 language_id.region = *segment; | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             state = ParseState::ParsingVariant; | 
					
						
							|  |  |  |             [[fallthrough]]; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         case ParseState::ParsingVariant: | 
					
						
							|  |  |  |             if (is_unicode_variant_subtag(*segment)) { | 
					
						
							|  |  |  |                 language_id.variants.append(*segment); | 
					
						
							|  |  |  |             } else { | 
					
						
							|  |  |  |                 lexer.retreat(segment->length() + 1); | 
					
						
							|  |  |  |                 state = ParseState::Done; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         default: | 
					
						
							|  |  |  |             VERIFY_NOT_REACHED(); | 
					
						
							|  |  |  |         } | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     return language_id; | 
					
						
							|  |  |  | } | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  | static Optional<LocaleExtension> parse_unicode_locale_extension(GenericLexer& lexer) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#unicode_locale_extensions
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // unicode_locale_extensions = sep [uU] ((sep keyword)+ | (sep attribute)+ (sep keyword)*)
 | 
					
						
							|  |  |  |     LocaleExtension locale_extension {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     enum class ParseState { | 
					
						
							|  |  |  |         ParsingAttributeOrKeyword, | 
					
						
							|  |  |  |         ParsingAttribute, | 
					
						
							|  |  |  |         ParsingKeyword, | 
					
						
							|  |  |  |         Done, | 
					
						
							|  |  |  |     }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto state = ParseState::ParsingAttributeOrKeyword; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     while (!lexer.is_eof() && (state != ParseState::Done)) { | 
					
						
							|  |  |  |         auto segment = consume_next_segment(lexer); | 
					
						
							|  |  |  |         if (!segment.has_value()) | 
					
						
							|  |  |  |             return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         if (state == ParseState::ParsingAttributeOrKeyword) | 
					
						
							|  |  |  |             state = is_key(*segment) ? ParseState::ParsingKeyword : ParseState::ParsingAttribute; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         switch (state) { | 
					
						
							|  |  |  |         case ParseState::ParsingAttribute: | 
					
						
							|  |  |  |             if (is_attribute(*segment)) { | 
					
						
							|  |  |  |                 locale_extension.attributes.append(*segment); | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             state = ParseState::ParsingKeyword; | 
					
						
							|  |  |  |             [[fallthrough]]; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         case ParseState::ParsingKeyword: { | 
					
						
							|  |  |  |             // keyword = key (sep type)?
 | 
					
						
							|  |  |  |             Keyword keyword { .key = *segment }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             if (!is_key(*segment)) { | 
					
						
							|  |  |  |                 lexer.retreat(segment->length() + 1); | 
					
						
							|  |  |  |                 state = ParseState::Done; | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             while (true) { | 
					
						
							|  |  |  |                 auto type = consume_next_segment(lexer); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 if (!type.has_value() || !is_single_type(*type)) { | 
					
						
							|  |  |  |                     if (type.has_value()) | 
					
						
							|  |  |  |                         lexer.retreat(type->length() + 1); | 
					
						
							|  |  |  |                     break; | 
					
						
							|  |  |  |                 } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 keyword.types.append(*type); | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             locale_extension.keywords.append(move(keyword)); | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         default: | 
					
						
							|  |  |  |             VERIFY_NOT_REACHED(); | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (locale_extension.attributes.is_empty() && locale_extension.keywords.is_empty()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  |     return locale_extension; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 17:11:48 -04:00
										 |  |  | static Optional<TransformedExtension> parse_transformed_extension(GenericLexer& lexer) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#transformed_extensions
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // transformed_extensions = sep [tT] ((sep tlang (sep tfield)*) | (sep tfield)+)
 | 
					
						
							|  |  |  |     TransformedExtension transformed_extension {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     enum class ParseState { | 
					
						
							|  |  |  |         ParsingLanguageOrField, | 
					
						
							|  |  |  |         ParsingLanguage, | 
					
						
							|  |  |  |         ParsingField, | 
					
						
							|  |  |  |         Done, | 
					
						
							|  |  |  |     }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto state = ParseState::ParsingLanguageOrField; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     while (!lexer.is_eof() && (state != ParseState::Done)) { | 
					
						
							|  |  |  |         auto segment = consume_next_segment(lexer); | 
					
						
							|  |  |  |         if (!segment.has_value()) | 
					
						
							|  |  |  |             return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         if (state == ParseState::ParsingLanguageOrField) | 
					
						
							|  |  |  |             state = is_unicode_language_subtag(*segment) ? ParseState::ParsingLanguage : ParseState::ParsingField; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         switch (state) { | 
					
						
							|  |  |  |         case ParseState::ParsingLanguage: | 
					
						
							|  |  |  |             lexer.retreat(segment->length()); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             if (auto language_id = parse_unicode_language_id(lexer); language_id.has_value()) { | 
					
						
							|  |  |  |                 transformed_extension.language = language_id.release_value(); | 
					
						
							|  |  |  |                 state = ParseState::ParsingField; | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         case ParseState::ParsingField: { | 
					
						
							|  |  |  |             // tfield = tkey tvalue;
 | 
					
						
							|  |  |  |             TransformedField field { .key = *segment }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             if (!is_transformed_key(*segment)) { | 
					
						
							|  |  |  |                 lexer.retreat(segment->length() + 1); | 
					
						
							|  |  |  |                 state = ParseState::Done; | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             while (true) { | 
					
						
							|  |  |  |                 auto value = consume_next_segment(lexer); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 if (!value.has_value() || !is_single_transformed_value(*value)) { | 
					
						
							|  |  |  |                     if (value.has_value()) | 
					
						
							|  |  |  |                         lexer.retreat(value->length() + 1); | 
					
						
							|  |  |  |                     break; | 
					
						
							|  |  |  |                 } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 field.values.append(*value); | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             if (field.values.is_empty()) | 
					
						
							|  |  |  |                 return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             transformed_extension.fields.append(move(field)); | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         default: | 
					
						
							|  |  |  |             VERIFY_NOT_REACHED(); | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (!transformed_extension.language.has_value() && transformed_extension.fields.is_empty()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  |     return transformed_extension; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 17:24:20 -04:00
										 |  |  | static Optional<OtherExtension> parse_other_extension(char key, GenericLexer& lexer) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#other_extensions
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ;
 | 
					
						
							|  |  |  |     OtherExtension other_extension { .key = key }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (!is_ascii_alphanumeric(key) || (key == 'x') || (key == 'X')) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     while (true) { | 
					
						
							|  |  |  |         auto segment = consume_next_segment(lexer); | 
					
						
							|  |  |  |         if (!segment.has_value()) | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         if ((segment->length() < 2) || (segment->length() > 8) || !all_of(*segment, is_ascii_alphanumeric)) { | 
					
						
							|  |  |  |             lexer.retreat(segment->length() + 1); | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         other_extension.values.append(*segment); | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (other_extension.values.is_empty()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  |     return other_extension; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  | static Optional<Extension> parse_extension(GenericLexer& lexer) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#extensions
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // extensions = unicode_locale_extensions | transformed_extensions | other_extensions
 | 
					
						
							|  |  |  |     size_t starting_position = lexer.tell(); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (auto header = consume_next_segment(lexer); header.has_value() && (header->length() == 1)) { | 
					
						
							| 
									
										
										
										
											2021-08-27 17:24:20 -04:00
										 |  |  |         switch (char key = (*header)[0]) { | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  |         case 'u': | 
					
						
							|  |  |  |         case 'U': | 
					
						
							|  |  |  |             if (auto extension = parse_unicode_locale_extension(lexer); extension.has_value()) | 
					
						
							|  |  |  |                 return Extension { extension.release_value() }; | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 17:11:48 -04:00
										 |  |  |         case 't': | 
					
						
							|  |  |  |         case 'T': | 
					
						
							|  |  |  |             if (auto extension = parse_transformed_extension(lexer); extension.has_value()) | 
					
						
							|  |  |  |                 return Extension { extension.release_value() }; | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  |         default: | 
					
						
							| 
									
										
										
										
											2021-08-27 17:24:20 -04:00
										 |  |  |             if (auto extension = parse_other_extension(key, lexer); extension.has_value()) | 
					
						
							|  |  |  |                 return Extension { extension.release_value() }; | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  |             break; | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     lexer.retreat(lexer.tell() - starting_position); | 
					
						
							|  |  |  |     return {}; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 14:31:48 -04:00
										 |  |  | static Vector<String> parse_private_use_extensions(GenericLexer& lexer) | 
					
						
							| 
									
										
										
										
											2021-08-27 17:51:17 -04:00
										 |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#pu_extensions
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // pu_extensions = = sep [xX] (sep alphanum{1,8})+ ;
 | 
					
						
							|  |  |  |     size_t starting_position = lexer.tell(); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto header = consume_next_segment(lexer); | 
					
						
							|  |  |  |     if (!header.has_value()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 14:31:48 -04:00
										 |  |  |     auto parse_values = [&]() -> Vector<String> { | 
					
						
							|  |  |  |         Vector<String> extensions; | 
					
						
							| 
									
										
										
										
											2021-08-27 17:51:17 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |         while (true) { | 
					
						
							|  |  |  |             auto segment = consume_next_segment(lexer); | 
					
						
							|  |  |  |             if (!segment.has_value()) | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             if ((segment->length() < 1) || (segment->length() > 8) || !all_of(*segment, is_ascii_alphanumeric)) { | 
					
						
							|  |  |  |                 lexer.retreat(segment->length() + 1); | 
					
						
							|  |  |  |                 break; | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             extensions.append(*segment); | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         return extensions; | 
					
						
							|  |  |  |     }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if ((header->length() == 1) && (((*header)[0] == 'x') || ((*header)[0] == 'X'))) { | 
					
						
							|  |  |  |         if (auto extensions = parse_values(); !extensions.is_empty()) | 
					
						
							|  |  |  |             return extensions; | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     lexer.retreat(lexer.tell() - starting_position); | 
					
						
							|  |  |  |     return {}; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  | Optional<LanguageID> parse_unicode_language_id(StringView language) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     GenericLexer lexer { language }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto language_id = parse_unicode_language_id(lexer); | 
					
						
							|  |  |  |     if (!lexer.is_eof()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |     return language_id; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Optional<LocaleID> parse_unicode_locale_id(StringView locale) | 
					
						
							|  |  |  | { | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     GenericLexer lexer { locale }; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#Unicode_locale_identifier
 | 
					
						
							|  |  |  |     //
 | 
					
						
							|  |  |  |     // unicode_locale_id = unicode_language_id
 | 
					
						
							|  |  |  |     //                     extensions*
 | 
					
						
							|  |  |  |     //                     pu_extensions?
 | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  |     auto language_id = parse_unicode_language_id(lexer); | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  |     if (!language_id.has_value()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  |     LocaleID locale_id { language_id.release_value() }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     while (true) { | 
					
						
							|  |  |  |         auto extension = parse_extension(lexer); | 
					
						
							|  |  |  |         if (!extension.has_value()) | 
					
						
							|  |  |  |             break; | 
					
						
							|  |  |  |         locale_id.extensions.append(extension.release_value()); | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 17:51:17 -04:00
										 |  |  |     locale_id.private_use_extensions = parse_private_use_extensions(lexer); | 
					
						
							| 
									
										
										
										
											2021-08-27 10:12:17 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |     if (!lexer.is_eof()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-27 16:38:06 -04:00
										 |  |  |     return locale_id; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:51:08 -04:00
										 |  |  | static void perform_hard_coded_key_value_substitutions(String& key, String& value) | 
					
						
							|  |  |  | { | 
					
						
							| 
									
										
										
										
											2021-08-30 18:04:15 -04:00
										 |  |  |     // FIXME: In the XML export of CLDR, there are some aliases defined in the following files:
 | 
					
						
							| 
									
										
										
										
											2021-08-31 12:19:19 -04:00
										 |  |  |     // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/calendar.xml
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:51:08 -04:00
										 |  |  |     // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/collation.xml
 | 
					
						
							| 
									
										
										
										
											2021-08-31 11:12:03 -04:00
										 |  |  |     // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/measure.xml
 | 
					
						
							| 
									
										
										
										
											2021-08-31 12:00:07 -04:00
										 |  |  |     // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/timezone.xml
 | 
					
						
							| 
									
										
										
										
											2021-08-30 18:04:15 -04:00
										 |  |  |     // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/transform.xml
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:51:08 -04:00
										 |  |  |     //
 | 
					
						
							|  |  |  |     // There doesn't seem to be a counterpart in the JSON export. Since there aren't many such
 | 
					
						
							|  |  |  |     // aliases, until an XML parser is implemented, those aliases are implemented here.
 | 
					
						
							| 
									
										
										
										
											2021-08-31 12:19:19 -04:00
										 |  |  |     if ((key == "ca"sv) && (value == "islamicc"sv)) { | 
					
						
							|  |  |  |         value = "islamic-civil"sv; | 
					
						
							|  |  |  |     } else if (key.is_one_of("kb"sv, "kc"sv, "kh"sv, "kk"sv, "kn"sv) && (value == "yes"sv)) { | 
					
						
							| 
									
										
										
										
											2021-08-30 15:51:08 -04:00
										 |  |  |         value = "true"sv; | 
					
						
							| 
									
										
										
										
											2021-08-31 11:00:52 -04:00
										 |  |  |     } else if (key == "ks"sv) { | 
					
						
							|  |  |  |         if (value == "primary"sv) | 
					
						
							|  |  |  |             value = "level1"sv; | 
					
						
							|  |  |  |         else if (value == "tertiary"sv) | 
					
						
							|  |  |  |             value = "level3"sv; | 
					
						
							|  |  |  |         // Note: There are also aliases for "secondary", "quaternary", "quarternary", and "identical",
 | 
					
						
							|  |  |  |         // but those are semantically incorrect values (they are too long), so they can be skipped.
 | 
					
						
							|  |  |  |     } else if ((key == "m0"sv) && (value == "names"sv)) { | 
					
						
							| 
									
										
										
										
											2021-08-30 18:04:15 -04:00
										 |  |  |         value = "prprname"sv; | 
					
						
							| 
									
										
										
										
											2021-08-31 11:12:03 -04:00
										 |  |  |     } else if ((key == "ms"sv) && (value == "imperial"sv)) { | 
					
						
							|  |  |  |         value = "uksystem"sv; | 
					
						
							| 
									
										
										
										
											2021-08-31 12:00:07 -04:00
										 |  |  |     } else if (key == "tz"sv) { | 
					
						
							|  |  |  |         // Formatter disabled because this block is easier to read / check against timezone.xml as one-liners.
 | 
					
						
							|  |  |  |         // clang-format off
 | 
					
						
							|  |  |  |         if (value == "aqams"sv) value = "nzakl"sv; | 
					
						
							|  |  |  |         else if (value == "cnckg"sv) value = "cnsha"sv; | 
					
						
							|  |  |  |         else if (value == "cnhrb"sv) value = "cnsha"sv; | 
					
						
							|  |  |  |         else if (value == "cnkhg"sv) value = "cnurc"sv; | 
					
						
							|  |  |  |         else if (value == "cuba"sv) value = "cuhav"sv; | 
					
						
							|  |  |  |         else if (value == "egypt"sv) value = "egcai"sv; | 
					
						
							|  |  |  |         else if (value == "eire"sv) value = "iedub"sv; | 
					
						
							|  |  |  |         else if (value == "est"sv) value = "utcw05"sv; | 
					
						
							|  |  |  |         else if (value == "gmt0"sv) value = "gmt"sv; | 
					
						
							|  |  |  |         else if (value == "hongkong"sv) value = "hkhkg"sv; | 
					
						
							|  |  |  |         else if (value == "hst"sv) value = "utcw10"sv; | 
					
						
							|  |  |  |         else if (value == "iceland"sv) value = "isrey"sv; | 
					
						
							|  |  |  |         else if (value == "iran"sv) value = "irthr"sv; | 
					
						
							|  |  |  |         else if (value == "israel"sv) value = "jeruslm"sv; | 
					
						
							|  |  |  |         else if (value == "jamaica"sv) value = "jmkin"sv; | 
					
						
							|  |  |  |         else if (value == "japan"sv) value = "jptyo"sv; | 
					
						
							|  |  |  |         else if (value == "kwajalein"sv) value = "mhkwa"sv; | 
					
						
							|  |  |  |         else if (value == "libya"sv) value = "lytip"sv; | 
					
						
							|  |  |  |         else if (value == "mst"sv) value = "utcw07"sv; | 
					
						
							|  |  |  |         else if (value == "navajo"sv) value = "usden"sv; | 
					
						
							|  |  |  |         else if (value == "poland"sv) value = "plwaw"sv; | 
					
						
							|  |  |  |         else if (value == "portugal"sv) value = "ptlis"sv; | 
					
						
							|  |  |  |         else if (value == "prc"sv) value = "cnsha"sv; | 
					
						
							|  |  |  |         else if (value == "roc"sv) value = "twtpe"sv; | 
					
						
							|  |  |  |         else if (value == "rok"sv) value = "krsel"sv; | 
					
						
							|  |  |  |         else if (value == "singapore"sv) value = "sgsin"sv; | 
					
						
							|  |  |  |         else if (value == "turkey"sv) value = "trist"sv; | 
					
						
							|  |  |  |         else if (value == "uct"sv) value = "utc"sv; | 
					
						
							|  |  |  |         else if (value == "usnavajo"sv) value = "usden"sv; | 
					
						
							|  |  |  |         else if (value == "zulu"sv) value = "utc"sv; | 
					
						
							|  |  |  |         // clang-format on
 | 
					
						
							| 
									
										
										
										
											2021-08-31 11:00:52 -04:00
										 |  |  |     } | 
					
						
							| 
									
										
										
										
											2021-08-30 15:51:08 -04:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-31 12:19:19 -04:00
										 |  |  | static void perform_hard_coded_key_multi_value_substitutions(String const& key, Vector<String>& values) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // Similar to perform_hard_coded_key_value_substitutions, some aliases depend on multiple
 | 
					
						
							|  |  |  |     // variants being present in the original locale. Those are canonicalized separately here.
 | 
					
						
							|  |  |  |     // https://github.com/unicode-org/cldr-staging/blob/master/production/common/bcp47/calendar.xml
 | 
					
						
							|  |  |  |     if ((key != "ca"sv) || (values.size() != 3)) | 
					
						
							|  |  |  |         return; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     static Vector<String> ethiopic_amete_alem { "ethiopic"sv, "amete"sv, "alem"sv }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (values == ethiopic_amete_alem) | 
					
						
							|  |  |  |         values = { "ethioaa"sv }; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  | static void transform_unicode_locale_id_to_canonical_syntax(LocaleID& locale_id) | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | { | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |     auto canonicalize_language = [](LanguageID& language_id, bool force_lowercase) { | 
					
						
							|  |  |  |         language_id.language = language_id.language->to_lowercase(); | 
					
						
							|  |  |  |         if (language_id.script.has_value()) | 
					
						
							|  |  |  |             language_id.script = language_id.script->to_titlecase(); | 
					
						
							|  |  |  |         if (language_id.region.has_value()) | 
					
						
							|  |  |  |             language_id.region = language_id.region->to_uppercase(); | 
					
						
							|  |  |  |         for (auto& variant : language_id.variants) | 
					
						
							|  |  |  |             variant = variant.to_lowercase(); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         if (auto alias = resolve_language_alias(*language_id.language); alias.has_value()) { | 
					
						
							|  |  |  |             auto language_alias = parse_unicode_language_id(*alias); | 
					
						
							|  |  |  |             VERIFY(language_alias.has_value()); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             language_id.language = move(language_alias->language); | 
					
						
							|  |  |  |             if (!language_id.script.has_value() && language_alias->script.has_value()) | 
					
						
							|  |  |  |                 language_id.script = move(language_alias->script); | 
					
						
							|  |  |  |             if (!language_id.region.has_value() && language_alias->region.has_value()) | 
					
						
							|  |  |  |                 language_id.region = move(language_alias->region); | 
					
						
							|  |  |  |             if (language_id.variants.is_empty() && !language_alias->variants.is_empty()) | 
					
						
							|  |  |  |                 language_id.variants = move(language_alias->variants); | 
					
						
							|  |  |  |         } | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         if (language_id.script.has_value()) { | 
					
						
							|  |  |  |             if (auto alias = resolve_script_tag_alias(*language_id.script); alias.has_value()) | 
					
						
							|  |  |  |                 language_id.script = move(*alias); | 
					
						
							|  |  |  |         } | 
					
						
							| 
									
										
										
										
											2021-08-28 13:02:20 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         if (language_id.region.has_value()) { | 
					
						
							|  |  |  |             if (auto alias = resolve_territory_alias(*language_id.region); alias.has_value()) { | 
					
						
							|  |  |  |                 auto aliases = alias->split_view(' '); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 // FIXME: Territory subtag aliases should also consult the CLDR likelySubtags.json file.
 | 
					
						
							|  |  |  |                 //        For now, implement the spec's recommendation of using just the first alias.
 | 
					
						
							|  |  |  |                 language_id.region = aliases[0].to_string(); | 
					
						
							|  |  |  |             } | 
					
						
							| 
									
										
										
										
											2021-08-28 13:02:20 -04:00
										 |  |  |         } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         quick_sort(language_id.variants); | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         for (auto& variant : language_id.variants) { | 
					
						
							|  |  |  |             variant = variant.to_lowercase(); | 
					
						
							|  |  |  |             if (auto alias = resolve_variant_alias(variant); alias.has_value()) | 
					
						
							|  |  |  |                 variant = move(*alias); | 
					
						
							|  |  |  |         } | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         if (force_lowercase) { | 
					
						
							|  |  |  |             if (language_id.script.has_value()) | 
					
						
							|  |  |  |                 language_id.script = language_id.script->to_lowercase(); | 
					
						
							|  |  |  |             if (language_id.region.has_value()) | 
					
						
							|  |  |  |                 language_id.region = language_id.region->to_lowercase(); | 
					
						
							|  |  |  |         } | 
					
						
							|  |  |  |     }; | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |     auto canonicalize_key_value_list = [&](auto& key, auto& values, bool remove_true_values) { | 
					
						
							|  |  |  |         key = key.to_lowercase(); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         auto raw_values = move(values); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |         for (auto& value : raw_values) { | 
					
						
							|  |  |  |             value = value.to_lowercase(); | 
					
						
							| 
									
										
										
										
											2021-08-30 15:51:08 -04:00
										 |  |  |             perform_hard_coded_key_value_substitutions(key, value); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |             // Note: The spec says to remove "true" type and tfield values but that is believed to be a bug in the spec
 | 
					
						
							|  |  |  |             // because, for tvalues, that would result in invalid syntax:
 | 
					
						
							|  |  |  |             //     https://unicode-org.atlassian.net/browse/CLDR-14318
 | 
					
						
							|  |  |  |             // This has also been noted by test262:
 | 
					
						
							|  |  |  |             //     https://github.com/tc39/test262/blob/18bb955771669541c56c28748603f6afdb2e25ff/test/intl402/Intl/getCanonicalLocales/transformed-ext-canonical.js
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |             if (remove_true_values && (value == "true"sv)) | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |                 continue; | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |             if (key.is_one_of("sd"sv, "rg"sv)) { | 
					
						
							|  |  |  |                 if (auto alias = resolve_subdivision_alias(value); alias.has_value()) { | 
					
						
							|  |  |  |                     auto aliases = alias->split_view(' '); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                     // FIXME: Subdivision subtags do not appear in the CLDR likelySubtags.json file.
 | 
					
						
							|  |  |  |                     //        Implement the spec's recommendation of using just the first alias for now,
 | 
					
						
							|  |  |  |                     //        but we should determine if there's anything else needed here.
 | 
					
						
							|  |  |  |                     values.append(aliases[0].to_string()); | 
					
						
							|  |  |  |                     continue; | 
					
						
							|  |  |  |                 } | 
					
						
							|  |  |  |             } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |             values.append(move(value)); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |         } | 
					
						
							| 
									
										
										
										
											2021-08-31 12:19:19 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |         perform_hard_coded_key_multi_value_substitutions(key, values); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |     }; | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |     canonicalize_language(locale_id.language_id, false); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     quick_sort(locale_id.extensions, [](auto const& left, auto const& right) { | 
					
						
							|  |  |  |         auto key = [](auto const& extension) { | 
					
						
							|  |  |  |             return extension.visit( | 
					
						
							|  |  |  |                 [](LocaleExtension const&) { return 'u'; }, | 
					
						
							|  |  |  |                 [](TransformedExtension const&) { return 't'; }, | 
					
						
							|  |  |  |                 [](OtherExtension const& ext) { return static_cast<char>(to_ascii_lowercase(ext.key)); }); | 
					
						
							|  |  |  |         }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         return key(left) < key(right); | 
					
						
							|  |  |  |     }); | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |     for (auto& extension : locale_id.extensions) { | 
					
						
							|  |  |  |         extension.visit( | 
					
						
							|  |  |  |             [&](LocaleExtension& ext) { | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |                 for (auto& attribute : ext.attributes) | 
					
						
							|  |  |  |                     attribute = attribute.to_lowercase(); | 
					
						
							|  |  |  |                 for (auto& keyword : ext.keywords) | 
					
						
							|  |  |  |                     canonicalize_key_value_list(keyword.key, keyword.types, true); | 
					
						
							| 
									
										
										
										
											2021-08-31 12:19:19 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |                 quick_sort(ext.attributes); | 
					
						
							|  |  |  |                 quick_sort(ext.keywords, [](auto const& a, auto const& b) { return a.key < b.key; }); | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |             }, | 
					
						
							|  |  |  |             [&](TransformedExtension& ext) { | 
					
						
							|  |  |  |                 if (ext.language.has_value()) | 
					
						
							|  |  |  |                     canonicalize_language(*ext.language, true); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 for (auto& field : ext.fields) | 
					
						
							|  |  |  |                     canonicalize_key_value_list(field.key, field.values, false); | 
					
						
							| 
									
										
										
										
											2021-08-31 12:19:19 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |                 quick_sort(ext.fields, [](auto const& a, auto const& b) { return a.key < b.key; }); | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |             }, | 
					
						
							|  |  |  |             [&](OtherExtension& ext) { | 
					
						
							|  |  |  |                 ext.key = static_cast<char>(to_ascii_lowercase(ext.key)); | 
					
						
							|  |  |  |                 for (auto& value : ext.values) | 
					
						
							|  |  |  |                     value = value.to_lowercase(); | 
					
						
							|  |  |  |             }); | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     for (auto& extension : locale_id.private_use_extensions) | 
					
						
							|  |  |  |         extension = extension.to_lowercase(); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Optional<String> canonicalize_unicode_locale_id(LocaleID& locale_id) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     // https://unicode.org/reports/tr35/#Canonical_Unicode_Locale_Identifiers
 | 
					
						
							|  |  |  |     StringBuilder builder; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto append_sep_and_string = [&](Optional<String> const& string) { | 
					
						
							|  |  |  |         if (!string.has_value()) | 
					
						
							|  |  |  |             return; | 
					
						
							|  |  |  |         builder.appendff("-{}", *string); | 
					
						
							|  |  |  |     }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     if (!locale_id.language_id.language.has_value()) | 
					
						
							|  |  |  |         return {}; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     transform_unicode_locale_id_to_canonical_syntax(locale_id); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     builder.append(locale_id.language_id.language->to_lowercase()); | 
					
						
							|  |  |  |     append_sep_and_string(locale_id.language_id.script); | 
					
						
							|  |  |  |     append_sep_and_string(locale_id.language_id.region); | 
					
						
							|  |  |  |     for (auto const& variant : locale_id.language_id.variants) | 
					
						
							|  |  |  |         append_sep_and_string(variant); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     auto append_key_value_list = [&](auto const& key, auto& values) { | 
					
						
							|  |  |  |         append_sep_and_string(key); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |         for (auto& value : values) | 
					
						
							|  |  |  |             append_sep_and_string(value); | 
					
						
							|  |  |  |     }; | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |     for (auto const& extension : locale_id.extensions) { | 
					
						
							|  |  |  |         extension.visit( | 
					
						
							|  |  |  |             [&](LocaleExtension const& ext) { | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |                 builder.append("-u"sv); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 for (auto const& attribute : ext.attributes) | 
					
						
							|  |  |  |                     append_sep_and_string(attribute); | 
					
						
							|  |  |  |                 for (auto const& keyword : ext.keywords) | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |                     append_key_value_list(keyword.key, keyword.types); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |             }, | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |             [&](TransformedExtension const& ext) { | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |                 builder.append("-t"sv); | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 if (ext.language.has_value()) { | 
					
						
							|  |  |  |                     append_sep_and_string(ext.language->language); | 
					
						
							|  |  |  |                     append_sep_and_string(ext.language->script); | 
					
						
							|  |  |  |                     append_sep_and_string(ext.language->region); | 
					
						
							|  |  |  |                     for (auto const& variant : ext.language->variants) | 
					
						
							|  |  |  |                         append_sep_and_string(variant); | 
					
						
							|  |  |  |                 } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  |                 for (auto const& field : ext.fields) | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |                     append_key_value_list(field.key, field.values); | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |             }, | 
					
						
							| 
									
										
										
										
											2021-08-30 15:34:31 -04:00
										 |  |  |             [&](OtherExtension const& ext) { | 
					
						
							| 
									
										
										
										
											2021-08-28 13:19:26 -04:00
										 |  |  |                 builder.appendff("-{:c}", to_ascii_lowercase(ext.key)); | 
					
						
							|  |  |  |                 for (auto const& value : ext.values) | 
					
						
							|  |  |  |                     append_sep_and_string(value); | 
					
						
							|  |  |  |             }); | 
					
						
							|  |  |  |     } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-28 13:26:57 -04:00
										 |  |  |     if (!locale_id.private_use_extensions.is_empty()) { | 
					
						
							|  |  |  |         builder.append("-x"sv); | 
					
						
							|  |  |  |         for (auto const& extension : locale_id.private_use_extensions) | 
					
						
							|  |  |  |             append_sep_and_string(extension); | 
					
						
							|  |  |  |     } | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | 
 | 
					
						
							|  |  |  |     return builder.build(); | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-24 22:17:08 -04:00
										 |  |  | String const& default_locale() | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  |     static String locale = "en"sv; | 
					
						
							|  |  |  |     return locale; | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | bool is_locale_available([[maybe_unused]] StringView locale) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							| 
									
										
										
										
											2021-08-26 06:56:17 -04:00
										 |  |  |     return Detail::locale_from_string(locale).has_value(); | 
					
						
							| 
									
										
										
										
											2021-08-24 22:17:08 -04:00
										 |  |  | #else
 | 
					
						
							|  |  |  |     return false; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-26 08:17:01 -04:00
										 |  |  | Optional<StringView> get_locale_language_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView language) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::get_locale_language_mapping(locale, language); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return {}; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-26 06:56:17 -04:00
										 |  |  | Optional<StringView> get_locale_territory_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView territory) | 
					
						
							| 
									
										
										
										
											2021-08-24 22:17:08 -04:00
										 |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							| 
									
										
										
										
											2021-08-26 06:56:17 -04:00
										 |  |  |     return Detail::get_locale_territory_mapping(locale, territory); | 
					
						
							|  |  |  | #else
 | 
					
						
							| 
									
										
										
										
											2021-08-24 22:17:08 -04:00
										 |  |  |     return {}; | 
					
						
							| 
									
										
										
										
											2021-08-26 06:56:17 -04:00
										 |  |  | #endif
 | 
					
						
							| 
									
										
										
										
											2021-08-24 22:17:08 -04:00
										 |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-26 08:29:39 -04:00
										 |  |  | Optional<StringView> get_locale_script_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView script) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::get_locale_script_tag_mapping(locale, script); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return {}; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-26 08:38:54 -04:00
										 |  |  | Optional<StringView> get_locale_currency_mapping([[maybe_unused]] StringView locale, [[maybe_unused]] StringView currency) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::get_locale_currency_mapping(locale, currency); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return {}; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-30 14:56:23 -04:00
										 |  |  | Optional<StringView> resolve_language_alias(StringView language) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::resolve_language_alias(language); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return language; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Optional<StringView> resolve_territory_alias(StringView territory) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::resolve_territory_alias(territory); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return territory; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Optional<StringView> resolve_script_tag_alias(StringView script_tag) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::resolve_script_tag_alias(script_tag); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return script_tag; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Optional<StringView> resolve_variant_alias(StringView variant) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::resolve_variant_alias(variant); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return variant; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							|  |  |  | Optional<StringView> resolve_subdivision_alias(StringView subdivision) | 
					
						
							|  |  |  | { | 
					
						
							|  |  |  | #if ENABLE_UNICODE_DATA
 | 
					
						
							|  |  |  |     return Detail::resolve_subdivision_alias(subdivision); | 
					
						
							|  |  |  | #else
 | 
					
						
							|  |  |  |     return subdivision; | 
					
						
							|  |  |  | #endif
 | 
					
						
							|  |  |  | } | 
					
						
							|  |  |  | 
 | 
					
						
							| 
									
										
										
										
											2021-08-24 22:15:38 -04:00
										 |  |  | } |