mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2024-09-29 08:11:13 +00:00
LibUnicode: Replace code point scripts and script extensions with ICU
This commit is contained in:
Timothy Flynn
Andreas Kling
parent
986ff984cc
commit
e77dafc987
Notes:
sideshowbarker
2024-07-17 04:09:56 +09:00
Author: https://github.com/trflynn89 Commit: https://github.com/LadybirdBrowser/ladybird/commit/e77dafc987 Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/239
|
|
@ -13,15 +13,6 @@ set(UCD_ZIP_PATH "${UCD_PATH}/UCD.zip")
|
|||
set(UNICODE_DATA_SOURCE "UnicodeData.txt")
|
||||
set(UNICODE_DATA_PATH "${UCD_PATH}/${UNICODE_DATA_SOURCE}")
|
||||
|
||||
set(PROP_VALUE_ALIAS_SOURCE "PropertyValueAliases.txt")
|
||||
set(PROP_VALUE_ALIAS_PATH "${UCD_PATH}/${PROP_VALUE_ALIAS_SOURCE}")
|
||||
|
||||
set(SCRIPTS_SOURCE "Scripts.txt")
|
||||
set(SCRIPTS_PATH "${UCD_PATH}/${SCRIPTS_SOURCE}")
|
||||
|
||||
set(SCRIPT_EXTENSIONS_SOURCE "ScriptExtensions.txt")
|
||||
set(SCRIPT_EXTENSIONS_PATH "${UCD_PATH}/${SCRIPT_EXTENSIONS_SOURCE}")
|
||||
|
||||
string(REGEX REPLACE "([0-9]+\\.[0-9]+)\\.[0-9]+" "\\1" EMOJI_VERSION "${UCD_VERSION}")
|
||||
set(EMOJI_TEST_URL "https://www.unicode.org/Public/emoji/${EMOJI_VERSION}/emoji-test.txt")
|
||||
set(EMOJI_TEST_PATH "${UCD_PATH}/emoji-test.txt")
|
||||
|
|
@ -36,9 +27,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
|
|||
if (ENABLE_NETWORK_DOWNLOADS)
|
||||
download_file("${UCD_ZIP_URL}" "${UCD_ZIP_PATH}" SHA256 "${UCD_SHA256}")
|
||||
extract_path("${UCD_PATH}" "${UCD_ZIP_PATH}" "${UNICODE_DATA_SOURCE}" "${UNICODE_DATA_PATH}")
|
||||
extract_path("${UCD_PATH}" "${UCD_ZIP_PATH}" "${PROP_VALUE_ALIAS_SOURCE}" "${PROP_VALUE_ALIAS_PATH}")
|
||||
extract_path("${UCD_PATH}" "${UCD_ZIP_PATH}" "${SCRIPTS_SOURCE}" "${SCRIPTS_PATH}")
|
||||
extract_path("${UCD_PATH}" "${UCD_ZIP_PATH}" "${SCRIPT_EXTENSIONS_SOURCE}" "${SCRIPT_EXTENSIONS_PATH}")
|
||||
|
||||
download_file("${EMOJI_TEST_URL}" "${EMOJI_TEST_PATH}" SHA256 "${EMOJI_SHA256}")
|
||||
else()
|
||||
|
|
@ -63,7 +51,7 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
|
|||
"${UCD_VERSION_FILE}"
|
||||
"${UNICODE_DATA_HEADER}"
|
||||
"${UNICODE_DATA_IMPLEMENTATION}"
|
||||
arguments -u "${UNICODE_DATA_PATH}" -v "${PROP_VALUE_ALIAS_PATH}" -r "${SCRIPTS_PATH}" -x "${SCRIPT_EXTENSIONS_PATH}"
|
||||
arguments -u "${UNICODE_DATA_PATH}"
|
||||
)
|
||||
invoke_generator(
|
||||
"EmojiData"
|
||||
|
|
|
|||
|
|
@ -37,21 +37,6 @@ struct CodePointData {
|
|||
ByteString iso_comment;
|
||||
};
|
||||
|
||||
using PropertyTable = Vector<bool>;
|
||||
|
||||
static constexpr auto CODE_POINT_TABLES_MSB_COUNT = 16u;
|
||||
static_assert(CODE_POINT_TABLES_MSB_COUNT < 24u);
|
||||
|
||||
static constexpr auto CODE_POINT_TABLES_LSB_COUNT = 24u - CODE_POINT_TABLES_MSB_COUNT;
|
||||
static constexpr auto CODE_POINT_TABLES_LSB_MASK = NumericLimits<u32>::max() >> (NumericLimits<u32>::digits() - CODE_POINT_TABLES_LSB_COUNT);
|
||||
|
||||
template<typename PropertyType>
|
||||
struct CodePointTables {
|
||||
Vector<size_t> stage1;
|
||||
Vector<size_t> stage2;
|
||||
Vector<PropertyType> unique_properties;
|
||||
};
|
||||
|
||||
struct CodePointBidiClass {
|
||||
Unicode::CodePointRange code_point_range;
|
||||
ByteString bidi_class;
|
||||
|
|
@ -60,130 +45,10 @@ struct CodePointBidiClass {
|
|||
struct UnicodeData {
|
||||
Vector<CodePointData> code_point_data;
|
||||
|
||||
PropList script_list {
|
||||
{ "Unknown"sv, {} },
|
||||
};
|
||||
Vector<Alias> script_aliases;
|
||||
PropList script_extensions;
|
||||
|
||||
CodePointTables<PropertyTable> script_tables;
|
||||
CodePointTables<PropertyTable> script_extension_tables;
|
||||
|
||||
HashTable<ByteString> bidirectional_classes;
|
||||
Vector<CodePointBidiClass> code_point_bidirectional_classes;
|
||||
};
|
||||
|
||||
static ByteString sanitize_entry(ByteString const& entry)
|
||||
{
|
||||
auto sanitized = entry.replace("-"sv, "_"sv, ReplaceMode::All);
|
||||
sanitized = sanitized.replace(" "sv, "_"sv, ReplaceMode::All);
|
||||
|
||||
StringBuilder builder;
|
||||
bool next_is_upper = true;
|
||||
for (auto ch : sanitized) {
|
||||
if (next_is_upper)
|
||||
builder.append_code_point(to_ascii_uppercase(ch));
|
||||
else
|
||||
builder.append_code_point(ch);
|
||||
next_is_upper = ch == '_';
|
||||
}
|
||||
|
||||
return builder.to_byte_string();
|
||||
}
|
||||
|
||||
static ErrorOr<void> parse_prop_list(Core::InputBufferedFile& file, PropList& prop_list, bool multi_value_property = false, bool sanitize_property = false)
|
||||
{
|
||||
Array<u8, 1024> buffer;
|
||||
|
||||
while (TRY(file.can_read_line())) {
|
||||
auto line = TRY(file.read_line(buffer));
|
||||
|
||||
if (line.is_empty() || line.starts_with('#'))
|
||||
continue;
|
||||
|
||||
if (auto index = line.find('#'); index.has_value())
|
||||
line = line.substring_view(0, *index);
|
||||
|
||||
auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
|
||||
VERIFY(segments.size() == 2 || segments.size() == 3);
|
||||
|
||||
String combined_segment_buffer;
|
||||
|
||||
if (segments.size() == 3) {
|
||||
// For example, in DerivedCoreProperties.txt, there are lines such as:
|
||||
//
|
||||
// 094D ; InCB; Linker # Mn DEVANAGARI SIGN VIRAMA
|
||||
//
|
||||
// These are used in text segmentation to prevent breaking within some extended grapheme clusters.
|
||||
// So here, we combine the segments into a single property, which allows us to simply do code point
|
||||
// property lookups at runtime for specific Indic Conjunct Break sequences.
|
||||
combined_segment_buffer = MUST(String::join('_', Array { segments[1].trim_whitespace(), segments[2].trim_whitespace() }));
|
||||
segments[1] = combined_segment_buffer;
|
||||
}
|
||||
|
||||
auto code_point_range = parse_code_point_range(segments[0].trim_whitespace());
|
||||
Vector<StringView> properties;
|
||||
|
||||
if (multi_value_property)
|
||||
properties = segments[1].trim_whitespace().split_view(' ');
|
||||
else
|
||||
properties = { segments[1].trim_whitespace() };
|
||||
|
||||
for (auto& property : properties) {
|
||||
auto& code_points = prop_list.ensure(sanitize_property ? sanitize_entry(property).trim_whitespace() : ByteString { property.trim_whitespace() });
|
||||
code_points.append(code_point_range);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static ErrorOr<void> parse_value_alias_list(Core::InputBufferedFile& file, StringView desired_category, Vector<ByteString> const& value_list, Vector<Alias>& prop_aliases, bool primary_value_is_first = true, bool sanitize_alias = false)
|
||||
{
|
||||
TRY(file.seek(0, SeekMode::SetPosition));
|
||||
Array<u8, 1024> buffer;
|
||||
|
||||
auto append_alias = [&](auto alias, auto value) {
|
||||
// Note: The value alias file contains lines such as "Ahom = Ahom", which we should just skip.
|
||||
if (alias == value)
|
||||
return;
|
||||
|
||||
// FIXME: We will, eventually, need to find where missing properties are located and parse them.
|
||||
if (!value_list.contains_slow(value))
|
||||
return;
|
||||
|
||||
prop_aliases.append({ value, alias });
|
||||
};
|
||||
|
||||
while (TRY(file.can_read_line())) {
|
||||
auto line = TRY(file.read_line(buffer));
|
||||
|
||||
if (line.is_empty() || line.starts_with('#'))
|
||||
continue;
|
||||
|
||||
if (auto index = line.find('#'); index.has_value())
|
||||
line = line.substring_view(0, *index);
|
||||
|
||||
auto segments = line.split_view(';', SplitBehavior::KeepEmpty);
|
||||
auto category = segments[0].trim_whitespace();
|
||||
|
||||
if (category != desired_category)
|
||||
continue;
|
||||
|
||||
VERIFY((segments.size() == 3) || (segments.size() == 4));
|
||||
auto value = primary_value_is_first ? segments[1].trim_whitespace() : segments[2].trim_whitespace();
|
||||
auto alias = primary_value_is_first ? segments[2].trim_whitespace() : segments[1].trim_whitespace();
|
||||
append_alias(sanitize_alias ? sanitize_entry(alias) : ByteString { alias }, value);
|
||||
|
||||
if (segments.size() == 4) {
|
||||
alias = segments[3].trim_whitespace();
|
||||
append_alias(sanitize_alias ? sanitize_entry(alias) : ByteString { alias }, value);
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static ErrorOr<void> parse_unicode_data(Core::InputBufferedFile& file, UnicodeData& unicode_data)
|
||||
{
|
||||
Optional<u32> code_point_range_start;
|
||||
|
|
@ -285,7 +150,6 @@ enum class @name@ : @underlying@ {)~~~");
|
|||
namespace Unicode {
|
||||
)~~~");
|
||||
|
||||
generate_enum("Script"sv, {}, unicode_data.script_list.keys(), unicode_data.script_aliases);
|
||||
generate_enum("BidirectionalClass"sv, {}, unicode_data.bidirectional_classes.values());
|
||||
|
||||
generator.append(R"~~~(
|
||||
|
|
@ -301,9 +165,6 @@ static ErrorOr<void> generate_unicode_data_implementation(Core::InputBufferedFil
|
|||
StringBuilder builder;
|
||||
SourceGenerator generator { builder };
|
||||
|
||||
generator.set("CODE_POINT_TABLES_LSB_COUNT", TRY(String::number(CODE_POINT_TABLES_LSB_COUNT)));
|
||||
generator.set("CODE_POINT_TABLES_LSB_MASK", TRY(String::formatted("{:#x}", CODE_POINT_TABLES_LSB_MASK)));
|
||||
|
||||
generator.append(R"~~~(
|
||||
#include <AK/Array.h>
|
||||
#include <AK/BinarySearch.h>
|
||||
|
|
@ -333,75 +194,6 @@ struct CodePointBidiClassComparator : public CodePointRangeComparator {
|
|||
|
||||
)~~~");
|
||||
|
||||
auto append_property_table = [&](auto collection_snake, auto const& unique_properties) -> ErrorOr<void> {
|
||||
generator.set("name", TRY(String::formatted("{}_unique_properties", collection_snake)));
|
||||
generator.set("outer_size", TRY(String::number(unique_properties.size())));
|
||||
generator.set("inner_size", TRY(String::number(unique_properties[0].size())));
|
||||
|
||||
generator.append(R"~~~(
|
||||
static constexpr Array<Array<bool, @inner_size@>, @outer_size@> @name@ { {)~~~");
|
||||
|
||||
for (auto const& property_set : unique_properties) {
|
||||
generator.append(R"~~~(
|
||||
{ )~~~");
|
||||
|
||||
for (auto value : property_set) {
|
||||
generator.set("value", TRY(String::formatted("{}", value)));
|
||||
generator.append("@value@, ");
|
||||
}
|
||||
|
||||
generator.append(" },");
|
||||
}
|
||||
|
||||
generator.append(R"~~~(
|
||||
} };
|
||||
)~~~");
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
auto append_code_point_tables = [&](StringView collection_snake, auto const& tables, auto& append_unique_properties) -> ErrorOr<void> {
|
||||
auto append_stage = [&](auto const& stage, auto name, auto type) -> ErrorOr<void> {
|
||||
generator.set("name", TRY(String::formatted("{}_{}", collection_snake, name)));
|
||||
generator.set("size", TRY(String::number(stage.size())));
|
||||
generator.set("type", type);
|
||||
|
||||
generator.append(R"~~~(
|
||||
static constexpr Array<@type@, @size@> @name@ { {
|
||||
)~~~");
|
||||
|
||||
static constexpr size_t max_values_per_row = 300;
|
||||
size_t values_in_current_row = 0;
|
||||
|
||||
for (auto value : stage) {
|
||||
if (values_in_current_row++ > 0)
|
||||
generator.append(", ");
|
||||
|
||||
generator.set("value", TRY(String::number(value)));
|
||||
generator.append("@value@");
|
||||
|
||||
if (values_in_current_row == max_values_per_row) {
|
||||
values_in_current_row = 0;
|
||||
generator.append(",\n ");
|
||||
}
|
||||
}
|
||||
|
||||
generator.append(R"~~~(
|
||||
} };
|
||||
)~~~");
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
TRY(append_stage(tables.stage1, "stage1"sv, "u16"sv));
|
||||
TRY(append_stage(tables.stage2, "stage2"sv, "u16"sv));
|
||||
TRY(append_unique_properties(collection_snake, tables.unique_properties));
|
||||
return {};
|
||||
};
|
||||
|
||||
TRY(append_code_point_tables("s_scripts"sv, unicode_data.script_tables, append_property_table));
|
||||
TRY(append_code_point_tables("s_script_extensions"sv, unicode_data.script_extension_tables, append_property_table));
|
||||
|
||||
{
|
||||
constexpr size_t max_bidi_classes_per_row = 20;
|
||||
size_t bidi_classes_in_current_row = 0;
|
||||
|
|
@ -439,26 +231,6 @@ Optional<BidirectionalClass> bidirectional_class(u32 code_point)
|
|||
}
|
||||
)~~~");
|
||||
|
||||
auto append_prop_search = [&](StringView enum_title, StringView enum_snake, StringView collection_name) -> ErrorOr<void> {
|
||||
generator.set("enum_title", enum_title);
|
||||
generator.set("enum_snake", enum_snake);
|
||||
generator.set("collection_name", collection_name);
|
||||
|
||||
generator.append(R"~~~(
|
||||
bool code_point_has_@enum_snake@(u32 code_point, @enum_title@ @enum_snake@)
|
||||
{
|
||||
auto stage1_index = code_point >> @CODE_POINT_TABLES_LSB_COUNT@;
|
||||
auto stage2_index = @collection_name@_stage1[stage1_index] + (code_point & @CODE_POINT_TABLES_LSB_MASK@);
|
||||
auto unique_properties_index = @collection_name@_stage2[stage2_index];
|
||||
|
||||
auto const& property_set = @collection_name@_unique_properties[unique_properties_index];
|
||||
return property_set[to_underlying(@enum_snake@)];
|
||||
}
|
||||
)~~~");
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& prop_list, Vector<Alias> const& aliases) -> ErrorOr<void> {
|
||||
HashValueMap<StringView> hashes;
|
||||
TRY(hashes.try_ensure_capacity(prop_list.size() + aliases.size()));
|
||||
|
|
@ -482,10 +254,6 @@ bool code_point_has_@enum_snake@(u32 code_point, @enum_title@ @enum_snake@)
|
|||
return {};
|
||||
};
|
||||
|
||||
TRY(append_prop_search("Script"sv, "script"sv, "s_scripts"sv));
|
||||
TRY(append_prop_search("Script"sv, "script_extension"sv, "s_script_extensions"sv));
|
||||
TRY(append_from_string("Script"sv, "script"sv, unicode_data.script_list, unicode_data.script_aliases));
|
||||
|
||||
TRY(append_from_string("BidirectionalClass"sv, "bidirectional_class"sv, unicode_data.bidirectional_classes, {}));
|
||||
|
||||
generator.append(R"~~~(
|
||||
|
|
@ -496,277 +264,25 @@ bool code_point_has_@enum_snake@(u32 code_point, @enum_title@ @enum_snake@)
|
|||
return {};
|
||||
}
|
||||
|
||||
static Vector<u32> flatten_code_point_ranges(Vector<Unicode::CodePointRange> const& code_points)
|
||||
{
|
||||
Vector<u32> flattened;
|
||||
|
||||
for (auto const& range : code_points) {
|
||||
flattened.grow_capacity(range.last - range.first);
|
||||
for (u32 code_point = range.first; code_point <= range.last; ++code_point)
|
||||
flattened.append(code_point);
|
||||
}
|
||||
|
||||
return flattened;
|
||||
}
|
||||
|
||||
static Vector<Unicode::CodePointRange> form_code_point_ranges(Vector<u32> code_points)
|
||||
{
|
||||
Vector<Unicode::CodePointRange> ranges;
|
||||
|
||||
u32 range_start = code_points[0];
|
||||
u32 range_end = range_start;
|
||||
|
||||
for (size_t i = 1; i < code_points.size(); ++i) {
|
||||
u32 code_point = code_points[i];
|
||||
|
||||
if ((code_point - range_end) == 1) {
|
||||
range_end = code_point;
|
||||
} else {
|
||||
ranges.append({ range_start, range_end });
|
||||
range_start = code_point;
|
||||
range_end = code_point;
|
||||
}
|
||||
}
|
||||
|
||||
ranges.append({ range_start, range_end });
|
||||
return ranges;
|
||||
}
|
||||
|
||||
static void sort_and_merge_code_point_ranges(Vector<Unicode::CodePointRange>& code_points)
|
||||
{
|
||||
quick_sort(code_points, [](auto const& range1, auto const& range2) {
|
||||
return range1.first < range2.first;
|
||||
});
|
||||
|
||||
for (size_t i = 0; i < code_points.size() - 1;) {
|
||||
if (code_points[i].last >= code_points[i + 1].first) {
|
||||
code_points[i].last = max(code_points[i].last, code_points[i + 1].last);
|
||||
code_points.remove(i + 1);
|
||||
} else {
|
||||
++i;
|
||||
}
|
||||
}
|
||||
|
||||
auto all_code_points = flatten_code_point_ranges(code_points);
|
||||
code_points = form_code_point_ranges(all_code_points);
|
||||
}
|
||||
|
||||
static ErrorOr<void> normalize_script_extensions(PropList& script_extensions, PropList const& script_list, Vector<Alias> const& script_aliases)
|
||||
{
|
||||
// The ScriptExtensions UCD file lays out its code point ranges rather uniquely compared to
|
||||
// other files. The Script listed on each line may either be a full Script string or an aliased
|
||||
// abbreviation. Further, the extensions may or may not include the base Script list. Normalize
|
||||
// the extensions here to be keyed by the full Script name and always include the base list.
|
||||
auto extensions = move(script_extensions);
|
||||
script_extensions = TRY(script_list.clone());
|
||||
|
||||
for (auto const& extension : extensions) {
|
||||
auto it = find_if(script_aliases.begin(), script_aliases.end(), [&](auto const& alias) { return extension.key == alias.alias; });
|
||||
auto const& key = (it == script_aliases.end()) ? extension.key : it->name;
|
||||
|
||||
auto& code_points = script_extensions.find(key)->value;
|
||||
code_points.extend(extension.value);
|
||||
|
||||
sort_and_merge_code_point_ranges(code_points);
|
||||
}
|
||||
|
||||
// Lastly, the Common and Inherited script extensions are special. They must not contain any
|
||||
// code points which appear in other script extensions. The ScriptExtensions UCD file does not
|
||||
// list these extensions, therefore this peculiarity must be handled programmatically.
|
||||
// https://www.unicode.org/reports/tr24/#Assignment_ScriptX_Values
|
||||
auto code_point_has_other_extension = [&](StringView key, u32 code_point) {
|
||||
for (auto const& extension : extensions) {
|
||||
if (extension.key == key)
|
||||
continue;
|
||||
if (any_of(extension.value, [&](auto const& r) { return (r.first <= code_point) && (code_point <= r.last); }))
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
};
|
||||
|
||||
auto get_code_points_without_other_extensions = [&](StringView key) {
|
||||
auto code_points = flatten_code_point_ranges(script_list.find(key)->value);
|
||||
code_points.remove_all_matching([&](u32 c) { return code_point_has_other_extension(key, c); });
|
||||
return code_points;
|
||||
};
|
||||
|
||||
auto common_code_points = get_code_points_without_other_extensions("Common"sv);
|
||||
script_extensions.set("Common"sv, form_code_point_ranges(common_code_points));
|
||||
|
||||
auto inherited_code_points = get_code_points_without_other_extensions("Inherited"sv);
|
||||
script_extensions.set("Inherited"sv, form_code_point_ranges(inherited_code_points));
|
||||
return {};
|
||||
}
|
||||
|
||||
struct PropertyMetadata {
|
||||
static ErrorOr<PropertyMetadata> create(PropList& property_list)
|
||||
{
|
||||
PropertyMetadata data;
|
||||
TRY(data.property_values.try_ensure_capacity(property_list.size()));
|
||||
TRY(data.property_set.try_ensure_capacity(property_list.size()));
|
||||
|
||||
auto property_names = property_list.keys();
|
||||
quick_sort(property_names);
|
||||
|
||||
for (auto& property_name : property_names) {
|
||||
auto& code_point_ranges = property_list.get(property_name).value();
|
||||
data.property_values.unchecked_append(move(code_point_ranges));
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
Vector<typename PropList::ValueType> property_values;
|
||||
PropertyTable property_set;
|
||||
|
||||
Vector<size_t> current_block;
|
||||
HashMap<decltype(current_block), size_t> unique_blocks;
|
||||
};
|
||||
|
||||
// The goal here is to produce a set of tables that represent a category of code point properties for every code point.
|
||||
// The most naive method would be to generate a single table per category, each with one entry per code point. Each of
|
||||
// those tables would have a size of 0x10ffff though, which is a non-starter. Instead, we create a set of 2-stage lookup
|
||||
// tables per category.
|
||||
//
|
||||
// To do so, it's important to note that Unicode tends to organize code points with similar properties together. This
|
||||
// leads to long series of code points with identical properties. Therefore, if we divide the 0x10ffff code points into
|
||||
// fixed-size blocks, many of those blocks will also be identical.
|
||||
//
|
||||
// So we iterate over every code point, classifying each one for the category of interest. We represent a classification
|
||||
// as a list of booleans. We store the classification in the CodePointTables::unique_properties list for this category.
|
||||
// As the name implies, this list is de-duplicated; we store the index into this list in a separate list, which we call
|
||||
// a "block".
|
||||
//
|
||||
// As we iterate, we "pause" every BLOCK_SIZE code points to examine the block. If the block is unique so far, we extend
|
||||
// CodePointTables::stage2 with the entries of that block (so CodePointTables::stage2 is also a list of indices into
|
||||
// CodePointTables::unique_properties). We then append the index of the start of that block in CodePointTables::stage2
|
||||
// to CodePointTables::stage1.
|
||||
//
|
||||
// The value of BLOCK_SIZE is determined by CodePointTables::MSB_COUNT and CodePointTables::LSB_COUNT. We need 24 bits
|
||||
// to describe all code points; the blocks we create are based on splitting these bits into 2 segments. We currently use
|
||||
// a 16:8 bit split. So when perform a runtime lookup of a code point in the 2-stage tables, we:
|
||||
//
|
||||
// 1. Use most-significant 16 bits of the code point as the index into CodePointTables::stage1. That value is the
|
||||
// index into CodePointTables::stage2 of the start of the block that contains properties for this code point.
|
||||
//
|
||||
// 2. Add the least-significant 8 bits of the code point to that value, to use as the index into
|
||||
// CodePointTables::stage2. As described above, that value is the index into CodePointTables::unique_properties,
|
||||
// which contains the classification for this code point.
|
||||
//
|
||||
// Using the code point GeneralCategory as an example, we end up with a CodePointTables::stage1 with a size of ~4000,
|
||||
// a CodePointTables::stage2 with a size of ~40,000, and a CodePointTables::unique_properties with a size of ~30. So
|
||||
// this process reduces over 1 million entries (0x10ffff) to ~44,030.
|
||||
//
|
||||
// For much more in-depth reading, see: https://icu.unicode.org/design/struct/utrie
|
||||
static constexpr auto MAX_CODE_POINT = 0x10ffffu;
|
||||
|
||||
template<typename T>
|
||||
static ErrorOr<void> update_tables(u32 code_point, CodePointTables<T>& tables, auto& metadata, auto const& values)
|
||||
{
|
||||
static constexpr auto BLOCK_SIZE = CODE_POINT_TABLES_LSB_MASK + 1;
|
||||
|
||||
size_t unique_properties_index = 0;
|
||||
if (auto block_index = tables.unique_properties.find_first_index(values); block_index.has_value()) {
|
||||
unique_properties_index = *block_index;
|
||||
} else {
|
||||
unique_properties_index = tables.unique_properties.size();
|
||||
TRY(tables.unique_properties.try_append(values));
|
||||
}
|
||||
|
||||
TRY(metadata.current_block.try_append(unique_properties_index));
|
||||
|
||||
if (metadata.current_block.size() == BLOCK_SIZE || code_point == MAX_CODE_POINT) {
|
||||
size_t stage2_index = 0;
|
||||
if (auto block_index = metadata.unique_blocks.get(metadata.current_block); block_index.has_value()) {
|
||||
stage2_index = *block_index;
|
||||
} else {
|
||||
stage2_index = tables.stage2.size();
|
||||
TRY(tables.stage2.try_extend(metadata.current_block));
|
||||
|
||||
TRY(metadata.unique_blocks.try_set(metadata.current_block, stage2_index));
|
||||
}
|
||||
|
||||
TRY(tables.stage1.try_append(stage2_index));
|
||||
metadata.current_block.clear_with_capacity();
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
static ErrorOr<void> create_code_point_tables(UnicodeData& unicode_data)
|
||||
{
|
||||
auto update_property_tables = [&]<typename T>(u32 code_point, CodePointTables<T>& tables, PropertyMetadata& metadata) -> ErrorOr<void> {
|
||||
static Unicode::CodePointRangeComparator comparator {};
|
||||
|
||||
for (auto& property_values : metadata.property_values) {
|
||||
size_t ranges_to_remove = 0;
|
||||
auto has_property = false;
|
||||
|
||||
for (auto const& range : property_values) {
|
||||
if (auto comparison = comparator(code_point, range); comparison <= 0) {
|
||||
has_property = comparison == 0;
|
||||
break;
|
||||
}
|
||||
|
||||
++ranges_to_remove;
|
||||
}
|
||||
|
||||
metadata.property_set.unchecked_append(has_property);
|
||||
property_values.remove(0, ranges_to_remove);
|
||||
}
|
||||
|
||||
TRY(update_tables(code_point, tables, metadata, metadata.property_set));
|
||||
metadata.property_set.clear_with_capacity();
|
||||
|
||||
return {};
|
||||
};
|
||||
|
||||
auto script_metadata = TRY(PropertyMetadata::create(unicode_data.script_list));
|
||||
auto script_extension_metadata = TRY(PropertyMetadata::create(unicode_data.script_extensions));
|
||||
|
||||
for (u32 code_point = 0; code_point <= MAX_CODE_POINT; ++code_point) {
|
||||
TRY(update_property_tables(code_point, unicode_data.script_tables, script_metadata));
|
||||
TRY(update_property_tables(code_point, unicode_data.script_extension_tables, script_extension_metadata));
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
ErrorOr<int> serenity_main(Main::Arguments arguments)
|
||||
{
|
||||
StringView generated_header_path;
|
||||
StringView generated_implementation_path;
|
||||
StringView unicode_data_path;
|
||||
StringView prop_value_alias_path;
|
||||
StringView scripts_path;
|
||||
StringView script_extensions_path;
|
||||
|
||||
Core::ArgsParser args_parser;
|
||||
args_parser.add_option(generated_header_path, "Path to the Unicode Data header file to generate", "generated-header-path", 'h', "generated-header-path");
|
||||
args_parser.add_option(generated_implementation_path, "Path to the Unicode Data implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
|
||||
args_parser.add_option(unicode_data_path, "Path to UnicodeData.txt file", "unicode-data-path", 'u', "unicode-data-path");
|
||||
args_parser.add_option(prop_value_alias_path, "Path to PropertyValueAliases.txt file", "prop-value-alias-path", 'v', "prop-value-alias-path");
|
||||
args_parser.add_option(scripts_path, "Path to Scripts.txt file", "scripts-path", 'r', "scripts-path");
|
||||
args_parser.add_option(script_extensions_path, "Path to ScriptExtensions.txt file", "script-extensions-path", 'x', "script-extensions-path");
|
||||
args_parser.parse(arguments);
|
||||
|
||||
auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
|
||||
auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
|
||||
auto unicode_data_file = TRY(open_file(unicode_data_path, Core::File::OpenMode::Read));
|
||||
auto prop_value_alias_file = TRY(open_file(prop_value_alias_path, Core::File::OpenMode::Read));
|
||||
auto scripts_file = TRY(open_file(scripts_path, Core::File::OpenMode::Read));
|
||||
auto script_extensions_file = TRY(open_file(script_extensions_path, Core::File::OpenMode::Read));
|
||||
|
||||
UnicodeData unicode_data {};
|
||||
TRY(parse_prop_list(*scripts_file, unicode_data.script_list));
|
||||
TRY(parse_prop_list(*script_extensions_file, unicode_data.script_extensions, true));
|
||||
|
||||
TRY(parse_unicode_data(*unicode_data_file, unicode_data));
|
||||
TRY(parse_value_alias_list(*prop_value_alias_file, "sc"sv, unicode_data.script_list.keys(), unicode_data.script_aliases, false));
|
||||
TRY(normalize_script_extensions(unicode_data.script_extensions, unicode_data.script_list, unicode_data.script_aliases));
|
||||
|
||||
TRY(create_code_point_tables(unicode_data));
|
||||
|
||||
TRY(generate_unicode_data_header(*generated_header_file, unicode_data));
|
||||
TRY(generate_unicode_data_implementation(*generated_implementation_file, unicode_data));
|
||||
|
|
|
|||
|
|
@ -1651,9 +1651,9 @@ bool ECMA262Parser::parse_atom_escape(ByteCode& stack, size_t& match_length_mini
|
|||
},
|
||||
[&](Script script) {
|
||||
if (script.is_extension)
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script });
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script.value() });
|
||||
else
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script });
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script.value() });
|
||||
},
|
||||
[](Empty&) { VERIFY_NOT_REACHED(); });
|
||||
stack.insert_bytecode_compare_values(move(compares));
|
||||
|
|
@ -2000,9 +2000,9 @@ bool ECMA262Parser::parse_nonempty_class_ranges(Vector<CompareTypeAndValuePair>&
|
|||
else if (atom.is_general_category)
|
||||
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::GeneralCategory, (ByteCodeValueType)(atom.general_category.value()) });
|
||||
else if (atom.is_script)
|
||||
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)(atom.script) });
|
||||
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)(atom.script.value()) });
|
||||
else if (atom.is_script_extension)
|
||||
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)(atom.script) });
|
||||
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)(atom.script.value()) });
|
||||
else
|
||||
ranges.empend(CompareTypeAndValuePair { CharacterCompareType::CharClass, (ByteCodeValueType)atom.character_class });
|
||||
} else {
|
||||
|
|
@ -2339,9 +2339,9 @@ bool ECMA262Parser::parse_class_set_operand(Vector<regex::CompareTypeAndValuePai
|
|||
},
|
||||
[&](Script script) {
|
||||
if (script.is_extension)
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script });
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script.value() });
|
||||
else
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script });
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script.value() });
|
||||
},
|
||||
[](Empty&) { VERIFY_NOT_REACHED(); });
|
||||
return true;
|
||||
|
|
@ -2426,9 +2426,9 @@ bool ECMA262Parser::parse_nested_class(Vector<regex::CompareTypeAndValuePair>& c
|
|||
},
|
||||
[&](Script script) {
|
||||
if (script.is_extension)
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script });
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::ScriptExtension, (ByteCodeValueType)script.script.value() });
|
||||
else
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script });
|
||||
compares.empend(CompareTypeAndValuePair { CharacterCompareType::Script, (ByteCodeValueType)script.script.value() });
|
||||
},
|
||||
[](Empty&) { VERIFY_NOT_REACHED(); });
|
||||
return true;
|
||||
|
|
|
|||
|
|
@ -4,13 +4,17 @@
|
|||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#define AK_DONT_REPLACE_STD
|
||||
|
||||
#include <AK/Array.h>
|
||||
#include <AK/CharacterTypes.h>
|
||||
#include <AK/Find.h>
|
||||
#include <AK/Traits.h>
|
||||
#include <LibLocale/ICU.h>
|
||||
#include <LibUnicode/CharacterTypes.h>
|
||||
|
||||
#include <unicode/uchar.h>
|
||||
#include <unicode/uscript.h>
|
||||
|
||||
namespace Unicode {
|
||||
|
||||
|
|
@ -262,9 +266,49 @@ bool is_ecma262_property(Property property)
|
|||
}
|
||||
}
|
||||
|
||||
Optional<Script> __attribute__((weak)) script_from_string(StringView) { return {}; }
|
||||
bool __attribute__((weak)) code_point_has_script(u32, Script) { return {}; }
|
||||
bool __attribute__((weak)) code_point_has_script_extension(u32, Script) { return {}; }
|
||||
Optional<Script> script_from_string(StringView script)
|
||||
{
|
||||
static auto script_names = []() {
|
||||
Array<PropertyName<Script>, static_cast<size_t>(USCRIPT_CODE_LIMIT)> names;
|
||||
|
||||
for (Script script = 0; script < USCRIPT_CODE_LIMIT; ++script) {
|
||||
auto icu_script = static_cast<UScriptCode>(script.value());
|
||||
|
||||
if (char const* name = uscript_getName(icu_script))
|
||||
names[script.value()].long_name = StringView { name, strlen(name) };
|
||||
if (char const* name = uscript_getShortName(icu_script))
|
||||
names[script.value()].short_name = StringView { name, strlen(name) };
|
||||
if (char const* name = u_getPropertyValueName(UCHAR_SCRIPT, icu_script, ADDITIONAL_NAME))
|
||||
names[script.value()].additional_name = StringView { name, strlen(name) };
|
||||
}
|
||||
|
||||
return names;
|
||||
}();
|
||||
|
||||
if (auto index = find_index(script_names.begin(), script_names.end(), script); index != script_names.size())
|
||||
return static_cast<Script>(index);
|
||||
return {};
|
||||
}
|
||||
|
||||
bool code_point_has_script(u32 code_point, Script script)
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
|
||||
auto icu_code_point = static_cast<UChar32>(code_point);
|
||||
auto icu_script = static_cast<UScriptCode>(script.value());
|
||||
|
||||
if (auto result = uscript_getScript(icu_code_point, &status); Locale::icu_success(status))
|
||||
return result == icu_script;
|
||||
return false;
|
||||
}
|
||||
|
||||
bool code_point_has_script_extension(u32 code_point, Script script)
|
||||
{
|
||||
auto icu_code_point = static_cast<UChar32>(code_point);
|
||||
auto icu_script = static_cast<UScriptCode>(script.value());
|
||||
|
||||
return static_cast<bool>(uscript_hasScript(icu_code_point, icu_script));
|
||||
}
|
||||
|
||||
Optional<BidirectionalClass> __attribute__((weak)) bidirectional_class_from_string(StringView) { return {}; }
|
||||
Optional<BidirectionalClass> __attribute__((weak)) bidirectional_class(u32) { return {}; }
|
||||
|
|
|
|||
|
|
@ -13,12 +13,12 @@ namespace Unicode {
|
|||
|
||||
enum class BidirectionalClass : u8;
|
||||
enum class EmojiGroup : u8;
|
||||
enum class Script : u8;
|
||||
|
||||
struct CurrencyCode;
|
||||
struct Emoji;
|
||||
|
||||
AK_TYPEDEF_DISTINCT_NUMERIC_GENERAL(u32, GeneralCategory, CastToUnderlying, Comparison, Increment);
|
||||
AK_TYPEDEF_DISTINCT_NUMERIC_GENERAL(u32, Property, CastToUnderlying, Comparison, Increment);
|
||||
AK_TYPEDEF_DISTINCT_NUMERIC_GENERAL(u32, Script, CastToUnderlying, Comparison, Increment);
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue