beenull/ladybird
1
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2024-09-29 16:21:29 +00:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

LibJS+LibLocale: Replace calendar weekday information with ICU

Browse source
This commit is contained in:
Timothy Flynn 2024-06-12 17:00:45 -04:00 committed by Andreas Kling
parent 9cb1857dc6
commit 4b3e26c583
Notes: sideshowbarker 2024-07-17 00:59:43 +09:00 
Author: https://github.com/trflynn89
Commit: https://github.com/LadybirdBrowser/ladybird/commit/4b3e26c583
Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/145
7 changed files with 87 additions and 351 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
Meta/CMake/locale_data.cmake
View file

@ -38,9 +38,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
message(STATUS "Skipping download of ${CLDR_ZIP_URL}, expecting the archive to have been extracted to ${CLDR_PATH}")
endif()
set(DATE_TIME_FORMAT_DATA_HEADER DateTimeFormatData.h)
set(DATE_TIME_FORMAT_DATA_IMPLEMENTATION DateTimeFormatData.cpp)
set(LOCALE_DATA_HEADER LocaleData.h)
set(LOCALE_DATA_IMPLEMENTATION LocaleData.cpp)
@ -53,14 +50,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
set(RELATIVE_TIME_FORMAT_DATA_HEADER RelativeTimeFormatData.h)
set(RELATIVE_TIME_FORMAT_DATA_IMPLEMENTATION RelativeTimeFormatData.cpp)
invoke_generator(
"DateTimeFormatData"
Lagom::GenerateDateTimeFormatData
"${CLDR_VERSION_FILE}"
"${DATE_TIME_FORMAT_DATA_HEADER}"
"${DATE_TIME_FORMAT_DATA_IMPLEMENTATION}"
arguments -r "${CLDR_CORE_PATH}"
)
invoke_generator(
"LocaleData"
Lagom::GenerateLocaleData
@ -95,8 +84,6 @@ if (ENABLE_UNICODE_DATABASE_DOWNLOAD)
)
set(LOCALE_DATA_SOURCES
${DATE_TIME_FORMAT_DATA_HEADER}
${DATE_TIME_FORMAT_DATA_IMPLEMENTATION}
${LOCALE_DATA_HEADER}
${LOCALE_DATA_IMPLEMENTATION}
${NUMBER_FORMAT_DATA_HEADER}

1
Meta/Lagom/Tools/CodeGenerators/LibLocale/CMakeLists.txt
View file

@ -1,4 +1,3 @@
lagom_tool(GenerateDateTimeFormatData SOURCES GenerateDateTimeFormatData.cpp LIBS LibMain LibTimeZone)
lagom_tool(GenerateLocaleData SOURCES GenerateLocaleData.cpp LIBS LibMain)
lagom_tool(GenerateNumberFormatData SOURCES GenerateNumberFormatData.cpp LIBS LibMain)
lagom_tool(GeneratePluralRulesData SOURCES GeneratePluralRulesData.cpp LIBS LibMain)

267
Meta/Lagom/Tools/CodeGenerators/LibLocale/GenerateDateTimeFormatData.cpp
View file

@ -1,267 +0,0 @@
/*
* Copyright (c) 2021-2024, Tim Flynn <trflynn89@serenityos.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include "../LibUnicode/GeneratorUtil.h" // FIXME: Move this somewhere common.
#include <AK/AllOf.h>
#include <AK/ByteString.h>
#include <AK/CharacterTypes.h>
#include <AK/Error.h>
#include <AK/Format.h>
#include <AK/HashMap.h>
#include <AK/JsonObject.h>
#include <AK/JsonParser.h>
#include <AK/JsonValue.h>
#include <AK/LexicalPath.h>
#include <AK/SourceGenerator.h>
#include <AK/StringBuilder.h>
#include <LibCore/ArgsParser.h>
#include <LibCore/Directory.h>
#include <LibLocale/DateTimeFormat.h>
struct CLDR {
HashMap<ByteString, u8> minimum_days;
Vector<ByteString> minimum_days_regions;
HashMap<ByteString, Locale::Weekday> first_day;
Vector<ByteString> first_day_regions;
HashMap<ByteString, Locale::Weekday> weekend_start;
Vector<ByteString> weekend_start_regions;
HashMap<ByteString, Locale::Weekday> weekend_end;
Vector<ByteString> weekend_end_regions;
};
static ErrorOr<void> parse_week_data(ByteString core_path, CLDR& cldr)
{
// https://unicode.org/reports/tr35/tr35-dates.html#Week_Data
LexicalPath week_data_path(move(core_path));
week_data_path = week_data_path.append("supplemental"sv);
week_data_path = week_data_path.append("weekData.json"sv);
auto week_data = TRY(read_json_file(week_data_path.string()));
auto const& supplemental_object = week_data.as_object().get_object("supplemental"sv).value();
auto const& week_data_object = supplemental_object.get_object("weekData"sv).value();
auto parse_weekday = [](StringView day) -> Locale::Weekday {
if (day == "sun"sv)
return Locale::Weekday::Sunday;
if (day == "mon"sv)
return Locale::Weekday::Monday;
if (day == "tue"sv)
return Locale::Weekday::Tuesday;
if (day == "wed"sv)
return Locale::Weekday::Wednesday;
if (day == "thu"sv)
return Locale::Weekday::Thursday;
if (day == "fri"sv)
return Locale::Weekday::Friday;
if (day == "sat"sv)
return Locale::Weekday::Saturday;
VERIFY_NOT_REACHED();
};
auto parse_regional_weekdays = [&](auto const& region, auto const& weekday, auto& weekdays_map, auto& weekday_regions) {
if (region.ends_with("alt-variant"sv))
return;
weekdays_map.set(region, parse_weekday(weekday));
if (!weekday_regions.contains_slow(region))
weekday_regions.append(region);
};
auto const& minimum_days_object = week_data_object.get_object("minDays"sv).value();
auto const& first_day_object = week_data_object.get_object("firstDay"sv).value();
auto const& weekend_start_object = week_data_object.get_object("weekendStart"sv).value();
auto const& weekend_end_object = week_data_object.get_object("weekendEnd"sv).value();
minimum_days_object.for_each_member([&](auto const& region, auto const& value) {
auto minimum_days = value.as_string().template to_number<u8>();
cldr.minimum_days.set(region, *minimum_days);
if (!cldr.minimum_days_regions.contains_slow(region))
cldr.minimum_days_regions.append(region);
});
first_day_object.for_each_member([&](auto const& region, auto const& value) {
parse_regional_weekdays(region, value.as_string(), cldr.first_day, cldr.first_day_regions);
});
weekend_start_object.for_each_member([&](auto const& region, auto const& value) {
parse_regional_weekdays(region, value.as_string(), cldr.weekend_start, cldr.weekend_start_regions);
});
weekend_end_object.for_each_member([&](auto const& region, auto const& value) {
parse_regional_weekdays(region, value.as_string(), cldr.weekend_end, cldr.weekend_end_regions);
});
return {};
}
static ErrorOr<void> parse_all_locales(ByteString core_path, CLDR& cldr)
{
TRY(parse_week_data(core_path, cldr));
return {};
}
static ByteString format_identifier(StringView owner, ByteString identifier)
{
identifier = identifier.replace("-"sv, "_"sv, ReplaceMode::All);
identifier = identifier.replace("/"sv, "_"sv, ReplaceMode::All);
if (all_of(identifier, is_ascii_digit))
return ByteString::formatted("{}_{}", owner[0], identifier);
if (is_ascii_lower_alpha(identifier[0]))
return ByteString::formatted("{:c}{}", to_ascii_uppercase(identifier[0]), identifier.substring_view(1));
return identifier;
}
static ErrorOr<void> generate_unicode_locale_header(Core::InputBufferedFile& file, CLDR& cldr)
{
StringBuilder builder;
SourceGenerator generator { builder };
generator.append(R"~~~(
#pragma once
#include <AK/Types.h>
namespace Locale {
)~~~");
generate_enum(generator, format_identifier, "MinimumDaysRegion"sv, {}, cldr.minimum_days_regions);
generate_enum(generator, format_identifier, "FirstDayRegion"sv, {}, cldr.first_day_regions);
generate_enum(generator, format_identifier, "WeekendStartRegion"sv, {}, cldr.weekend_start_regions);
generate_enum(generator, format_identifier, "WeekendEndRegion"sv, {}, cldr.weekend_end_regions);
generator.append(R"~~~(
}
)~~~");
TRY(file.write_until_depleted(generator.as_string_view().bytes()));
return {};
}
static ErrorOr<void> generate_unicode_locale_implementation(Core::InputBufferedFile& file, CLDR& cldr)
{
StringBuilder builder;
SourceGenerator generator { builder };
generator.append(R"~~~(
#include <AK/Array.h>
#include <AK/BinarySearch.h>
#include <AK/Error.h>
#include <AK/Optional.h>
#include <AK/String.h>
#include <AK/StringView.h>
#include <LibLocale/DateTimeFormat.h>
#include <LibLocale/DateTimeFormatData.h>
#include <LibLocale/Locale.h>
#include <LibLocale/LocaleData.h>
namespace Locale {
)~~~");
auto append_mapping = [&](auto const& keys, auto const& map, auto type, auto name, auto mapping_getter) {
generator.set("type", type);
generator.set("name", name);
generator.set("size", ByteString::number(keys.size()));
generator.append(R"~~~(
static constexpr Array<@type@, @size@> @name@ { {)~~~");
bool first = true;
for (auto const& key : keys) {
auto const& value = map.find(key)->value;
auto mapping = mapping_getter(value);
generator.append(first ? " "sv : ", "sv);
generator.append(ByteString::number(mapping));
first = false;
}
generator.append(" } };");
};
append_mapping(cldr.minimum_days_regions, cldr.minimum_days, "u8"sv, "s_minimum_days"sv, [](auto minimum_days) { return minimum_days; });
append_mapping(cldr.first_day_regions, cldr.first_day, "u8"sv, "s_first_day"sv, [](auto first_day) { return to_underlying(first_day); });
append_mapping(cldr.weekend_start_regions, cldr.weekend_start, "u8"sv, "s_weekend_start"sv, [](auto weekend_start) { return to_underlying(weekend_start); });
append_mapping(cldr.weekend_end_regions, cldr.weekend_end, "u8"sv, "s_weekend_end"sv, [](auto weekend_end) { return to_underlying(weekend_end); });
generator.append("\n");
auto append_from_string = [&](StringView enum_title, StringView enum_snake, auto const& values, Vector<Alias> const& aliases = {}) -> ErrorOr<void> {
HashValueMap<ByteString> hashes;
TRY(hashes.try_ensure_capacity(values.size()));
for (auto const& value : values)
hashes.set(value.hash(), format_identifier(enum_title, value));
for (auto const& alias : aliases)
hashes.set(alias.alias.hash(), format_identifier(enum_title, alias.alias));
generate_value_from_string(generator, "{}_from_string"sv, enum_title, enum_snake, move(hashes));
return {};
};
TRY(append_from_string("MinimumDaysRegion"sv, "minimum_days_region"sv, cldr.minimum_days_regions));
TRY(append_from_string("FirstDayRegion"sv, "first_day_region"sv, cldr.first_day_regions));
TRY(append_from_string("WeekendStartRegion"sv, "weekend_start_region"sv, cldr.weekend_start_regions));
TRY(append_from_string("WeekendEndRegion"sv, "weekend_end_region"sv, cldr.weekend_end_regions));
auto append_regional_lookup = [&](StringView return_type, StringView lookup_type) {
generator.set("return_type", return_type);
generator.set("lookup_type", lookup_type);
generator.append(R"~~~(
Optional<@return_type@> get_regional_@lookup_type@(StringView region)
{
auto region_value = @lookup_type@_region_from_string(region);
if (!region_value.has_value())
return {};
auto region_index = to_underlying(*region_value);
auto @lookup_type@ = s_@lookup_type@.at(region_index);
return static_cast<@return_type@>(@lookup_type@);
}
)~~~");
};
append_regional_lookup("u8"sv, "minimum_days"sv);
append_regional_lookup("Weekday"sv, "first_day"sv);
append_regional_lookup("Weekday"sv, "weekend_start"sv);
append_regional_lookup("Weekday"sv, "weekend_end"sv);
generator.append(R"~~~(
}
)~~~");
TRY(file.write_until_depleted(generator.as_string_view().bytes()));
return {};
}
ErrorOr<int> serenity_main(Main::Arguments arguments)
{
StringView generated_header_path;
StringView generated_implementation_path;
StringView core_path;
Core::ArgsParser args_parser;
args_parser.add_option(generated_header_path, "Path to the Unicode locale header file to generate", "generated-header-path", 'h', "generated-header-path");
args_parser.add_option(generated_implementation_path, "Path to the Unicode locale implementation file to generate", "generated-implementation-path", 'c', "generated-implementation-path");
args_parser.add_option(core_path, "Path to cldr-core directory", "core-path", 'r', "core-path");
args_parser.parse(arguments);
auto generated_header_file = TRY(open_file(generated_header_path, Core::File::OpenMode::Write));
auto generated_implementation_file = TRY(open_file(generated_implementation_path, Core::File::OpenMode::Write));
CLDR cldr;
TRY(parse_all_locales(core_path, cldr));
TRY(generate_unicode_locale_header(*generated_header_file, cldr));
TRY(generate_unicode_locale_implementation(*generated_implementation_file, cldr));
return 0;
}

31
Userland/Libraries/LibJS/Runtime/Intl/Locale.cpp
View file

@ -254,10 +254,10 @@ StringView weekday_to_string(StringView weekday)
VERIFY_NOT_REACHED();
}
static u8 weekday_to_integer(Optional<::Locale::Weekday> weekday, ::Locale::Weekday falllback)
static u8 weekday_to_integer(Optional<::Locale::Weekday> const& weekday, ::Locale::Weekday falllback)
{
// NOTE: This fallback will be used if LibUnicode data generation is disabled. Its value should
// be that of the default region ("001") in the CLDR.
// NOTE: This fallback will be used if the ICU data lookup failed. Its value should be that of the
// default region ("001") in the CLDR.
switch (weekday.value_or(falllback)) {
case ::Locale::Weekday::Monday:
return 1;
@ -274,25 +274,18 @@ static u8 weekday_to_integer(Optional<::Locale::Weekday> weekday, ::Locale::Week
case ::Locale::Weekday::Sunday:
return 7;
}
VERIFY_NOT_REACHED();
}
static Vector<u8> weekend_of_locale(StringView locale)
static Vector<u8> weekend_of_locale(ReadonlySpan<::Locale::Weekday> const& weekend_days)
{
auto weekend_start = weekday_to_integer(::Locale::get_locale_weekend_start(locale), ::Locale::Weekday::Saturday);
auto weekend_end = weekday_to_integer(::Locale::get_locale_weekend_end(locale), ::Locale::Weekday::Sunday);
// There currently aren't any regions in the CLDR which wrap around from Sunday (7) to Monday (1).
// If this changes, this logic will need to be updated to handle that.
VERIFY(weekend_start <= weekend_end);
Vector<u8> weekend;
weekend.ensure_capacity(weekend_end - weekend_start + 1);
weekend.ensure_capacity(weekend_days.size());
for (auto day = weekend_start; day <= weekend_end; ++day)
weekend.unchecked_append(day);
for (auto day : weekend_days)
weekend.unchecked_append(weekday_to_integer(day, day));
quick_sort(weekend);
return weekend;
}
@ -306,10 +299,12 @@ WeekInfo week_info_of_locale(Locale const& locale_object)
VERIFY(::Locale::parse_unicode_locale_id(locale).has_value());
// 3. Let r be a record whose fields are defined by Table 3, with values based on locale.
auto locale_week_info = ::Locale::week_info_of_locale(locale);
WeekInfo week_info {};
week_info.minimal_days = ::Locale::get_locale_minimum_days(locale).value_or(1);
week_info.first_day = weekday_to_integer(::Locale::get_locale_first_day(locale), ::Locale::Weekday::Monday);
week_info.weekend = weekend_of_locale(locale);
week_info.minimal_days = locale_week_info.minimal_days_in_first_week;
week_info.first_day = weekday_to_integer(locale_week_info.first_day_of_week, ::Locale::Weekday::Monday);
week_info.weekend = weekend_of_locale(locale_week_info.weekend_days);
// 4. Let fw be loc.[[FirstDayOfWeek]].
// 5. If fw is not undefined, then

99
Userland/Libraries/LibLocale/DateTimeFormat.cpp
View file

@ -24,6 +24,7 @@
#include <unicode/gregocal.h>
#include <unicode/smpdtfmt.h>
#include <unicode/timezone.h>
#include <unicode/ucal.h>
namespace Locale {
@ -530,38 +531,6 @@ static T find_regional_values_for_locale(StringView locale, GetRegionalValues&&
return return_default_values();
}
Optional<MinimumDaysRegion> __attribute__((weak)) minimum_days_region_from_string(StringView) { return {}; }
Optional<u8> __attribute__((weak)) get_regional_minimum_days(StringView) { return {}; }
Optional<u8> get_locale_minimum_days(StringView locale)
{
return find_regional_values_for_locale<Optional<u8>>(locale, get_regional_minimum_days);
}
Optional<FirstDayRegion> __attribute__((weak)) first_day_region_from_string(StringView) { return {}; }
Optional<Weekday> __attribute__((weak)) get_regional_first_day(StringView) { return {}; }
Optional<Weekday> get_locale_first_day(StringView locale)
{
return find_regional_values_for_locale<Optional<Weekday>>(locale, get_regional_first_day);
}
Optional<WeekendStartRegion> __attribute__((weak)) weekend_start_region_from_string(StringView) { return {}; }
Optional<Weekday> __attribute__((weak)) get_regional_weekend_start(StringView) { return {}; }
Optional<Weekday> get_locale_weekend_start(StringView locale)
{
return find_regional_values_for_locale<Optional<Weekday>>(locale, get_regional_weekend_start);
}
Optional<WeekendEndRegion> __attribute__((weak)) weekend_end_region_from_string(StringView) { return {}; }
Optional<Weekday> __attribute__((weak)) get_regional_weekend_end(StringView) { return {}; }
Optional<Weekday> get_locale_weekend_end(StringView locale)
{
return find_regional_values_for_locale<Optional<Weekday>>(locale, get_regional_weekend_end);
}
// ICU does not contain a field enumeration for "literal" partitions. Define a custom field so that we may provide a
// type for those partitions.
static constexpr i32 LITERAL_FIELD = -1;
@ -961,4 +930,70 @@ NonnullOwnPtr<DateTimeFormat> DateTimeFormat::create_for_pattern_options(
return adopt_own(*new DateTimeFormatImpl(locale_data->locale(), pattern, time_zone_identifier, move(formatter)));
}
static constexpr Weekday icu_calendar_day_to_weekday(UCalendarDaysOfWeek day)
{
switch (day) {
case UCAL_SUNDAY:
return Weekday::Sunday;
case UCAL_MONDAY:
return Weekday::Monday;
case UCAL_TUESDAY:
return Weekday::Tuesday;
case UCAL_WEDNESDAY:
return Weekday::Wednesday;
case UCAL_THURSDAY:
return Weekday::Thursday;
case UCAL_FRIDAY:
return Weekday::Friday;
case UCAL_SATURDAY:
return Weekday::Saturday;
}
VERIFY_NOT_REACHED();
}
WeekInfo week_info_of_locale(StringView locale)
{
UErrorCode status = U_ZERO_ERROR;
auto locale_data = LocaleData::for_locale(locale);
if (!locale_data.has_value())
return {};
auto calendar = adopt_own_if_nonnull(icu::Calendar::createInstance(locale_data->locale(), status));
if (icu_failure(status))
return {};
WeekInfo week_info;
week_info.minimal_days_in_first_week = calendar->getMinimalDaysInFirstWeek();
if (auto day = calendar->getFirstDayOfWeek(status); icu_success(status))
week_info.first_day_of_week = icu_calendar_day_to_weekday(day);
auto append_if_weekend = [&](auto day) {
auto type = calendar->getDayOfWeekType(day, status);
if (icu_failure(status))
return;
switch (type) {
case UCAL_WEEKEND_ONSET:
case UCAL_WEEKEND_CEASE:
case UCAL_WEEKEND:
week_info.weekend_days.append(icu_calendar_day_to_weekday(day));
break;
default:
break;
}
};
append_if_weekend(UCAL_SUNDAY);
append_if_weekend(UCAL_MONDAY);
append_if_weekend(UCAL_TUESDAY);
append_if_weekend(UCAL_WEDNESDAY);
append_if_weekend(UCAL_THURSDAY);
append_if_weekend(UCAL_FRIDAY);
append_if_weekend(UCAL_SATURDAY);
return week_info;
}
}

23
Userland/Libraries/LibLocale/DateTimeFormat.h
View file

@ -97,22 +97,6 @@ struct CalendarPattern {
Optional<CalendarPatternStyle> time_zone_name;
};
Optional<MinimumDaysRegion> minimum_days_region_from_string(StringView minimum_days_region);
Optional<u8> get_regional_minimum_days(StringView region);
Optional<u8> get_locale_minimum_days(StringView locale);
Optional<FirstDayRegion> first_day_region_from_string(StringView first_day_region);
Optional<Weekday> get_regional_first_day(StringView region);
Optional<Weekday> get_locale_first_day(StringView locale);
Optional<WeekendStartRegion> weekend_start_region_from_string(StringView weekend_start_region);
Optional<Weekday> get_regional_weekend_start(StringView region);
Optional<Weekday> get_locale_weekend_start(StringView locale);
Optional<WeekendEndRegion> weekend_end_region_from_string(StringView weekend_end_region);
Optional<Weekday> get_regional_weekend_end(StringView region);
Optional<Weekday> get_locale_weekend_end(StringView locale);
class DateTimeFormat {
public:
static NonnullOwnPtr<DateTimeFormat> create_for_date_and_time_style(
@ -148,4 +132,11 @@ protected:
DateTimeFormat() = default;
};
struct WeekInfo {
u8 minimal_days_in_first_week { 1 };
Optional<Weekday> first_day_of_week;
Vector<Weekday> weekend_days;
};
WeekInfo week_info_of_locale(StringView locale);
}

4
Userland/Libraries/LibLocale/Forward.h
View file

@ -11,7 +11,6 @@
namespace Locale {
enum class CalendarPatternStyle : u8;
enum class FirstDayRegion : u8;
enum class HourCycle : u8;
enum class Key : u8;
enum class KeywordCalendar : u8;
@ -21,13 +20,10 @@ enum class KeywordColNumeric : u8;
enum class KeywordHours : u8;
enum class KeywordNumbers : u8;
enum class Locale : u16;
enum class MinimumDaysRegion : u8;
enum class NumericSymbol : u8;
enum class PluralCategory : u8;
enum class Style : u8;
enum class Weekday : u8;
enum class WeekendEndRegion : u8;
enum class WeekendStartRegion : u8;
class NumberFormat;