From e3dbf7c5e72f19ffde1d5e28cbfdef88b3fbe77e Mon Sep 17 00:00:00 2001 From: Martok Date: Mon, 5 Jun 2023 00:18:19 +0200 Subject: [PATCH] Issue #2259 - Support Unicode extensions with multiple value subtags in BCP47 language tags Based-on: m-c 1321789 --- js/src/builtin/intl/Collator.cpp | 24 ++-- js/src/builtin/intl/CommonFunctions.js | 186 +++++++++++++++++-------- js/src/builtin/intl/DateTimeFormat.cpp | 41 ++++-- 3 files changed, 165 insertions(+), 86 deletions(-) diff --git a/js/src/builtin/intl/Collator.cpp b/js/src/builtin/intl/Collator.cpp index 69c7b2745d..ad77736471 100644 --- a/js/src/builtin/intl/Collator.cpp +++ b/js/src/builtin/intl/Collator.cpp @@ -227,6 +227,15 @@ js::intl_availableCollations(JSContext* cx, unsigned argc, Value* vp) return false; uint32_t index = 0; + + // The first element of the collations array must be |null| per + // ES2017 Intl, 10.2.3 Internal Slots. + if (!DefineElement(cx, collations, index++, NullHandleValue)) + return false; + + RootedString jscollation(cx); + RootedValue element(cx); + for (uint32_t i = 0; i < count; i++) { const char* collation = uenum_next(values, nullptr, &status); if (U_FAILURE(status)) { @@ -241,21 +250,12 @@ js::intl_availableCollations(JSContext* cx, unsigned argc, Value* vp) if (StringsAreEqual(collation, "standard") || StringsAreEqual(collation, "search")) continue; - // ICU returns old-style keyword values; map them to BCP 47 equivalents - // (see http://bugs.icu-project.org/trac/ticket/9620). - if (StringsAreEqual(collation, "dictionary")) - collation = "dict"; - else if (StringsAreEqual(collation, "gb2312han")) - collation = "gb2312"; - else if (StringsAreEqual(collation, "phonebook")) - collation = "phonebk"; - else if (StringsAreEqual(collation, "traditional")) - collation = "trad"; + // ICU returns old-style keyword values; map them to BCP 47 equivalents. + jscollation = JS_NewStringCopyZ(cx, uloc_toUnicodeLocaleType("co", collation)); - RootedString jscollation(cx, JS_NewStringCopyZ(cx, collation)); if (!jscollation) return false; - RootedValue element(cx, StringValue(jscollation)); + element = StringValue(jscollation); if (!DefineElement(cx, collations, index++, element)) return false; } diff --git a/js/src/builtin/intl/CommonFunctions.js b/js/src/builtin/intl/CommonFunctions.js index c1999f001e..59fbeaea3b 100644 --- a/js/src/builtin/intl/CommonFunctions.js +++ b/js/src/builtin/intl/CommonFunctions.js @@ -799,10 +799,7 @@ function LookupMatcher(availableLocales, requestedLocales) { if (locale !== noExtensionsLocale) { var unicodeLocaleExtensionSequenceRE = getUnicodeLocaleExtensionSequenceRE(); var extensionMatch = regexp_exec_no_statics(unicodeLocaleExtensionSequenceRE, locale); - var extension = extensionMatch[0]; - var extensionIndex = extensionMatch.index; - result.extension = extension; - result.extensionIndex = extensionIndex; + result.extension = extensionMatch[0]; } } else { result.locale = DefaultLocale(); @@ -825,6 +822,77 @@ function BestFitMatcher(availableLocales, requestedLocales) { return LookupMatcher(availableLocales, requestedLocales); } +/** + * Returns the Unicode extension value subtags for the requested key subtag. + * + * NOTE: PR to add UnicodeExtensionValue to ECMA-402 isn't yet written. + */ +function UnicodeExtensionValue(extension, key) { + assert(typeof extension === "string", "extension is a string value"); + assert(function() { + var unicodeLocaleExtensionSequenceRE = getUnicodeLocaleExtensionSequenceRE(); + var extensionMatch = regexp_exec_no_statics(unicodeLocaleExtensionSequenceRE, extension); + return extensionMatch !== null && extensionMatch[0] === extension; + }(), "extension is a Unicode extension subtag"); + assert(typeof key === "string", "key is a string value"); + assert(key.length === 2, "key is a Unicode extension key subtag"); + + // Step 1. + var size = extension.length; + + // Step 2. + var searchValue = "-" + key + "-"; + + // Step 3. + var pos = callFunction(std_String_indexOf, extension, searchValue); + + // Step 4. + if (pos !== -1) { + // Step 4.a. + var start = pos + 4; + + // Step 4.b. + var end = start; + + // Step 4.c. + var k = start; + + // Steps 4.d-e. + while (true) { + // Step 4.e.i. + var e = callFunction(std_String_indexOf, extension, "-", k); + + // Step 4.e.ii. + var len = e === -1 ? size - k : e - k; + + // Step 4.e.iii. + if (len === 2) + break; + + // Step 4.e.iv. + if (e === -1) { + end = size; + break; + } + + // Step 4.e.v. + end = e; + k = e + 1; + } + + // Step 4.f. + return callFunction(String_substring, extension, start, end); + } + + // Step 5. + searchValue = "-" + key; + + // Steps 6-7. + if (callFunction(std_String_endsWith, extension, searchValue)) + return ""; + + // Step 8 (implicit). +} /** * Compares a BCP 47 language priority list against availableLocales and @@ -847,19 +915,8 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte // Step 4. var foundLocale = r.locale; - // Step 5.a. + // Step 5 (Not applicable in this implementation). var extension = r.extension; - var extensionIndex, extensionSubtags, extensionSubtagsLength; - - // Step 5. - if (extension !== undefined) { - // Step 5.b. - extensionIndex = r.extensionIndex; - - // Steps 5.d-e. - extensionSubtags = StringSplitString(ToString(extension), "-"); - extensionSubtagsLength = extensionSubtags.length; - } // Steps 6-7. var result = new Record(); @@ -868,11 +925,11 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte // Step 8. var supportedExtension = "-u"; - // Steps 9-11. + // Steps 9-12. var i = 0; var len = relevantExtensionKeys.length; while (i < len) { - // Steps 11.a-c. + // Steps 12.a-c. var key = relevantExtensionKeys[i]; // In this implementation, localeData is a function, not an object. @@ -880,51 +937,41 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte var keyLocaleData = foundLocaleData[key]; // Locale data provides default value. - // Step 11.d. + // Step 12.d. var value = keyLocaleData[0]; + assert(typeof value === "string" || value === null, "unexpected locale data value"); // Locale tag may override. - // Step 11.e. + // Step 12.e. var supportedExtensionAddition = ""; - // Step 11.f is implemented by Utilities.js. + // Step 12.f. + if (extension !== undefined) { + // NB: The step annotations don't yet match the ES2017 Intl draft, + // 94045d234762ad107a3d09bb6f7381a65f1a2f9b, because the PR to add + // the new UnicodeExtensionValue abstract operation still needs to + // be written. - var valuePos; + // Step 12.f.i. + var requestedValue = UnicodeExtensionValue(extension, key); - // Step 11.g. - if (extensionSubtags !== undefined) { - // Step 11.g.i. - var keyPos = callFunction(ArrayIndexOf, extensionSubtags, key); - - // Step 11.g.ii. - if (keyPos !== -1) { - // Step 11.g.ii.1. - if (keyPos + 1 < extensionSubtagsLength && - extensionSubtags[keyPos + 1].length > 2) - { - // Step 11.g.ii.1.a. - var requestedValue = extensionSubtags[keyPos + 1]; - - // Step 11.g.ii.1.b. - valuePos = callFunction(ArrayIndexOf, keyLocaleData, requestedValue); - - // Step 11.g.ii.1.c. - if (valuePos !== -1) { + // Step 12.f.ii. + if (requestedValue !== undefined) { + // Step 12.f.ii.1. + if (requestedValue !== "") { + // Step 12.f.ii.1.a. + if (callFunction(ArrayIndexOf, keyLocaleData, requestedValue) !== -1) { value = requestedValue; supportedExtensionAddition = "-" + key + "-" + value; } } else { - // Step 11.g.ii.2. + // Step 12.f.ii.2. // According to the LDML spec, if there's no type value, // and true is an allowed value, it's used. - // Step 11.g.ii.2.a. - valuePos = callFunction(ArrayIndexOf, keyLocaleData, "true"); - - // Step 11.g.ii.2.b. - if (valuePos !== -1) + if (callFunction(ArrayIndexOf, keyLocaleData, "true") !== -1) value = "true"; } } @@ -932,35 +979,54 @@ function ResolveLocale(availableLocales, requestedLocales, options, relevantExte // Options override all. - // Step 11.h.i. + // Step 12.g.i. var optionsValue = options[key]; - // Step 11.h, 11.h.ii. + // Step 12.g, 12.gg.ii. if (optionsValue !== undefined && + optionsValue !== value && callFunction(ArrayIndexOf, keyLocaleData, optionsValue) !== -1) { - // Step 11.h.ii.1. - if (optionsValue !== value) { - value = optionsValue; - supportedExtensionAddition = ""; - } + value = optionsValue; + supportedExtensionAddition = ""; } - // Steps 11.i-k. + // Steps 12.h-j. result[key] = value; supportedExtension += supportedExtensionAddition; i++; } - // Step 12. + // Step 13. if (supportedExtension.length > 2) { - var preExtension = callFunction(String_substring, foundLocale, 0, extensionIndex); - var postExtension = callFunction(String_substring, foundLocale, extensionIndex); - foundLocale = preExtension + supportedExtension + postExtension; + assert(!callFunction(std_String_startsWith, foundLocale, "x-"), + "unexpected privateuse-only locale returned from ICU"); + + // Step 13.a. + var privateIndex = callFunction(std_String_indexOf, foundLocale, "-x-"); + + // Steps 13.b-c. + if (privateIndex === -1) { + foundLocale += supportedExtension; + } else { + var preExtension = callFunction(String_substring, foundLocale, 0, privateIndex); + var postExtension = callFunction(String_substring, foundLocale, privateIndex); + foundLocale = preExtension + supportedExtension + postExtension; + } + + // Step 13.d. + assert(IsStructurallyValidLanguageTag(foundLocale), "invalid locale after concatenation"); + + // Step 13.e (Not required in this implementation, because we don't + // canonicalize Unicode extension subtags). + assert(foundLocale === CanonicalizeLanguageTag(foundLocale), "same locale with extension"); + } - // Steps 13-14. + // Step 14. result.locale = foundLocale; + + // Step 15. return result; } diff --git a/js/src/builtin/intl/DateTimeFormat.cpp b/js/src/builtin/intl/DateTimeFormat.cpp index cf0db19dc4..fed655736a 100644 --- a/js/src/builtin/intl/DateTimeFormat.cpp +++ b/js/src/builtin/intl/DateTimeFormat.cpp @@ -241,19 +241,16 @@ js::intl_DateTimeFormat_availableLocales(JSContext* cx, unsigned argc, Value* vp return true; } -// ICU returns old-style keyword values; map them to BCP 47 equivalents -// (see http://bugs.icu-project.org/trac/ticket/9620). -static const char* -bcp47CalendarName(const char* icuName) +struct CalendarAlias { - if (StringsAreEqual(icuName, "ethiopic-amete-alem")) - return "ethioaa"; - if (StringsAreEqual(icuName, "gregorian")) - return "gregory"; - if (StringsAreEqual(icuName, "islamic-civil")) - return "islamicc"; - return icuName; -} + const char* const calendar; + const char* const alias; +}; + +const CalendarAlias calendarAliases[] = { + { "islamic-civil", "islamicc" }, + { "ethioaa", "ethiopic-amete-alem" } +}; bool js::intl_availableCalendars(JSContext* cx, unsigned argc, Value* vp) @@ -286,7 +283,8 @@ js::intl_availableCalendars(JSContext* cx, unsigned argc, Value* vp) return false; } - jscalendar = JS_NewStringCopyZ(cx, bcp47CalendarName(calendar)); + // ICU returns old-style keyword values; map them to BCP 47 equivalents + jscalendar = JS_NewStringCopyZ(cx, uloc_toUnicodeLocaleType("ca", calendar)); if (!jscalendar) return false; } @@ -316,12 +314,27 @@ js::intl_availableCalendars(JSContext* cx, unsigned argc, Value* vp) return false; } - jscalendar = JS_NewStringCopyZ(cx, bcp47CalendarName(calendar)); + // ICU returns old-style keyword values; map them to BCP 47 equivalents + calendar = uloc_toUnicodeLocaleType("ca", calendar); + + jscalendar = JS_NewStringCopyZ(cx, calendar); if (!jscalendar) return false; element = StringValue(jscalendar); if (!DefineElement(cx, calendars, index++, element)) return false; + + // ICU doesn't return calendar aliases, append them here. + for (const auto& calendarAlias : calendarAliases) { + if (StringsAreEqual(calendar, calendarAlias.calendar)) { + jscalendar = JS_NewStringCopyZ(cx, calendarAlias.alias); + if (!jscalendar) + return false; + element = StringValue(jscalendar); + if (!DefineElement(cx, calendars, index++, element)) + return false; + } + } } args.rval().setObject(*calendars);