wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-10-23 07:37:29 +09:00
Code Issues Projects Releases Packages Wiki Activity

Issue #2259 - Performance improvements for LanguageTag parsing

Browse source 
- parsing: dont normalise things that don't need to be normalised anymore:
  extension, private-use, variant, language, script, and region subtags
- Add missing() and present() methods to LanguageSubtag
- Change mozilla::Range to mozilla::Span for slightly better code

Based-on: m-c 1592588
This commit is contained in:
Martok 2023-06-18 15:05:33 +02:00 • committed by roytam1
commit 0b3079d8e6
8 changed files with 505 additions and 503 deletions
Download patch file Download diff file Expand all files Collapse all files

5
js/src/builtin/intl/IntlObject.cpp
View file

@ -545,10 +545,7 @@ js::intl_BestAvailableLocale(JSContext* cx, unsigned argc, Value* vp)
JS_TRY_VAR_OR_RETURN_FALSE(cx, ok, intl::LanguageTagParser::tryParse(cx, locale, tag)); JS_TRY_VAR_OR_RETURN_FALSE(cx, ok, intl::LanguageTagParser::tryParse(cx, locale, tag));
MOZ_ASSERT(ok, "locale is a structurally valid language tag"); MOZ_ASSERT(ok, "locale is a structurally valid language tag");
auto isUnicodeExtension = [](const auto& extension) { MOZ_ASSERT(!tag.unicodeExtension(),
return extension[0] == 'u';
};
MOZ_ASSERT(std::none_of(tag.extensions().begin(), tag.extensions().end(), isUnicodeExtension),
"locale must contain no Unicode extensions"); "locale must contain no Unicode extensions");
if (!tag.canonicalize(cx, intl::LanguageTag::UnicodeExtensionCanonicalForm::No)) { if (!tag.canonicalize(cx, intl::LanguageTag::UnicodeExtensionCanonicalForm::No)) {

544
js/src/builtin/intl/LanguageTag.cpp
View file

@ -8,7 +8,7 @@
#include "mozilla/Assertions.h" #include "mozilla/Assertions.h"
#include "mozilla/MathAlgorithms.h" #include "mozilla/MathAlgorithms.h"
#include "mozilla/Range.h" #include "mozilla/Span.h"
#include "mozilla/TextUtils.h" #include "mozilla/TextUtils.h"
#include "mozilla/Variant.h" #include "mozilla/Variant.h"
@ -40,102 +40,93 @@ namespace intl {
using namespace js::intl::LanguageTagLimits; using namespace js::intl::LanguageTagLimits;
using ConstCharRange = mozilla::Range<const char>;
#ifdef DEBUG
template <typename CharT> template <typename CharT>
bool IsStructurallyValidLanguageTag( bool IsStructurallyValidLanguageTag(mozilla::Span<const CharT> language) {
const mozilla::Range<const CharT>& language) {
// Tell the analysis the |std::all_of| function can't GC. // Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc; JS::AutoSuppressGCAnalysis nogc;
// unicode_language_subtag = alpha{2,3} | alpha{5,8}; // unicode_language_subtag = alpha{2,3} | alpha{5,8};
size_t length = language.length(); size_t length = language.size();
const CharT* str = language.begin().get(); const CharT* str = language.data();
return ((2 <= length && length <= 3) || (5 <= length && length <= 8)) && return ((2 <= length && length <= 3) || (5 <= length && length <= 8)) &&
std::all_of(str, str + length, mozilla::IsAsciiLowercaseAlpha<CharT>); std::all_of(str, str + length, mozilla::IsAsciiAlpha<CharT>);
} }
template bool IsStructurallyValidLanguageTag( template bool IsStructurallyValidLanguageTag(
const mozilla::Range<const Latin1Char>& language); mozilla::Span<const char> language);
template bool IsStructurallyValidLanguageTag( template bool IsStructurallyValidLanguageTag(
const mozilla::Range<const char16_t>& language); mozilla::Span<const Latin1Char> language);
template bool IsStructurallyValidLanguageTag(
mozilla::Span<const char16_t> language);
template <typename CharT> template <typename CharT>
bool IsStructurallyValidScriptTag(const mozilla::Range<const CharT>& script) { bool IsStructurallyValidScriptTag(mozilla::Span<const CharT> script) {
// Tell the analysis the |std::all_of| function can't GC. // Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc; JS::AutoSuppressGCAnalysis nogc;
// unicode_script_subtag = alpha{4} ; // unicode_script_subtag = alpha{4} ;
size_t length = script.length(); size_t length = script.size();
const CharT* str = script.begin().get(); const CharT* str = script.data();
return length == 4 && mozilla::IsAsciiUppercaseAlpha<CharT>(str[0]) && return length == 4 &&
std::all_of(str + 1, str + length, std::all_of(str, str + length, mozilla::IsAsciiAlpha<CharT>);
mozilla::IsAsciiLowercaseAlpha<CharT>);
} }
template bool IsStructurallyValidScriptTag( template bool IsStructurallyValidScriptTag(
const mozilla::Range<const Latin1Char>& script); mozilla::Span<const char> script);
template bool IsStructurallyValidScriptTag( template bool IsStructurallyValidScriptTag(
const mozilla::Range<const char16_t>& script); mozilla::Span<const Latin1Char> script);
template bool IsStructurallyValidScriptTag(
mozilla::Span<const char16_t> script);
template <typename CharT> template <typename CharT>
bool IsStructurallyValidRegionTag(const mozilla::Range<const CharT>& region) { bool IsStructurallyValidRegionTag(mozilla::Span<const CharT> region) {
// Tell the analysis the |std::all_of| function can't GC. // Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc; JS::AutoSuppressGCAnalysis nogc;
// unicode_region_subtag = (alpha{2} | digit{3}) ; // unicode_region_subtag = (alpha{2} | digit{3}) ;
size_t length = region.length(); size_t length = region.size();
const CharT* str = region.begin().get(); const CharT* str = region.data();
return (length == 2 && std::all_of(str, str + length, return (length == 2 &&
mozilla::IsAsciiUppercaseAlpha<CharT>)) || std::all_of(str, str + length, mozilla::IsAsciiAlpha<CharT>)) ||
(length == 3 && (length == 3 &&
std::all_of(str, str + length, mozilla::IsAsciiDigit<CharT>)); std::all_of(str, str + length, mozilla::IsAsciiDigit<CharT>));
} }
template bool IsStructurallyValidRegionTag( template bool IsStructurallyValidRegionTag(
const mozilla::Range<const Latin1Char>& region); mozilla::Span<const char> region);
template bool IsStructurallyValidRegionTag( template bool IsStructurallyValidRegionTag(
const mozilla::Range<const char16_t>& region); mozilla::Span<const Latin1Char> region);
template bool IsStructurallyValidRegionTag(
mozilla::Span<const char16_t> region);
bool IsStructurallyValidVariantTag(const ConstCharRange& variant) { #ifdef DEBUG
bool IsStructurallyValidVariantTag(mozilla::Span<const char> variant) {
// unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3}) ; // unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3}) ;
auto isAsciiLowercaseAlphanumeric = [](char c) { size_t length = variant.size();
return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c); const char* str = variant.data();
};
size_t length = variant.length();
const char* str = variant.begin().get();
return ((5 <= length && length <= 8) || return ((5 <= length && length <= 8) ||
(length == 4 && mozilla::IsAsciiDigit(str[0]))) && (length == 4 && mozilla::IsAsciiDigit(str[0]))) &&
std::all_of(str, str + length, isAsciiLowercaseAlphanumeric); std::all_of(str, str + length, mozilla::IsAsciiAlphanumeric<char>);
} }
bool IsStructurallyValidUnicodeExtensionTag(const ConstCharRange& extension) { bool IsStructurallyValidUnicodeExtensionTag(
auto isAsciiLowercaseAlphanumericOrDash = [](char c) { mozilla::Span<const char> extension) {
return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c) || return LanguageTagParser::canParseUnicodeExtension(extension);
c == '-';
};
size_t length = extension.length();
const char* str = extension.begin().get();
return LanguageTagParser::canParseUnicodeExtension(extension) &&
std::all_of(str, str + length, isAsciiLowercaseAlphanumericOrDash);
} }
static bool IsStructurallyValidExtensionTag(const ConstCharRange& extension) { static bool IsStructurallyValidExtensionTag(
mozilla::Span<const char> extension) {
// other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ; // other_extensions = sep [alphanum-[tTuUxX]] (sep alphanum{2,8})+ ;
// NB: Allow any extension, including Unicode and Transform here, because // NB: Allow any extension, including Unicode and Transform here, because
// this function is only used for an assertion. // this function is only used for an assertion.
auto isAsciiDigitOrLowercaseAlpha = [](char c) {
return mozilla::IsAsciiDigit(c) || mozilla::IsAsciiLowercaseAlpha(c);
};
size_t length = extension.length(); size_t length = extension.size();
const char* str = extension.begin().get(); const char* str = extension.data();
const char* const end = extension.data() + length;
if (length <= 2) { if (length <= 2) {
return false; return false;
} }
if (!isAsciiDigitOrLowercaseAlpha(str[0]) || str[0] == 'x') { if (!mozilla::IsAsciiAlphanumeric(str[0]) || str[0] == 'x' || str[0] == 'X') {
return false; return false;
} }
str++; str++;
@ -143,11 +134,11 @@ static bool IsStructurallyValidExtensionTag(const ConstCharRange& extension) {
return false; return false;
} }
while (true) { while (true) {
const char* sep = reinterpret_cast<const char*>( const char* sep =
memchr(str, '-', extension.end().get() - str)); reinterpret_cast<const char*>(memchr(str, '-', end - str));
size_t len = (sep ? sep : extension.end().get()) - str; size_t len = (sep ? sep : end) - str;
if (len < 2 || len > 8 || if (len < 2 || len > 8 ||
!std::all_of(str, str + len, isAsciiDigitOrLowercaseAlpha)) { !std::all_of(str, str + len, mozilla::IsAsciiAlphanumeric<char>)) {
return false; return false;
} }
if (!sep) { if (!sep) {
@ -157,23 +148,28 @@ static bool IsStructurallyValidExtensionTag(const ConstCharRange& extension) {
} }
} }
bool IsStructurallyValidPrivateUseTag(const ConstCharRange& privateUse) { bool IsStructurallyValidPrivateUseTag(mozilla::Span<const char> privateUse) {
// pu_extensions = sep [xX] (sep alphanum{1,8})+ ; // pu_extensions = sep [xX] (sep alphanum{1,8})+ ;
auto isAsciiDigitOrLowercaseAlpha = [](char c) {
return mozilla::IsAsciiDigit(c) || mozilla::IsAsciiLowercaseAlpha(c);
};
size_t length = privateUse.length(); size_t length = privateUse.size();
const char* str = privateUse.begin().get(); const char* str = privateUse.data();
if (length <= 2 || *str++ != 'x' || *str++ != '-') { const char* const end = privateUse.data() + length;
if (length <= 2) {
return false;
}
if (str[0] != 'x' && str[0] != 'X') {
return false;
}
str++;
if (*str++ != '-') {
return false; return false;
} }
while (true) { while (true) {
const char* sep = reinterpret_cast<const char*>( const char* sep =
memchr(str, '-', privateUse.end().get() - str)); reinterpret_cast<const char*>(memchr(str, '-', end - str));
size_t len = (sep ? sep : privateUse.end().get()) - str; size_t len = (sep ? sep : end) - str;
if (len == 0 || len > 8 || if (len == 0 || len > 8 ||
!std::all_of(str, str + len, isAsciiDigitOrLowercaseAlpha)) { !std::all_of(str, str + len, mozilla::IsAsciiAlphanumeric<char>)) {
return false; return false;
} }
if (!sep) { if (!sep) {
@ -184,15 +180,33 @@ bool IsStructurallyValidPrivateUseTag(const ConstCharRange& privateUse) {
} }
#endif #endif
ptrdiff_t LanguageTag::unicodeExtensionIndex() const {
// The extension subtags aren't necessarily sorted, so we can't use binary
// search here.
auto p = std::find_if(
extensions().begin(), extensions().end(),
[](const auto& ext) { return ext[0] == 'u' || ext[0] == 'U'; });
if (p != extensions().end()) {
return std::distance(extensions().begin(), p);
}
return -1;
}
const char* LanguageTag::unicodeExtension() const {
ptrdiff_t index = unicodeExtensionIndex();
if (index >= 0) {
return extensions()[index].get();
}
return nullptr;
}
bool LanguageTag::setUnicodeExtension(UniqueChars extension) { bool LanguageTag::setUnicodeExtension(UniqueChars extension) {
MOZ_ASSERT(IsStructurallyValidUnicodeExtensionTag( MOZ_ASSERT(IsStructurallyValidUnicodeExtensionTag(
{extension.get(), strlen(extension.get())})); mozilla::MakeCStringSpan(extension.get())));
// Replace the existing Unicode extension subtag or append a new one. // Replace the existing Unicode extension subtag or append a new one.
auto p = std::find_if(extensions().begin(), extensions().end(), ptrdiff_t index = unicodeExtensionIndex();
[](const auto& ext) { return ext[0] == 'u'; }); if (index >= 0) {
if (p != extensions().end()) {
size_t index = std::distance(extensions().begin(), p);
extensions_[index] = std::move(extension); extensions_[index] = std::move(extension);
return true; return true;
} }
@ -200,10 +214,8 @@ bool LanguageTag::setUnicodeExtension(UniqueChars extension) {
} }
void LanguageTag::clearUnicodeExtension() { void LanguageTag::clearUnicodeExtension() {
auto p = std::find_if(extensions().begin(), extensions().end(), ptrdiff_t index = unicodeExtensionIndex();
[](const auto& ext) { return ext[0] == 'u'; }); if (index >= 0) {
if (p != extensions().end()) {
size_t index = std::distance(extensions().begin(), p);
extensions_.erase(extensions_.begin() + index); extensions_.erase(extensions_.begin() + index);
} }
} }
@ -252,35 +264,53 @@ bool LanguageTag::canonicalizeBaseName(JSContext* cx) {
// normalizing the case and ordering all subtags. The canonical syntax form // normalizing the case and ordering all subtags. The canonical syntax form
// itself is specified in UTS 35, 3.2.1. // itself is specified in UTS 35, 3.2.1.
// The |LanguageTag| fields are already in normalized case, so we can skip // Language codes need to be in lower case. "JA" -> "ja"
// this step. language_.toLowerCase();
MOZ_ASSERT(IsStructurallyValidLanguageTag(language().range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language().span()));
MOZ_ASSERT(script().length() == 0 ||
IsStructurallyValidScriptTag(script().range()));
MOZ_ASSERT(region().length() == 0 ||
IsStructurallyValidRegionTag(region().range()));
#ifdef DEBUG
auto validVariant = [](const auto& variant) {
const char* str = variant.get();
return IsStructurallyValidVariantTag({str, strlen(str)});
};
MOZ_ASSERT(std::all_of(variants().begin(), variants().end(), validVariant));
auto validExtension = [](const auto& extension) { // The first character of a script code needs to be capitalized.
const char* str = extension.get(); // "hans" -> "Hans"
return IsStructurallyValidExtensionTag({str, strlen(str)}); script_.toTitleCase();
}; MOZ_ASSERT(script().missing() ||
MOZ_ASSERT( IsStructurallyValidScriptTag(script().span()));
std::all_of(extensions().begin(), extensions().end(), validExtension));
#endif // Region codes need to be in upper case. "bu" -> "BU"
MOZ_ASSERT(!privateuse() || IsStructurallyValidPrivateUseTag( region_.toUpperCase();
{privateuse(), strlen(privateuse())})); MOZ_ASSERT(region().missing() ||
IsStructurallyValidRegionTag(region().span()));
// The canonical case for variant subtags is lowercase.
for (UniqueChars& variant : variants_) {
char* variantChars = variant.get();
size_t variantLength = strlen(variantChars);
AsciiToLowerCase(variantChars, variantLength, variantChars);
MOZ_ASSERT(IsStructurallyValidVariantTag({variantChars, variantLength}));
}
// Extensions and privateuse subtags are case normalized in the
// |canonicalizeExtensions| method.
// The second step in UTS 35, 3.2.1, is to order all subtags. // The second step in UTS 35, 3.2.1, is to order all subtags.
// 1. Any variants are in alphabetical order. if (variants_.length() > 1) {
if (!SortAlphabetically(cx, variants_)) { // 1. Any variants are in alphabetical order.
return false; if (!SortAlphabetically(cx, variants_)) {
return false;
}
// Reject the Locale identifier if a duplicate variant was found, e.g.
// "en-variant-Variant".
const UniqueChars* duplicate = std::adjacent_find(
variants().begin(), variants().end(), [](const auto& a, const auto& b) {
return strcmp(a.get(), b.get()) == 0;
});
if (duplicate != variants().end()) {
JS_ReportErrorNumberASCII(cx, GetErrorMessage, nullptr,
JSMSG_DUPLICATE_VARIANT_SUBTAG,
duplicate->get());
return false;
}
} }
// 2. Any extensions are in alphabetical order by their singleton. // 2. Any extensions are in alphabetical order by their singleton.
@ -301,7 +331,7 @@ bool LanguageTag::canonicalizeBaseName(JSContext* cx) {
// No script replacements are currently present. // No script replacements are currently present.
// Replace deprecated region subtags with their preferred values. // Replace deprecated region subtags with their preferred values.
if (region().length() > 0) { if (region().present()) {
if (!regionMapping(region_) && complexRegionMapping(region_)) { if (!regionMapping(region_) && complexRegionMapping(region_)) {
performComplexRegionMappings(); performComplexRegionMappings();
} }
@ -320,6 +350,16 @@ bool LanguageTag::canonicalizeBaseName(JSContext* cx) {
bool LanguageTag::canonicalizeExtensions( bool LanguageTag::canonicalizeExtensions(
JSContext* cx, UnicodeExtensionCanonicalForm canonicalForm) { JSContext* cx, UnicodeExtensionCanonicalForm canonicalForm) {
// The canonical case for all extension subtags is lowercase.
for (UniqueChars& extension : extensions_) {
char* extensionChars = extension.get();
size_t extensionLength = strlen(extensionChars);
AsciiToLowerCase(extensionChars, extensionLength, extensionChars);
MOZ_ASSERT(
IsStructurallyValidExtensionTag({extensionChars, extensionLength}));
}
// Any extensions are in alphabetical order by their singleton. // Any extensions are in alphabetical order by their singleton.
// "u-ca-chinese-t-zh-latn" -> "t-zh-latn-u-ca-chinese" // "u-ca-chinese-t-zh-latn" -> "t-zh-latn-u-ca-chinese"
if (!SortAlphabetically(cx, extensions_)) { if (!SortAlphabetically(cx, extensions_)) {
@ -337,6 +377,15 @@ bool LanguageTag::canonicalizeExtensions(
} }
} }
} }
// The canonical case for privateuse subtags is lowercase.
if (char* privateuse = privateuse_.get()) {
size_t privateuseLength = strlen(privateuse);
AsciiToLowerCase(privateuse, privateuseLength, privateuse);
MOZ_ASSERT(
IsStructurallyValidPrivateUseTag({privateuse, privateuseLength}));
}
return true; return true;
} }
@ -362,7 +411,8 @@ bool LanguageTag::canonicalizeUnicodeExtension(
const char* const extension = unicodeExtension.get(); const char* const extension = unicodeExtension.get();
MOZ_ASSERT(extension[0] == 'u'); MOZ_ASSERT(extension[0] == 'u');
MOZ_ASSERT(extension[1] == '-'); MOZ_ASSERT(extension[1] == '-');
MOZ_ASSERT(IsStructurallyValidExtensionTag({extension, strlen(extension)})); MOZ_ASSERT(
IsStructurallyValidExtensionTag(mozilla::MakeCStringSpan(extension)));
size_t length = strlen(extension); size_t length = strlen(extension);
@ -376,7 +426,7 @@ bool LanguageTag::canonicalizeUnicodeExtension(
JS_TRY_VAR_OR_RETURN_FALSE( JS_TRY_VAR_OR_RETURN_FALSE(
cx, ok, cx, ok,
LanguageTagParser::parseUnicodeExtension( LanguageTagParser::parseUnicodeExtension(
cx, ConstCharRange(extension, length), attributes, keywords)); cx, mozilla::MakeSpan(extension, length), attributes, keywords));
MOZ_ASSERT(ok, "unexpected invalid Unicode extension subtag"); MOZ_ASSERT(ok, "unexpected invalid Unicode extension subtag");
auto attributesLessOrEqual = [extension](const Attribute& a, auto attributesLessOrEqual = [extension](const Attribute& a,
@ -475,12 +525,13 @@ bool LanguageTag::canonicalizeUnicodeExtension(
static constexpr size_t UnicodeKeyWithSepLength = UnicodeKeyLength + 1; static constexpr size_t UnicodeKeyWithSepLength = UnicodeKeyLength + 1;
static auto isTrue = [](const ConstCharRange& type) { using StringSpan = mozilla::Span<const char>;
static auto isTrue = [](StringSpan type) {
constexpr char True[] = "true"; constexpr char True[] = "true";
const size_t TrueLength = strlen(True); const size_t TrueLength = strlen(True);
return type.length() == TrueLength && return type.size() == TrueLength &&
std::char_traits<char>::compare(type.begin().get(), True, std::char_traits<char>::compare(type.data(), True, TrueLength) == 0;
TrueLength) == 0;
}; };
auto appendKey = [&sb, extension](const Keyword& keyword) { auto appendKey = [&sb, extension](const Keyword& keyword) {
@ -489,7 +540,7 @@ bool LanguageTag::canonicalizeUnicodeExtension(
}; };
auto appendKeyword = [&sb, extension](const Keyword& keyword, auto appendKeyword = [&sb, extension](const Keyword& keyword,
const ConstCharRange& type) { StringSpan type) {
MOZ_ASSERT(keyword.length() > UnicodeKeyLength); MOZ_ASSERT(keyword.length() > UnicodeKeyLength);
// Elide the Unicode extension type "true". // Elide the Unicode extension type "true".
@ -501,7 +552,7 @@ bool LanguageTag::canonicalizeUnicodeExtension(
}; };
auto appendReplacement = [&sb, extension](const Keyword& keyword, auto appendReplacement = [&sb, extension](const Keyword& keyword,
const ConstCharRange& replacement) { StringSpan replacement) {
MOZ_ASSERT(keyword.length() > UnicodeKeyLength); MOZ_ASSERT(keyword.length() > UnicodeKeyLength);
// Elide the type "true" if present in the replacement. // Elide the type "true" if present in the replacement.
@ -511,7 +562,7 @@ bool LanguageTag::canonicalizeUnicodeExtension(
// Otherwise append the Unicode key (including the separator) and the // Otherwise append the Unicode key (including the separator) and the
// replaced type. // replaced type.
return sb.append(keyword.begin(extension), UnicodeKeyWithSepLength) && return sb.append(keyword.begin(extension), UnicodeKeyWithSepLength) &&
sb.append(replacement.begin().get(), replacement.length()); sb.append(replacement.data(), replacement.size());
}; };
// Append all Unicode extension keywords. // Append all Unicode extension keywords.
@ -539,15 +590,15 @@ bool LanguageTag::canonicalizeUnicodeExtension(
return false; return false;
} }
} else { } else {
ConstCharRange key(keyword.begin(extension), UnicodeKeyLength); StringSpan key(keyword.begin(extension), UnicodeKeyLength);
ConstCharRange type(keyword.begin(extension) + UnicodeKeyWithSepLength, StringSpan type(keyword.begin(extension) + UnicodeKeyWithSepLength,
keyword.length() - UnicodeKeyWithSepLength); keyword.length() - UnicodeKeyWithSepLength);
if (canonicalForm == UnicodeExtensionCanonicalForm::Yes) { if (canonicalForm == UnicodeExtensionCanonicalForm::Yes) {
// Search if there's a replacement for the current Unicode keyword. // Search if there's a replacement for the current Unicode keyword.
if (const char* replacement = replaceUnicodeExtensionType(key, type)) { if (const char* replacement = replaceUnicodeExtensionType(key, type)) {
if (!appendReplacement( if (!appendReplacement(keyword,
keyword, ConstCharRange(replacement, strlen(replacement)))) { mozilla::MakeCStringSpan(replacement))) {
return false; return false;
} }
} else { } else {
@ -584,9 +635,9 @@ template <class Buffer>
static bool LanguageTagToString(JSContext* cx, const LanguageTag& tag, static bool LanguageTagToString(JSContext* cx, const LanguageTag& tag,
Buffer& sb) { Buffer& sb) {
auto appendSubtag = [&sb](const auto& subtag) { auto appendSubtag = [&sb](const auto& subtag) {
auto range = subtag.range(); auto span = subtag.span();
MOZ_ASSERT(range.length() > 0); MOZ_ASSERT(span.size() > 0);
return sb.append(range.begin().get(), range.length()); return sb.append(span.data(), span.size());
}; };
auto appendSubtagZ = [&sb](const char* subtag) { auto appendSubtagZ = [&sb](const char* subtag) {
@ -609,14 +660,14 @@ static bool LanguageTagToString(JSContext* cx, const LanguageTag& tag,
} }
// Append the script subtag if present. // Append the script subtag if present.
if (tag.script().length() > 0) { if (tag.script().present()) {
if (!sb.append('-') || !appendSubtag(tag.script())) { if (!sb.append('-') || !appendSubtag(tag.script())) {
return false; return false;
} }
} }
// Append the region subtag if present. // Append the region subtag if present.
if (tag.region().length() > 0) { if (tag.region().present()) {
if (!sb.append('-') || !appendSubtag(tag.region())) { if (!sb.append('-') || !appendSubtag(tag.region())) {
return false; return false;
} }
@ -661,7 +712,8 @@ bool LanguageTag::canonicalizeTransformExtension(
const char* const extension = transformExtension.get(); const char* const extension = transformExtension.get();
MOZ_ASSERT(extension[0] == 't'); MOZ_ASSERT(extension[0] == 't');
MOZ_ASSERT(extension[1] == '-'); MOZ_ASSERT(extension[1] == '-');
MOZ_ASSERT(IsStructurallyValidExtensionTag({extension, strlen(extension)})); MOZ_ASSERT(
IsStructurallyValidExtensionTag(mozilla::MakeCStringSpan(extension)));
size_t length = strlen(extension); size_t length = strlen(extension);
@ -674,7 +726,7 @@ bool LanguageTag::canonicalizeTransformExtension(
JS_TRY_VAR_OR_RETURN_FALSE( JS_TRY_VAR_OR_RETURN_FALSE(
cx, ok, cx, ok,
LanguageTagParser::parseTransformExtension( LanguageTagParser::parseTransformExtension(
cx, ConstCharRange(extension, length), tag, fields)); cx, mozilla::MakeSpan(extension, length), tag, fields));
MOZ_ASSERT(ok, "unexpected invalid transform extension subtag"); MOZ_ASSERT(ok, "unexpected invalid transform extension subtag");
auto tfieldLessOrEqual = [extension](const TField& a, const TField& b) { auto tfieldLessOrEqual = [extension](const TField& a, const TField& b) {
@ -720,7 +772,7 @@ bool LanguageTag::canonicalizeTransformExtension(
// [1] https://unicode.org/reports/tr35/#Language_Tag_to_Locale_Identifier // [1] https://unicode.org/reports/tr35/#Language_Tag_to_Locale_Identifier
// [2] https://unicode.org/reports/tr35/#Canonical_Unicode_Locale_Identifiers // [2] https://unicode.org/reports/tr35/#Canonical_Unicode_Locale_Identifiers
// [3] https://github.com/tc39/ecma402/issues/330 // [3] https://github.com/tc39/ecma402/issues/330
if (tag.language().length() > 0) { if (tag.language().present()) {
if (!sb.append('-')) { if (!sb.append('-')) {
return false; return false;
} }
@ -786,14 +838,14 @@ static bool HasLikelySubtags(LikelySubtags likelySubtags,
// used. // used.
if (likelySubtags == LikelySubtags::Add) { if (likelySubtags == LikelySubtags::Add) {
return !tag.language().equalTo("und") && return !tag.language().equalTo("und") &&
(tag.script().length() > 0 && !tag.script().equalTo("Zzzz")) && (tag.script().present() && !tag.script().equalTo("Zzzz")) &&
(tag.region().length() > 0 && !tag.region().equalTo("ZZ")); (tag.region().present() && !tag.region().equalTo("ZZ"));
} }
// The language tag is already minimized if it only contains a language // The language tag is already minimized if it only contains a language
// subtag whose value is not the placeholder value "und". // subtag whose value is not the placeholder value "und".
return !tag.language().equalTo("und") && tag.script().length() == 0 && return !tag.language().equalTo("und") && tag.script().missing() &&
tag.region().length() == 0; tag.region().missing();
} }
// Create an ICU locale ID from the given language tag. // Create an ICU locale ID from the given language tag.
@ -802,9 +854,9 @@ static bool CreateLocaleForLikelySubtags(const LanguageTag& tag,
MOZ_ASSERT(locale.length() == 0); MOZ_ASSERT(locale.length() == 0);
auto appendSubtag = [&locale](const auto& subtag) { auto appendSubtag = [&locale](const auto& subtag) {
auto range = subtag.range(); auto span = subtag.span();
MOZ_ASSERT(range.length() > 0); MOZ_ASSERT(span.size() > 0);
return locale.append(range.begin().get(), range.length()); return locale.append(span.data(), span.size());
}; };
// Append the language subtag. // Append the language subtag.
@ -813,14 +865,14 @@ static bool CreateLocaleForLikelySubtags(const LanguageTag& tag,
} }
// Append the script subtag if present. // Append the script subtag if present.
if (tag.script().length() > 0) { if (tag.script().present()) {
if (!locale.append('_') || !appendSubtag(tag.script())) { if (!locale.append('_') || !appendSubtag(tag.script())) {
return false; return false;
} }
} }
// Append the region subtag if present. // Append the region subtag if present.
if (tag.region().length() > 0) { if (tag.region().present()) {
if (!locale.append('_') || !appendSubtag(tag.region())) { if (!locale.append('_') || !appendSubtag(tag.region())) {
return false; return false;
} }
@ -857,12 +909,12 @@ static bool AssignFromLocaleId(JSContext* cx, LocaleId& localeId,
memmove(localeId.begin(), und, length); memmove(localeId.begin(), und, length);
} }
ConstCharRange localeRange(localeId.begin(), localeId.length() - 1); mozilla::Span<const char> localeSpan(localeId.begin(), localeId.length() - 1);
// Retrieve the language, script, and region subtags from the locale ID, but // Retrieve the language, script, and region subtags from the locale ID, but
// ignore any other subtags. // ignore any other subtags.
LanguageTag localeTag(cx); LanguageTag localeTag(cx);
if (!LanguageTagParser::parseBaseName(cx, localeRange, localeTag)) { if (!LanguageTagParser::parseBaseName(cx, localeSpan, localeTag)) {
return false; return false;
} }
@ -1025,18 +1077,6 @@ UniqueChars LanguageTagParser::chars(JSContext* cx, size_t index,
return chars; return chars;
} }
UniqueChars LanguageTagParser::extension(JSContext* cx, const Token& start,
const Token& end) const {
MOZ_ASSERT(start.index() < end.index());
size_t length = end.index() - 1 - start.index();
UniqueChars extension = chars(cx, start.index(), length);
if (extension) {
AsciiToLowerCase(extension.get(), length, extension.get());
}
return extension;
}
// Parse the `unicode_language_id` production. // Parse the `unicode_language_id` production.
// //
// unicode_language_id = unicode_language_subtag // unicode_language_id = unicode_language_subtag
@ -1051,55 +1091,22 @@ UniqueChars LanguageTagParser::extension(JSContext* cx, const Token& start,
// //
// |tok| is the current token from |ts|. // |tok| is the current token from |ts|.
// //
// The trailing |parseType| argument corresponds to one of two modes. // All subtags will be added unaltered to |tag|, without canonicalizing their
// // case or, in the case of variant subtags, detecting and rejecting duplicate
// In the |BaseNameParsing::Normal| mode, our input is in unknown case and is // variants. Users must subsequently |canonicalizeBaseName| to perform these
// potentially invalid. |tag| will be filled with canonically-cased output, and // actions.
// duplicate variants will lead to an error.
//
// In the |BaseNameParsing::WithinTransformExtension| mode, our input is the
// `tlang` in a lowercased `transform_extensions`. |tag| subtags will be
// directly copied from the input (i.e. in lowercase). Variant subtags in the
// `tlang` subtag may contain duplicates.
// //
// Do not use this function directly: use |parseBaseName| or // Do not use this function directly: use |parseBaseName| or
// |parseTlangFromTransformExtension| instead. // |parseTlangFromTransformExtension| instead.
JS::Result<bool> LanguageTagParser::internalParseBaseName( JS::Result<bool> LanguageTagParser::internalParseBaseName(JSContext* cx,
JSContext* cx, LanguageTagParser& ts, LanguageTag& tag, Token& tok, LanguageTagParser& ts,
BaseNameParsing parseType) { LanguageTag& tag,
#ifdef DEBUG Token& tok) {
auto isAsciiLowerCase = [](const auto& range) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
const char* ptr = range.begin().get();
size_t length = range.length();
return std::all_of(ptr, ptr + length, mozilla::IsAsciiLowercaseAlpha<char>);
};
auto isAsciiDigit = [](const auto& range) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
const char* ptr = range.begin().get();
size_t length = range.length();
return std::all_of(ptr, ptr + length, mozilla::IsAsciiDigit<char>);
};
#endif
if (ts.isLanguage(tok)) { if (ts.isLanguage(tok)) {
ts.copyChars(tok, tag.language_); ts.copyChars(tok, tag.language_);
// Language codes need to be in lower case. "JA" -> "ja"
if (parseType == BaseNameParsing::Normal) {
tag.language_.toLowerCase();
} else {
MOZ_ASSERT(isAsciiLowerCase(tag.language_.range()));
}
tok = ts.nextToken(); tok = ts.nextToken();
} else { } else {
MOZ_ASSERT(parseType == BaseNameParsing::Normal);
// The language subtag is mandatory. // The language subtag is mandatory.
return false; return false;
} }
@ -1107,28 +1114,12 @@ JS::Result<bool> LanguageTagParser::internalParseBaseName(
if (ts.isScript(tok)) { if (ts.isScript(tok)) {
ts.copyChars(tok, tag.script_); ts.copyChars(tok, tag.script_);
// The first character of a script code needs to be capitalized.
// "hans" -> "Hans"
if (parseType == BaseNameParsing::Normal) {
tag.script_.toTitleCase();
} else {
MOZ_ASSERT(isAsciiLowerCase(tag.script_.range()));
}
tok = ts.nextToken(); tok = ts.nextToken();
} }
if (ts.isRegion(tok)) { if (ts.isRegion(tok)) {
ts.copyChars(tok, tag.region_); ts.copyChars(tok, tag.region_);
// Region codes need to be in upper case. "bu" -> "BU"
if (parseType == BaseNameParsing::Normal) {
tag.region_.toUpperCase();
} else {
MOZ_ASSERT_IF(tok.length() == 2, isAsciiLowerCase(tag.region_.range()));
MOZ_ASSERT_IF(tok.length() == 3, isAsciiDigit(tag.region_.range()));
}
tok = ts.nextToken(); tok = ts.nextToken();
} }
@ -1139,28 +1130,6 @@ JS::Result<bool> LanguageTagParser::internalParseBaseName(
if (!variant) { if (!variant) {
return cx->alreadyReportedOOM(); return cx->alreadyReportedOOM();
} }
if (parseType == BaseNameParsing::Normal) {
// Locale identifiers are case insensitive (UTS 35, section 3.2).
// All seen variants are compared ignoring case differences by using the
// lower case form. This allows to properly detect and reject variant
// repetitions with differing case, e.g. "en-variant-Variant".
AsciiToLowerCase(variant.get(), tok.length(), variant.get());
// Reject the Locale identifier if a duplicate variant was found.
//
// This linear-time verification step means the whole variant subtag
// checking is potentially quadratic. Language tags are unlikely to be
// deliberately pathological, so this is okay at least for now.
for (const auto& seenVariant : variants) {
if (strcmp(variant.get(), seenVariant.get()) == 0) {
return false;
}
}
} else {
// When parsing variants in a `tlang` subtag, duplicates are allowed.
}
if (!variants.append(std::move(variant))) { if (!variants.append(std::move(variant))) {
return cx->alreadyReportedOOM(); return cx->alreadyReportedOOM();
} }
@ -1332,10 +1301,11 @@ bool LanguageTagParser::parse(JSContext* cx, JSLinearString* locale,
return false; return false;
} }
bool LanguageTagParser::parseBaseName(JSContext* cx, ConstCharRange locale, bool LanguageTagParser::parseBaseName(JSContext* cx,
mozilla::Span<const char> locale,
LanguageTag& tag) { LanguageTag& tag) {
LocaleChars localeChars = StringChars(locale.begin().get()); LocaleChars localeChars = StringChars(locale.data());
LanguageTagParser ts(localeChars, locale.length()); LanguageTagParser ts(localeChars, locale.size());
Token tok = ts.nextToken(); Token tok = ts.nextToken();
// Parse only the base-name part and ignore any trailing characters. // Parse only the base-name part and ignore any trailing characters.
@ -1344,12 +1314,10 @@ bool LanguageTagParser::parseBaseName(JSContext* cx, ConstCharRange locale,
if (ok) { if (ok) {
return true; return true;
} }
if (UniqueChars localeChars = if (UniqueChars localeChars = DuplicateString(cx, locale.data(),
DuplicateString(locale.begin().get(), locale.length())) { locale.size())) {
JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr, JS_ReportErrorNumberUTF8(cx, GetErrorMessage, nullptr,
JSMSG_INVALID_LANGUAGE_TAG, localeChars.get()); JSMSG_INVALID_LANGUAGE_TAG, localeChars.get());
} else {
JS_ReportOutOfMemory(cx);
} }
return false; return false;
} }
@ -1357,10 +1325,10 @@ bool LanguageTagParser::parseBaseName(JSContext* cx, ConstCharRange locale,
// Parse |extension|, which must be a valid `transformed_extensions` subtag, and // Parse |extension|, which must be a valid `transformed_extensions` subtag, and
// fill |tag| and |fields| from the `tlang` and `tfield` components. // fill |tag| and |fields| from the `tlang` and `tfield` components.
JS::Result<bool> LanguageTagParser::parseTransformExtension( JS::Result<bool> LanguageTagParser::parseTransformExtension(
JSContext* cx, ConstCharRange extension, LanguageTag& tag, JSContext* cx, mozilla::Span<const char> extension, LanguageTag& tag,
TFieldVector& fields) { TFieldVector& fields) {
LocaleChars extensionChars = StringChars(extension.begin().get()); LocaleChars extensionChars = StringChars(extension.data());
LanguageTagParser ts(extensionChars, extension.length()); LanguageTagParser ts(extensionChars, extension.size());
Token tok = ts.nextToken(); Token tok = ts.nextToken();
if (!ts.isExtensionStart(tok) || ts.singletonKey(tok) != 't') { if (!ts.isExtensionStart(tok) || ts.singletonKey(tok) != 't') {
@ -1417,10 +1385,10 @@ JS::Result<bool> LanguageTagParser::parseTransformExtension(
// and fill |attributes| and |keywords| from the `attribute` and `keyword` // and fill |attributes| and |keywords| from the `attribute` and `keyword`
// components. // components.
JS::Result<bool> LanguageTagParser::parseUnicodeExtension( JS::Result<bool> LanguageTagParser::parseUnicodeExtension(
JSContext* cx, ConstCharRange extension, AttributesVector& attributes, JSContext* cx, mozilla::Span<const char> extension,
KeywordsVector& keywords) { AttributesVector& attributes, KeywordsVector& keywords) {
LocaleChars extensionChars = StringChars(extension.begin().get()); LocaleChars extensionChars = StringChars(extension.data());
LanguageTagParser ts(extensionChars, extension.length()); LanguageTagParser ts(extensionChars, extension.size());
Token tok = ts.nextToken(); Token tok = ts.nextToken();
// unicode_locale_extensions = sep [uU] ((sep keyword)+ | // unicode_locale_extensions = sep [uU] ((sep keyword)+ |
@ -1467,9 +1435,10 @@ JS::Result<bool> LanguageTagParser::parseUnicodeExtension(
return tok.isNone(); return tok.isNone();
} }
bool LanguageTagParser::canParseUnicodeExtension(ConstCharRange extension) { bool LanguageTagParser::canParseUnicodeExtension(
LocaleChars extensionChars = StringChars(extension.begin().get()); mozilla::Span<const char> extension) {
LanguageTagParser ts(extensionChars, extension.length()); LocaleChars extensionChars = StringChars(extension.data());
LanguageTagParser ts(extensionChars, extension.size());
Token tok = ts.nextToken(); Token tok = ts.nextToken();
// unicode_locale_extensions = sep [uU] ((sep keyword)+ | // unicode_locale_extensions = sep [uU] ((sep keyword)+ |
@ -1522,118 +1491,89 @@ bool LanguageTagParser::canParseUnicodeExtensionType(
return tok.isNone(); return tok.isNone();
} }
bool ParseStandaloneLanguagTag(HandleLinearString str, LanguageSubtag& result) { bool ParseStandaloneLanguageTag(HandleLinearString str,
auto isLanguage = [](const auto* language, size_t length) { LanguageSubtag& result) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
using T = std::remove_pointer_t<decltype(language)>;
return length >= 2 && length != 4 && length <= 8 &&
std::all_of(language, language + length, mozilla::IsAsciiAlpha<T>);
};
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
if (str->hasLatin1Chars()) { if (str->hasLatin1Chars()) {
if (!isLanguage(str->latin1Chars(nogc), str->length())) { if (!IsStructurallyValidLanguageTag<Latin1Char>(str->latin1Range(nogc))) {
return false; return false;
} }
result.set(str->latin1Range(nogc)); result.set<Latin1Char>(str->latin1Range(nogc));
} else { } else {
if (!isLanguage(str->twoByteChars(nogc), str->length())) { if (!IsStructurallyValidLanguageTag<char16_t>(str->twoByteRange(nogc))) {
return false; return false;
} }
result.set(str->twoByteRange(nogc)); result.set<char16_t>(str->twoByteRange(nogc));
} }
result.toLowerCase();
return true; return true;
} }
bool ParseStandaloneScriptTag(HandleLinearString str, ScriptSubtag& result) { bool ParseStandaloneScriptTag(HandleLinearString str, ScriptSubtag& result) {
auto isScript = [](const auto* script, size_t length) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
using T = std::remove_pointer_t<decltype(script)>;
return length == ScriptLength &&
std::all_of(script, script + ScriptLength, mozilla::IsAsciiAlpha<T>);
};
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
if (str->hasLatin1Chars()) { if (str->hasLatin1Chars()) {
if (!isScript(str->latin1Chars(nogc), str->length())) { if (!IsStructurallyValidScriptTag<Latin1Char>(str->latin1Range(nogc))) {
return false; return false;
} }
result.set(str->latin1Range(nogc)); result.set<Latin1Char>(str->latin1Range(nogc));
} else { } else {
if (!isScript(str->twoByteChars(nogc), str->length())) { if (!IsStructurallyValidScriptTag<char16_t>(str->twoByteRange(nogc))) {
return false; return false;
} }
result.set(str->twoByteRange(nogc)); result.set<char16_t>(str->twoByteRange(nogc));
} }
result.toTitleCase();
return true; return true;
} }
bool ParseStandaloneRegionTag(HandleLinearString str, RegionSubtag& result) { bool ParseStandaloneRegionTag(HandleLinearString str, RegionSubtag& result) {
auto isRegion = [](const auto* region, size_t length) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
using T = std::remove_pointer_t<decltype(region)>;
return (length == AlphaRegionLength &&
std::all_of(region, region + AlphaRegionLength,
mozilla::IsAsciiAlpha<T>)) ||
(length == DigitRegionLength &&
std::all_of(region, region + DigitRegionLength,
mozilla::IsAsciiDigit<T>));
};
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
if (str->hasLatin1Chars()) { if (str->hasLatin1Chars()) {
if (!isRegion(str->latin1Chars(nogc), str->length())) { if (!IsStructurallyValidRegionTag<Latin1Char>(str->latin1Range(nogc))) {
return false; return false;
} }
result.set(str->latin1Range(nogc)); result.set<Latin1Char>(str->latin1Range(nogc));
} else { } else {
if (!isRegion(str->twoByteChars(nogc), str->length())) { if (!IsStructurallyValidRegionTag<char16_t>(str->twoByteRange(nogc))) {
return false; return false;
} }
result.set(str->twoByteRange(nogc)); result.set<char16_t>(str->twoByteRange(nogc));
} }
result.toUpperCase();
return true; return true;
} }
template <typename CharT> template <typename CharT>
static bool IsAsciiLowercaseAlpha(const mozilla::Range<const CharT>& range) { static bool IsAsciiLowercaseAlpha(mozilla::Span<const CharT> span) {
// Tell the analysis the |std::all_of| function can't GC. // Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc; JS::AutoSuppressGCAnalysis nogc;
const CharT* ptr = range.begin().get(); const CharT* ptr = span.data();
size_t length = range.length(); size_t length = span.size();
return std::all_of(ptr, ptr + length, mozilla::IsAsciiLowercaseAlpha<CharT>); return std::all_of(ptr, ptr + length, mozilla::IsAsciiLowercaseAlpha<CharT>);
} }
static bool IsAsciiLowercaseAlpha(JSLinearString* str) { static bool IsAsciiLowercaseAlpha(JSLinearString* str) {
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
return str->hasLatin1Chars() ? IsAsciiLowercaseAlpha(str->latin1Range(nogc)) if (str->hasLatin1Chars()) {
: IsAsciiLowercaseAlpha(str->twoByteRange(nogc)); return IsAsciiLowercaseAlpha<Latin1Char>(str->latin1Range(nogc));
}
return IsAsciiLowercaseAlpha<char16_t>(str->twoByteRange(nogc));
} }
template <typename CharT> template <typename CharT>
static bool IsAsciiAlpha(const mozilla::Range<const CharT>& range) { static bool IsAsciiAlpha(mozilla::Span<const CharT> span) {
// Tell the analysis the |std::all_of| function can't GC. // Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc; JS::AutoSuppressGCAnalysis nogc;
const CharT* ptr = range.begin().get(); const CharT* ptr = span.data();
size_t length = range.length(); size_t length = span.size();
return std::all_of(ptr, ptr + length, mozilla::IsAsciiAlpha<CharT>); return std::all_of(ptr, ptr + length, mozilla::IsAsciiAlpha<CharT>);
} }
static bool IsAsciiAlpha(JSLinearString* str) { static bool IsAsciiAlpha(JSLinearString* str) {
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
return str->hasLatin1Chars() ? IsAsciiAlpha(str->latin1Range(nogc)) if (str->hasLatin1Chars()) {
: IsAsciiAlpha(str->twoByteRange(nogc)); return IsAsciiAlpha<Latin1Char>(str->latin1Range(nogc));
}
return IsAsciiAlpha<char16_t>(str->twoByteRange(nogc));
} }
JS::Result<JSString*> ParseStandaloneISO639LanguageTag(JSContext* cx, JS::Result<JSString*> ParseStandaloneISO639LanguageTag(JSContext* cx,
@ -1656,10 +1596,10 @@ JS::Result<JSString*> ParseStandaloneISO639LanguageTag(JSContext* cx,
LanguageSubtag languageTag; LanguageSubtag languageTag;
if (str->hasLatin1Chars()) { if (str->hasLatin1Chars()) {
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
languageTag.set(str->latin1Range(nogc)); languageTag.set<Latin1Char>(str->latin1Range(nogc));
} else { } else {
JS::AutoCheckCannotGC nogc; JS::AutoCheckCannotGC nogc;
languageTag.set(str->twoByteRange(nogc)); languageTag.set<char16_t>(str->twoByteRange(nogc));
} }
if (!isLowerCase) { if (!isLowerCase) {
@ -1676,8 +1616,8 @@ JS::Result<JSString*> ParseStandaloneISO639LanguageTag(JSContext* cx,
// Take care to replace deprecated subtags with their preferred values. // Take care to replace deprecated subtags with their preferred values.
JSString* result; JSString* result;
if (LanguageTag::languageMapping(languageTag) || !isLowerCase) { if (LanguageTag::languageMapping(languageTag) || !isLowerCase) {
auto range = languageTag.range(); auto span = languageTag.span();
result = NewStringCopyN<CanGC>(cx, range.begin().get(), range.length()); result = NewStringCopyN<CanGC>(cx, span.data(), span.size());
} else { } else {
result = str; result = str;
} }

223
js/src/builtin/intl/LanguageTag.h
View file

@ -10,7 +10,7 @@
#define builtin_intl_LanguageTag_h #define builtin_intl_LanguageTag_h
#include "mozilla/Assertions.h" #include "mozilla/Assertions.h"
#include "mozilla/Range.h" #include "mozilla/Span.h"
#include "mozilla/TextUtils.h" #include "mozilla/TextUtils.h"
#include "mozilla/TypedEnumBits.h" #include "mozilla/TypedEnumBits.h"
#include "mozilla/Variant.h" #include "mozilla/Variant.h"
@ -36,57 +36,53 @@ namespace js {
namespace intl { namespace intl {
/**
* Return true if |language| is a valid language subtag.
*/
template <typename CharT>
bool IsStructurallyValidLanguageTag(mozilla::Span<const CharT> language);
/**
* Return true if |script| is a valid script subtag.
*/
template <typename CharT>
bool IsStructurallyValidScriptTag(mozilla::Span<const CharT> script);
/**
* Return true if |region| is a valid region subtag.
*/
template <typename CharT>
bool IsStructurallyValidRegionTag(mozilla::Span<const CharT> region);
#ifdef DEBUG #ifdef DEBUG
/** /**
* Return true if |language| is a valid, case-normalized language subtag. * Return true if |variant| is a valid variant subtag.
*/ */
template <typename CharT> bool IsStructurallyValidVariantTag(mozilla::Span<const char> variant);
bool IsStructurallyValidLanguageTag(
const mozilla::Range<const CharT>& language);
/** /**
* Return true if |script| is a valid, case-normalized script subtag. * Return true if |extension| is a valid Unicode extension subtag.
*/
template <typename CharT>
bool IsStructurallyValidScriptTag(const mozilla::Range<const CharT>& script);
/**
* Return true if |region| is a valid, case-normalized region subtag.
*/
template <typename CharT>
bool IsStructurallyValidRegionTag(const mozilla::Range<const CharT>& region);
/**
* Return true if |variant| is a valid, case-normalized variant subtag.
*/
bool IsStructurallyValidVariantTag(const mozilla::Range<const char>& variant);
/**
* Return true if |extension| is a valid, case-normalized Unicode extension
* subtag.
*/ */
bool IsStructurallyValidUnicodeExtensionTag( bool IsStructurallyValidUnicodeExtensionTag(
const mozilla::Range<const char>& extension); mozilla::Span<const char> extension);
/** /**
* Return true if |privateUse| is a valid, case-normalized private-use subtag. * Return true if |privateUse| is a valid private-use subtag.
*/ */
bool IsStructurallyValidPrivateUseTag( bool IsStructurallyValidPrivateUseTag(mozilla::Span<const char> privateUse);
const mozilla::Range<const char>& privateUse);
#endif #endif
template <typename CharT> template <typename CharT>
char AsciiToLowerCase(CharT c) { char AsciiToLowerCase(CharT c) {
MOZ_ASSERT(mozilla::IsAscii(c)); MOZ_ASSERT(mozilla::IsAscii(c));
return mozilla::IsAsciiUppercaseAlpha(c) ? (c | 0x20) : c; return mozilla::IsAsciiUppercaseAlpha(c) ? (c + 0x20) : c;
} }
template <typename CharT> template <typename CharT>
char AsciiToUpperCase(CharT c) { char AsciiToUpperCase(CharT c) {
MOZ_ASSERT(mozilla::IsAscii(c)); MOZ_ASSERT(mozilla::IsAscii(c));
return mozilla::IsAsciiLowercaseAlpha(c) ? (c & ~0x20) : c; return mozilla::IsAsciiLowercaseAlpha(c) ? (c - 0x20) : c;
} }
template <typename CharT> template <typename CharT>
@ -141,7 +137,7 @@ static constexpr size_t TransformKeyLength = 2;
template <size_t Length> template <size_t Length>
class LanguageTagSubtag final { class LanguageTagSubtag final {
uint8_t length_ = 0; uint8_t length_ = 0;
char chars_[Length]; char chars_[Length] = {}; // zero initialize
public: public:
LanguageTagSubtag() = default; LanguageTagSubtag() = default;
@ -150,21 +146,31 @@ class LanguageTagSubtag final {
LanguageTagSubtag& operator=(const LanguageTagSubtag&) = delete; LanguageTagSubtag& operator=(const LanguageTagSubtag&) = delete;
size_t length() const { return length_; } size_t length() const { return length_; }
bool missing() const { return length_ == 0; }
bool present() const { return length_ > 0; }
mozilla::Range<const char> range() const { return {chars_, length_}; } mozilla::Span<const char> span() const { return {chars_, length_}; }
template <typename CharT> template <typename CharT>
void set(const mozilla::Range<const CharT>& str) { void set(mozilla::Span<const CharT> str) {
MOZ_ASSERT(str.length() <= Length); MOZ_ASSERT(str.size() <= Length);
std::copy_n(str.begin().get(), str.length(), chars_); std::copy_n(str.data(), str.size(), chars_);
length_ = str.length(); length_ = str.size();
} }
void toLowerCase() { AsciiToLowerCase(chars_, length(), chars_); } // The toXYZCase() methods are using |Length| instead of |length()|, because
// current compilers (tested GCC and Clang) can't infer the maximum string
// length - even when using hints like |std::min| - and instead are emitting
// SIMD optimized code. Using a fixed sized length avoids emitting the SIMD
// code. (Emitting SIMD code doesn't make sense here, because the SIMD code
// only kicks in for long strings.) A fixed length will additionally ensure
// the compiler unrolls the loop in the case conversion code.
void toUpperCase() { AsciiToUpperCase(chars_, length(), chars_); } void toLowerCase() { AsciiToLowerCase(chars_, Length, chars_); }
void toTitleCase() { AsciiToTitleCase(chars_, length(), chars_); } void toUpperCase() { AsciiToUpperCase(chars_, Length, chars_); }
void toTitleCase() { AsciiToTitleCase(chars_, Length, chars_); }
template <size_t N> template <size_t N>
bool equalTo(const char (&str)[N]) const { bool equalTo(const char (&str)[N]) const {
@ -224,8 +230,7 @@ class MOZ_STACK_CLASS LanguageTag final {
MOZ_MUST_USE bool updateGrandfatheredMappings(JSContext* cx); MOZ_MUST_USE bool updateGrandfatheredMappings(JSContext* cx);
static const char* replaceUnicodeExtensionType( static const char* replaceUnicodeExtensionType(
const mozilla::Range<const char>& key, mozilla::Span<const char> key, mozilla::Span<const char> type);
const mozilla::Range<const char>& type);
public: public:
explicit LanguageTag(JSContext* cx) : variants_(cx), extensions_(cx) {} explicit LanguageTag(JSContext* cx) : variants_(cx), extensions_(cx) {}
@ -241,65 +246,68 @@ class MOZ_STACK_CLASS LanguageTag final {
const char* privateuse() const { return privateuse_.get(); } const char* privateuse() const { return privateuse_.get(); }
/** /**
* Set the language subtag. The input must be a valid, case-normalized * Return the Unicode extension subtag or nullptr if not present.
* language subtag. */
const char* unicodeExtension() const;
private:
ptrdiff_t unicodeExtensionIndex() const;
public:
/**
* Set the language subtag. The input must be a valid language subtag.
*/ */
template <size_t N> template <size_t N>
void setLanguage(const char (&language)[N]) { void setLanguage(const char (&language)[N]) {
mozilla::Range<const char> range(language, N - 1); mozilla::Span<const char> span(language, N - 1);
MOZ_ASSERT(IsStructurallyValidLanguageTag(range)); MOZ_ASSERT(IsStructurallyValidLanguageTag(span));
language_.set(range); language_.set(span);
} }
/** /**
* Set the language subtag. The input must be a valid, case-normalized * Set the language subtag. The input must be a valid language subtag.
* language subtag.
*/ */
void setLanguage(const LanguageSubtag& language) { void setLanguage(const LanguageSubtag& language) {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language.range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language.span()));
language_.set(language.range()); language_.set(language.span());
} }
/** /**
* Set the script subtag. The input must be a valid, case-normalized * Set the script subtag. The input must be a valid script subtag.
* script subtag or the empty string.
*/ */
template <size_t N> template <size_t N>
void setScript(const char (&script)[N]) { void setScript(const char (&script)[N]) {
mozilla::Range<const char> range(script, N - 1); mozilla::Span<const char> span(script, N - 1);
MOZ_ASSERT(IsStructurallyValidScriptTag(range)); MOZ_ASSERT(IsStructurallyValidScriptTag(span));
script_.set(range); script_.set(span);
} }
/** /**
* Set the script subtag. The input must be a valid, case-normalized * Set the script subtag. The input must be a valid script subtag or the empty
* script subtag or the empty string. * string.
*/ */
void setScript(const ScriptSubtag& script) { void setScript(const ScriptSubtag& script) {
MOZ_ASSERT(script.length() == 0 || MOZ_ASSERT(script.missing() || IsStructurallyValidScriptTag(script.span()));
IsStructurallyValidScriptTag(script.range())); script_.set(script.span());
script_.set(script.range());
} }
/** /**
* Set the region subtag. The input must be a valid, case-normalized * Set the region subtag. The input must be a valid region subtag.
* region subtag or the empty string.
*/ */
template <size_t N> template <size_t N>
void setRegion(const char (&region)[N]) { void setRegion(const char (&region)[N]) {
mozilla::Range<const char> range(region, N - 1); mozilla::Span<const char> span(region, N - 1);
MOZ_ASSERT(IsStructurallyValidRegionTag(range)); MOZ_ASSERT(IsStructurallyValidRegionTag(span));
region_.set(range); region_.set(span);
} }
/** /**
* Set the region subtag. The input must be a valid, case-normalized * Set the region subtag. The input must be a valid region subtag or the empty
* region subtag or the empty string. * empty string.
*/ */
void setRegion(const RegionSubtag& region) { void setRegion(const RegionSubtag& region) {
MOZ_ASSERT(region.length() == 0 || MOZ_ASSERT(region.missing() || IsStructurallyValidRegionTag(region.span()));
IsStructurallyValidRegionTag(region.range())); region_.set(region.span());
region_.set(region.range());
} }
/** /**
@ -308,8 +316,8 @@ class MOZ_STACK_CLASS LanguageTag final {
void clearVariants() { variants_.clearAndFree(); } void clearVariants() { variants_.clearAndFree(); }
/** /**
* Set the Unicode extension subtag. The input must be a valid, * Set the Unicode extension subtag. The input must be a valid Unicode
* case-normalized Unicode extension subtag. * extension subtag.
*/ */
bool setUnicodeExtension(JS::UniqueChars extension); bool setUnicodeExtension(JS::UniqueChars extension);
@ -319,8 +327,8 @@ class MOZ_STACK_CLASS LanguageTag final {
void clearUnicodeExtension(); void clearUnicodeExtension();
/** /**
* Set the private-use subtag. The input must be a valid, case-normalized * Set the private-use subtag. The input must be a valid private-use subtag
* private-use subtag or the empty string. * or nullptr.
*/ */
void setPrivateuse(JS::UniqueChars privateuse) { void setPrivateuse(JS::UniqueChars privateuse) {
MOZ_ASSERT(!privateuse || MOZ_ASSERT(!privateuse ||
@ -462,10 +470,10 @@ class MOZ_STACK_CLASS LanguageTagParser final {
size_t length = tok.length(); size_t length = tok.length();
if (locale_.is<const JS::Latin1Char*>()) { if (locale_.is<const JS::Latin1Char*>()) {
using T = const JS::Latin1Char; using T = const JS::Latin1Char;
subtag.set(mozilla::Range<T>(locale_.as<T*>() + index, length)); subtag.set(mozilla::MakeSpan(locale_.as<T*>() + index, length));
} else { } else {
using T = const char16_t; using T = const char16_t;
subtag.set(mozilla::Range<T>(locale_.as<T*>() + index, length)); subtag.set(mozilla::MakeSpan(locale_.as<T*>() + index, length));
} }
} }
@ -477,10 +485,15 @@ class MOZ_STACK_CLASS LanguageTagParser final {
return chars(cx, tok.index(), tok.length()); return chars(cx, tok.index(), tok.length());
} }
Token nextToken();
JS::UniqueChars extension(JSContext* cx, const Token& start, JS::UniqueChars extension(JSContext* cx, const Token& start,
const Token& end) const; const Token& end) const {
MOZ_ASSERT(start.index() < end.index());
size_t length = end.index() - 1 - start.index();
return chars(cx, start.index(), length);
}
Token nextToken();
// unicode_language_subtag = alpha{2,3} | alpha{5,8} ; // unicode_language_subtag = alpha{2,3} | alpha{5,8} ;
// //
@ -513,8 +526,7 @@ class MOZ_STACK_CLASS LanguageTagParser final {
// Always returns the lower case form of an alphabetical character. // Always returns the lower case form of an alphabetical character.
char singletonKey(const Token& tok) const { char singletonKey(const Token& tok) const {
MOZ_ASSERT(tok.length() == 1); MOZ_ASSERT(tok.length() == 1);
char c = charAt(tok.index()); return AsciiToLowerCase(charAt(tok.index()));
return mozilla::IsAsciiUppercaseAlpha(c) ? (c | 0x20) : c;
} }
// extensions = unicode_locale_extensions | // extensions = unicode_locale_extensions |
@ -581,23 +593,18 @@ class MOZ_STACK_CLASS LanguageTagParser final {
return 1 <= tok.length() && tok.length() <= 8; return 1 <= tok.length() && tok.length() <= 8;
} }
enum class BaseNameParsing : bool { Normal, WithinTransformExtension };
// Helper function for use in |parseBaseName| and // Helper function for use in |parseBaseName| and
// |parseTlangInTransformExtension|. Do not use this directly! // |parseTlangInTransformExtension|. Do not use this directly!
static JS::Result<bool> internalParseBaseName(JSContext* cx, static JS::Result<bool> internalParseBaseName(JSContext* cx,
LanguageTagParser& ts, LanguageTagParser& ts,
LanguageTag& tag, Token& tok, LanguageTag& tag, Token& tok);
BaseNameParsing parseType);
// Parse the `unicode_language_id` production, i.e. the // Parse the `unicode_language_id` production, i.e. the
// language/script/region/variants portion of a language tag, into |tag|, // language/script/region/variants portion of a language tag, into |tag|.
// which will be filled with canonical-cased components (lowercase language, // |tok| must be the current token.
// titlecase script, uppercase region, lowercased and alphabetized and
// deduplicated variants). |tok| must be the current token.
static JS::Result<bool> parseBaseName(JSContext* cx, LanguageTagParser& ts, static JS::Result<bool> parseBaseName(JSContext* cx, LanguageTagParser& ts,
LanguageTag& tag, Token& tok) { LanguageTag& tag, Token& tok) {
return internalParseBaseName(cx, ts, tag, tok, BaseNameParsing::Normal); return internalParseBaseName(cx, ts, tag, tok);
} }
// Parse the `tlang` production within a parsed 't' transform extension. // Parse the `tlang` production within a parsed 't' transform extension.
@ -611,17 +618,14 @@ class MOZ_STACK_CLASS LanguageTagParser final {
// Return an error on internal failure. Otherwise, return a success value. If // Return an error on internal failure. Otherwise, return a success value. If
// there was no `tlang`, then |tag.language().missing()|. But if there was a // there was no `tlang`, then |tag.language().missing()|. But if there was a
// `tlang`, then |tag| is filled with subtags exactly as they appeared in the // `tlang`, then |tag| is filled with subtags exactly as they appeared in the
// parse input: fully lowercase, variants in alphabetical order without // parse input.
// duplicates.
static JS::Result<JS::Ok> parseTlangInTransformExtension( static JS::Result<JS::Ok> parseTlangInTransformExtension(
JSContext* cx, LanguageTagParser& ts, LanguageTag& tag, Token& tok) { JSContext* cx, LanguageTagParser& ts, LanguageTag& tag, Token& tok) {
MOZ_ASSERT(ts.isLanguage(tok)); MOZ_ASSERT(ts.isLanguage(tok));
return internalParseBaseName(cx, ts, tag, tok, return internalParseBaseName(cx, ts, tag, tok).map([](bool parsed) {
BaseNameParsing::WithinTransformExtension) MOZ_ASSERT(parsed);
.map([](bool parsed) { return JS::Ok();
MOZ_ASSERT(parsed); });
return JS::Ok();
});
} }
friend class LanguageTag; friend class LanguageTag;
@ -650,14 +654,14 @@ class MOZ_STACK_CLASS LanguageTagParser final {
// `tlang` and `tfield` components. Data in |tag| is lowercase, consistent // `tlang` and `tfield` components. Data in |tag| is lowercase, consistent
// with |extension|. // with |extension|.
static JS::Result<bool> parseTransformExtension( static JS::Result<bool> parseTransformExtension(
JSContext* cx, mozilla::Range<const char> extension, LanguageTag& tag, JSContext* cx, mozilla::Span<const char> extension, LanguageTag& tag,
TFieldVector& fields); TFieldVector& fields);
// Parse |extension|, which must be a validated, fully lowercase // Parse |extension|, which must be a validated, fully lowercase
// `unicode_locale_extensions` subtag, and fill |attributes| and |keywords| // `unicode_locale_extensions` subtag, and fill |attributes| and |keywords|
// from the `attribute` and `keyword` components. // from the `attribute` and `keyword` components.
static JS::Result<bool> parseUnicodeExtension( static JS::Result<bool> parseUnicodeExtension(
JSContext* cx, mozilla::Range<const char> extension, JSContext* cx, mozilla::Span<const char> extension,
AttributesVector& attributes, KeywordsVector& keywords); AttributesVector& attributes, KeywordsVector& keywords);
public: public:
@ -673,11 +677,11 @@ class MOZ_STACK_CLASS LanguageTagParser final {
// Parse the input string as the base-name parts (language, script, region, // Parse the input string as the base-name parts (language, script, region,
// variants) of a language tag. Ignores any trailing characters. // variants) of a language tag. Ignores any trailing characters.
static bool parseBaseName(JSContext* cx, mozilla::Range<const char> locale, static bool parseBaseName(JSContext* cx, mozilla::Span<const char> locale,
LanguageTag& tag); LanguageTag& tag);
// Return true iff |extension| can be parsed as a Unicode extension subtag. // Return true iff |extension| can be parsed as a Unicode extension subtag.
static bool canParseUnicodeExtension(mozilla::Range<const char> extension); static bool canParseUnicodeExtension(mozilla::Span<const char> extension);
// Return true iff |unicodeType| can be parsed as a Unicode extension type. // Return true iff |unicodeType| can be parsed as a Unicode extension type.
static bool canParseUnicodeExtensionType(JSLinearString* unicodeType); static bool canParseUnicodeExtensionType(JSLinearString* unicodeType);
@ -687,24 +691,21 @@ MOZ_MAKE_ENUM_CLASS_BITWISE_OPERATORS(LanguageTagParser::TokenKind)
/** /**
* Parse a string as a standalone |language| tag. If |str| is a standalone * Parse a string as a standalone |language| tag. If |str| is a standalone
* language tag, store it in case-normalized form in |result| and return true. * language tag, store it in |result| and return true. Otherwise return false.
* Otherwise return false.
*/ */
MOZ_MUST_USE bool ParseStandaloneLanguagTag(JS::Handle<JSLinearString*> str, MOZ_MUST_USE bool ParseStandaloneLanguageTag(JS::Handle<JSLinearString*> str,
LanguageSubtag& result); LanguageSubtag& result);
/** /**
* Parse a string as a standalone |script| tag. If |str| is a standalone script * Parse a string as a standalone |script| tag. If |str| is a standalone script
* tag, store it in case-normalized form in |result| and return true. Otherwise * tag, store it in |result| and return true. Otherwise return false.
* return false.
*/ */
MOZ_MUST_USE bool ParseStandaloneScriptTag(JS::Handle<JSLinearString*> str, MOZ_MUST_USE bool ParseStandaloneScriptTag(JS::Handle<JSLinearString*> str,
ScriptSubtag& result); ScriptSubtag& result);
/** /**
* Parse a string as a standalone |region| tag. If |str| is a standalone region * Parse a string as a standalone |region| tag. If |str| is a standalone region
* tag, store it in case-normalized form in |result| and return true. Otherwise * tag, store it in |result| and return true. Otherwise return false.
* return false.
*/ */
MOZ_MUST_USE bool ParseStandaloneRegionTag(JS::Handle<JSLinearString*> str, MOZ_MUST_USE bool ParseStandaloneRegionTag(JS::Handle<JSLinearString*> str,
RegionSubtag& result); RegionSubtag& result);

109
js/src/builtin/intl/LanguageTagGenerated.cpp
View file

@ -3,7 +3,7 @@
// URL: https://unicode.org/Public/cldr/35.1/core.zip // URL: https://unicode.org/Public/cldr/35.1/core.zip
#include "mozilla/Assertions.h" #include "mozilla/Assertions.h"
#include "mozilla/Range.h" #include "mozilla/Span.h"
#include "mozilla/TextUtils.h" #include "mozilla/TextUtils.h"
#include <algorithm> #include <algorithm>
@ -18,7 +18,6 @@
#include "builtin/intl/LanguageTag.h" #include "builtin/intl/LanguageTag.h"
using namespace js::intl::LanguageTagLimits; using namespace js::intl::LanguageTagLimits;
using ConstCharRange = mozilla::Range<const char>;
template <size_t Length, size_t TagLength, size_t SubtagLength> template <size_t Length, size_t TagLength, size_t SubtagLength>
static inline bool HasReplacement( static inline bool HasReplacement(
@ -27,7 +26,7 @@ static inline bool HasReplacement(
MOZ_ASSERT(subtag.length() == TagLength - 1, MOZ_ASSERT(subtag.length() == TagLength - 1,
"subtag must have the same length as the list of subtags"); "subtag must have the same length as the list of subtags");
const char* ptr = subtag.range().begin().get(); const char* ptr = subtag.span().data();
return std::binary_search(std::begin(subtags), std::end(subtags), ptr, return std::binary_search(std::begin(subtags), std::end(subtags), ptr,
[](const char* a, const char* b) { [](const char* a, const char* b) {
return memcmp(a, b, TagLength - 1) < 0; return memcmp(a, b, TagLength - 1) < 0;
@ -42,7 +41,7 @@ static inline const char* SearchReplacement(
MOZ_ASSERT(subtag.length() == TagLength - 1, MOZ_ASSERT(subtag.length() == TagLength - 1,
"subtag must have the same length as the list of subtags"); "subtag must have the same length as the list of subtags");
const char* ptr = subtag.range().begin().get(); const char* ptr = subtag.span().data();
auto p = std::lower_bound(std::begin(subtags), std::end(subtags), ptr, auto p = std::lower_bound(std::begin(subtags), std::end(subtags), ptr,
[](const char* a, const char* b) { [](const char* a, const char* b) {
return memcmp(a, b, TagLength - 1) < 0; return memcmp(a, b, TagLength - 1) < 0;
@ -53,11 +52,40 @@ static inline const char* SearchReplacement(
return nullptr; return nullptr;
} }
#ifdef DEBUG
static bool IsCanonicallyCasedLanguageTag(mozilla::Span<const char> span) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
return std::all_of(span.begin(), span.end(), mozilla::IsAsciiLowercaseAlpha<char>);
}
static bool IsCanonicallyCasedRegionTag(mozilla::Span<const char> span) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
return std::all_of(span.begin(), span.end(), mozilla::IsAsciiUppercaseAlpha<char>) ||
std::all_of(span.begin(), span.end(), mozilla::IsAsciiDigit<char>);
}
static bool IsCanonicallyCasedVariantTag(mozilla::Span<const char> span) {
auto isAsciiLowercaseAlphaOrDigit = [](char c) {
return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c);
};
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
return std::all_of(span.begin(), span.end(), isAsciiLowercaseAlphaOrDigit);
}
#endif
// Mappings from language subtags to preferred values. // Mappings from language subtags to preferred values.
// Derived from CLDR Supplemental Data, version 35.1. // Derived from CLDR Supplemental Data, version 35.1.
// https://unicode.org/Public/cldr/35.1/core.zip // https://unicode.org/Public/cldr/35.1/core.zip
bool js::intl::LanguageTag::languageMapping(LanguageSubtag& language) { bool js::intl::LanguageTag::languageMapping(LanguageSubtag& language) {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language.range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language.span()));
MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language.span()));
if (language.length() == 2) { if (language.length() == 2) {
static const char languages[9][3] = { static const char languages[9][3] = {
@ -68,7 +96,7 @@ bool js::intl::LanguageTag::languageMapping(LanguageSubtag& language) {
}; };
if (const char* replacement = SearchReplacement(languages, aliases, language)) { if (const char* replacement = SearchReplacement(languages, aliases, language)) {
language.set(ConstCharRange(replacement, strlen(replacement))); language.set(mozilla::MakeCStringSpan(replacement));
return true; return true;
} }
return false; return false;
@ -149,7 +177,7 @@ bool js::intl::LanguageTag::languageMapping(LanguageSubtag& language) {
}; };
if (const char* replacement = SearchReplacement(languages, aliases, language)) { if (const char* replacement = SearchReplacement(languages, aliases, language)) {
language.set(ConstCharRange(replacement, strlen(replacement))); language.set(mozilla::MakeCStringSpan(replacement));
return true; return true;
} }
return false; return false;
@ -162,7 +190,8 @@ bool js::intl::LanguageTag::languageMapping(LanguageSubtag& language) {
// Derived from CLDR Supplemental Data, version 35.1. // Derived from CLDR Supplemental Data, version 35.1.
// https://unicode.org/Public/cldr/35.1/core.zip // https://unicode.org/Public/cldr/35.1/core.zip
bool js::intl::LanguageTag::complexLanguageMapping(const LanguageSubtag& language) { bool js::intl::LanguageTag::complexLanguageMapping(const LanguageSubtag& language) {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language.range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language.span()));
MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language.span()));
if (language.length() == 2) { if (language.length() == 2) {
return language.equalTo("sh"); return language.equalTo("sh");
@ -183,7 +212,8 @@ bool js::intl::LanguageTag::complexLanguageMapping(const LanguageSubtag& languag
// Derived from CLDR Supplemental Data, version 35.1. // Derived from CLDR Supplemental Data, version 35.1.
// https://unicode.org/Public/cldr/35.1/core.zip // https://unicode.org/Public/cldr/35.1/core.zip
bool js::intl::LanguageTag::regionMapping(RegionSubtag& region) { bool js::intl::LanguageTag::regionMapping(RegionSubtag& region) {
MOZ_ASSERT(IsStructurallyValidRegionTag(region.range())); MOZ_ASSERT(IsStructurallyValidRegionTag(region.span()));
MOZ_ASSERT(IsCanonicallyCasedRegionTag(region.span()));
if (region.length() == 2) { if (region.length() == 2) {
static const char regions[23][3] = { static const char regions[23][3] = {
@ -198,7 +228,7 @@ bool js::intl::LanguageTag::regionMapping(RegionSubtag& region) {
}; };
if (const char* replacement = SearchReplacement(regions, aliases, region)) { if (const char* replacement = SearchReplacement(regions, aliases, region)) {
region.set(ConstCharRange(replacement, strlen(replacement))); region.set(mozilla::MakeCStringSpan(replacement));
return true; return true;
} }
return false; return false;
@ -271,7 +301,7 @@ bool js::intl::LanguageTag::regionMapping(RegionSubtag& region) {
}; };
if (const char* replacement = SearchReplacement(regions, aliases, region)) { if (const char* replacement = SearchReplacement(regions, aliases, region)) {
region.set(ConstCharRange(replacement, strlen(replacement))); region.set(mozilla::MakeCStringSpan(replacement));
return true; return true;
} }
return false; return false;
@ -282,7 +312,8 @@ bool js::intl::LanguageTag::regionMapping(RegionSubtag& region) {
// Derived from CLDR Supplemental Data, version 35.1. // Derived from CLDR Supplemental Data, version 35.1.
// https://unicode.org/Public/cldr/35.1/core.zip // https://unicode.org/Public/cldr/35.1/core.zip
bool js::intl::LanguageTag::complexRegionMapping(const RegionSubtag& region) { bool js::intl::LanguageTag::complexRegionMapping(const RegionSubtag& region) {
MOZ_ASSERT(IsStructurallyValidRegionTag(region.range())); MOZ_ASSERT(IsStructurallyValidRegionTag(region.span()));
MOZ_ASSERT(IsCanonicallyCasedRegionTag(region.span()));
if (region.length() == 2) { if (region.length() == 2) {
return region.equalTo("AN") || return region.equalTo("AN") ||
@ -304,11 +335,12 @@ bool js::intl::LanguageTag::complexRegionMapping(const RegionSubtag& region) {
// Derived from CLDR Supplemental Data, version 35.1. // Derived from CLDR Supplemental Data, version 35.1.
// https://unicode.org/Public/cldr/35.1/core.zip // https://unicode.org/Public/cldr/35.1/core.zip
void js::intl::LanguageTag::performComplexLanguageMappings() { void js::intl::LanguageTag::performComplexLanguageMappings() {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language().range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language().span()));
MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language().span()));
if (language().equalTo("cnr")) { if (language().equalTo("cnr")) {
setLanguage("sr"); setLanguage("sr");
if (region().length() == 0) { if (region().missing()) {
setRegion("ME"); setRegion("ME");
} }
} }
@ -316,20 +348,20 @@ void js::intl::LanguageTag::performComplexLanguageMappings() {
language().equalTo("prs") || language().equalTo("prs") ||
language().equalTo("tnf")) { language().equalTo("tnf")) {
setLanguage("fa"); setLanguage("fa");
if (region().length() == 0) { if (region().missing()) {
setRegion("AF"); setRegion("AF");
} }
} }
else if (language().equalTo("hbs") || else if (language().equalTo("hbs") ||
language().equalTo("sh")) { language().equalTo("sh")) {
setLanguage("sr"); setLanguage("sr");
if (script().length() == 0) { if (script().missing()) {
setScript("Latn"); setScript("Latn");
} }
} }
else if (language().equalTo("swc")) { else if (language().equalTo("swc")) {
setLanguage("sw"); setLanguage("sw");
if (region().length() == 0) { if (region().missing()) {
setRegion("CD"); setRegion("CD");
} }
} }
@ -339,8 +371,10 @@ void js::intl::LanguageTag::performComplexLanguageMappings() {
// Derived from CLDR Supplemental Data, version 35.1. // Derived from CLDR Supplemental Data, version 35.1.
// https://unicode.org/Public/cldr/35.1/core.zip // https://unicode.org/Public/cldr/35.1/core.zip
void js::intl::LanguageTag::performComplexRegionMappings() { void js::intl::LanguageTag::performComplexRegionMappings() {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language().range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language().span()));
MOZ_ASSERT(IsStructurallyValidRegionTag(region().range())); MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language().span()));
MOZ_ASSERT(IsStructurallyValidRegionTag(region().span()));
MOZ_ASSERT(IsCanonicallyCasedRegionTag(region().span()));
if (region().equalTo("172")) { if (region().equalTo("172")) {
if (language().equalTo("hy") || if (language().equalTo("hy") ||
@ -562,14 +596,17 @@ bool js::intl::LanguageTag::updateGrandfatheredMappings(JSContext* cx) {
// no-nyn, zh-min, and zh-min-nan require BCP47's extlang subtag // no-nyn, zh-min, and zh-min-nan require BCP47's extlang subtag
// that |unicode_locale_id| doesn't support.) // that |unicode_locale_id| doesn't support.)
// * No RG tag contains |extensions| or |pu_extensions|. // * No RG tag contains |extensions| or |pu_extensions|.
if (script().length() != 0 || if (script().present() ||
region().length() != 0 || region().present() ||
variants().length() != 1 || variants().length() != 1 ||
extensions().length() != 0 || extensions().length() != 0 ||
privateuse()) { privateuse()) {
return true; return true;
} }
MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language().span()));
MOZ_ASSERT(IsCanonicallyCasedVariantTag(mozilla::MakeCStringSpan(variants()[0].get())));
auto variantEqualTo = [this](const char* variant) { auto variantEqualTo = [this](const char* variant) {
return strcmp(variants()[0].get(), variant) == 0; return strcmp(variants()[0].get(), variant) == 0;
}; };
@ -619,34 +656,34 @@ bool js::intl::LanguageTag::updateGrandfatheredMappings(JSContext* cx) {
} }
template <size_t Length> template <size_t Length>
static inline bool IsUnicodeKey(const ConstCharRange& key, static inline bool IsUnicodeKey(mozilla::Span<const char> key,
const char (&str)[Length]) { const char (&str)[Length]) {
static_assert(Length == UnicodeKeyLength + 1, static_assert(Length == UnicodeKeyLength + 1,
"Unicode extension key is two characters long"); "Unicode extension key is two characters long");
return memcmp(key.begin().get(), str, Length - 1) == 0; return memcmp(key.data(), str, Length - 1) == 0;
} }
template <size_t Length> template <size_t Length>
static inline bool IsUnicodeType(const ConstCharRange& type, static inline bool IsUnicodeType(mozilla::Span<const char> type,
const char (&str)[Length]) { const char (&str)[Length]) {
static_assert(Length > UnicodeKeyLength + 1, static_assert(Length > UnicodeKeyLength + 1,
"Unicode extension type contains more than two characters"); "Unicode extension type contains more than two characters");
return type.length() == (Length - 1) && return type.size() == (Length - 1) &&
memcmp(type.begin().get(), str, Length - 1) == 0; memcmp(type.data(), str, Length - 1) == 0;
} }
static int32_t CompareUnicodeType(const char* a, const ConstCharRange& b) { static int32_t CompareUnicodeType(const char* a, mozilla::Span<const char> b) {
#ifdef DEBUG #ifdef DEBUG
auto isNull = [](char c) { auto isNull = [](char c) {
return c == '\0'; return c == '\0';
}; };
#endif #endif
MOZ_ASSERT(std::none_of(b.begin().get(), b.end().get(), isNull), MOZ_ASSERT(std::none_of(b.begin(), b.end(), isNull),
"unexpected null-character in string"); "unexpected null-character in string");
using UnsignedChar = unsigned char; using UnsignedChar = unsigned char;
for (size_t i = 0; i < b.length(); i++) { for (size_t i = 0; i < b.size(); i++) {
// |a| is zero-terminated and |b| doesn't contain a null-terminator. So if // |a| is zero-terminated and |b| doesn't contain a null-terminator. So if
// we've reached the end of |a|, the below if-statement will always be true. // we've reached the end of |a|, the below if-statement will always be true.
// That ensures we don't read past the end of |a|. // That ensures we don't read past the end of |a|.
@ -657,13 +694,13 @@ static int32_t CompareUnicodeType(const char* a, const ConstCharRange& b) {
// Return zero if both strings are equal or a negative number if |b| is a // Return zero if both strings are equal or a negative number if |b| is a
// prefix of |a|. // prefix of |a|.
return -int32_t(UnsignedChar(a[b.length()])); return -int32_t(UnsignedChar(a[b.size()]));
}; };
template <size_t Length> template <size_t Length>
static inline const char* SearchReplacement(const char* (&types)[Length], static inline const char* SearchReplacement(const char* (&types)[Length],
const char* (&aliases)[Length], const char* (&aliases)[Length],
const ConstCharRange& type) { mozilla::Span<const char> type) {
auto p = std::lower_bound(std::begin(types), std::end(types), type, auto p = std::lower_bound(std::begin(types), std::end(types), type,
[](const auto& a, const auto& b) { [](const auto& a, const auto& b) {
@ -682,7 +719,7 @@ static inline const char* SearchReplacement(const char* (&types)[Length],
* Spec: https://www.unicode.org/reports/tr35/#Unicode_Locale_Extension_Data_Files * Spec: https://www.unicode.org/reports/tr35/#Unicode_Locale_Extension_Data_Files
*/ */
const char* js::intl::LanguageTag::replaceUnicodeExtensionType( const char* js::intl::LanguageTag::replaceUnicodeExtensionType(
const ConstCharRange& key, const ConstCharRange& type) { mozilla::Span<const char> key, mozilla::Span<const char> type) {
#ifdef DEBUG #ifdef DEBUG
static auto isAsciiLowercaseAlphanumeric = [](char c) { static auto isAsciiLowercaseAlphanumeric = [](char c) {
return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c); return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c);
@ -693,12 +730,12 @@ const char* js::intl::LanguageTag::replaceUnicodeExtensionType(
}; };
#endif #endif
MOZ_ASSERT(key.length() == UnicodeKeyLength); MOZ_ASSERT(key.size() == UnicodeKeyLength);
MOZ_ASSERT(std::all_of(key.begin().get(), key.end().get(), MOZ_ASSERT(std::all_of(key.begin(), key.end(),
isAsciiLowercaseAlphanumeric)); isAsciiLowercaseAlphanumeric));
MOZ_ASSERT(type.length() > UnicodeKeyLength); MOZ_ASSERT(type.size() > UnicodeKeyLength);
MOZ_ASSERT(std::all_of(type.begin().get(), type.end().get(), MOZ_ASSERT(std::all_of(type.begin(), type.end(),
isAsciiLowercaseAlphanumericOrDash)); isAsciiLowercaseAlphanumericOrDash));
if (IsUnicodeKey(key, "ca")) { if (IsUnicodeKey(key, "ca")) {

44
js/src/builtin/intl/Locale.cpp
View file

@ -12,7 +12,7 @@
#include "mozilla/Assertions.h" #include "mozilla/Assertions.h"
#include "mozilla/Casting.h" #include "mozilla/Casting.h"
#include "mozilla/Maybe.h" #include "mozilla/Maybe.h"
#include "mozilla/Range.h" #include "mozilla/Span.h"
#include "mozilla/TextUtils.h" #include "mozilla/TextUtils.h"
#include <algorithm> #include <algorithm>
@ -56,10 +56,10 @@ static inline bool IsLocale(HandleValue v) {
// Return the length of the base-name subtags. // Return the length of the base-name subtags.
static size_t BaseNameLength(const LanguageTag& tag) { static size_t BaseNameLength(const LanguageTag& tag) {
size_t baseNameLength = tag.language().length(); size_t baseNameLength = tag.language().length();
if (tag.script().length() > 0) { if (tag.script().present()) {
baseNameLength += 1 + tag.script().length(); baseNameLength += 1 + tag.script().length();
} }
if (tag.region().length() > 0) { if (tag.region().present()) {
baseNameLength += 1 + tag.region().length(); baseNameLength += 1 + tag.region().length();
} }
for (const auto& variant : tag.variants()) { for (const auto& variant : tag.variants()) {
@ -75,7 +75,7 @@ struct IndexAndLength {
IndexAndLength(size_t index, size_t length) : index(index), length(length){}; IndexAndLength(size_t index, size_t length) : index(index), length(length){};
template <typename T> template <typename T>
mozilla::Range<const T> rangeOf(const T* ptr) const { mozilla::Span<const T> spanOf(const T* ptr) const {
return {ptr + index, length}; return {ptr + index, length};
} }
}; };
@ -85,6 +85,9 @@ static mozilla::Maybe<IndexAndLength> UnicodeExtensionPosition(
const LanguageTag& tag) { const LanguageTag& tag) {
size_t index = 0; size_t index = 0;
for (const auto& extension : tag.extensions()) { for (const auto& extension : tag.extensions()) {
MOZ_ASSERT(!mozilla::IsAsciiUppercaseAlpha(extension[0]),
"extensions are case normalized to lowercase");
size_t extensionLength = strlen(extension.get()); size_t extensionLength = strlen(extension.get());
if (extension[0] == 'u') { if (extension[0] == 'u') {
return mozilla::Some(IndexAndLength{index, extensionLength}); return mozilla::Some(IndexAndLength{index, extensionLength});
@ -287,7 +290,7 @@ static bool ApplyOptionsToTag(JSContext* cx, LanguageTag& tag,
// Step 4. // Step 4.
intl::LanguageSubtag language; intl::LanguageSubtag language;
if (option && !intl::ParseStandaloneLanguagTag(option, language)) { if (option && !intl::ParseStandaloneLanguageTag(option, language)) {
if (UniqueChars str = StringToNewUTF8CharsZ(cx, *option)) { if (UniqueChars str = StringToNewUTF8CharsZ(cx, *option)) {
JS_ReportErrorNumberUTF8(cx, js::GetErrorMessage, nullptr, JS_ReportErrorNumberUTF8(cx, js::GetErrorMessage, nullptr,
JSMSG_INVALID_OPTION_VALUE, "language", JSMSG_INVALID_OPTION_VALUE, "language",
@ -329,19 +332,19 @@ static bool ApplyOptionsToTag(JSContext* cx, LanguageTag& tag,
// Step 9 (Already performed in caller). // Step 9 (Already performed in caller).
// Skip steps 10-13 when no subtags were modified. // Skip steps 10-13 when no subtags were modified.
if (language.length() > 0 || script.length() > 0 || region.length() > 0) { if (language.present() || script.present() || region.present()) {
// Step 10. // Step 10.
if (language.length() > 0) { if (language.present()) {
tag.setLanguage(language); tag.setLanguage(language);
} }
// Step 11. // Step 11.
if (script.length() > 0) { if (script.present()) {
tag.setScript(script); tag.setScript(script);
} }
// Step 12. // Step 12.
if (region.length() > 0) { if (region.present()) {
tag.setRegion(region); tag.setRegion(region);
} }
@ -378,16 +381,11 @@ static bool ApplyUnicodeExtensionToTag(JSContext* cx, LanguageTag& tag,
return false; return false;
} }
// Check if there's an existing Unicode extension subtag. (The extension // Check if there's an existing Unicode extension subtag.
// subtags aren't necessarily sorted, so we can't use binary search here.)
const UniqueChars* existingUnicodeExtension =
std::find_if(tag.extensions().begin(), tag.extensions().end(),
[](const auto& extension) { return extension[0] == 'u'; });
const char* unicodeExtensionEnd = nullptr; const char* unicodeExtensionEnd = nullptr;
const char* unicodeExtensionKeywords = nullptr; const char* unicodeExtensionKeywords = nullptr;
if (existingUnicodeExtension != tag.extensions().end()) { if (const char* unicodeExtension = tag.unicodeExtension()) {
const char* unicodeExtension = existingUnicodeExtension->get();
unicodeExtensionEnd = unicodeExtension + strlen(unicodeExtension); unicodeExtensionEnd = unicodeExtension + strlen(unicodeExtension);
SepKeywordIterator<char> iter(unicodeExtension, unicodeExtensionEnd); SepKeywordIterator<char> iter(unicodeExtension, unicodeExtensionEnd);
@ -423,8 +421,6 @@ static bool ApplyUnicodeExtensionToTag(JSContext* cx, LanguageTag& tag,
// keyword with the same key is detected as a duplicate when canonicalizing // keyword with the same key is detected as a duplicate when canonicalizing
// the Unicode extension subtag and gets discarded. // the Unicode extension subtag and gets discarded.
size_t startNewKeywords = newExtension.length();
if (calendar) { if (calendar) {
if (!appendKeyword("-ca-", calendar)) { if (!appendKeyword("-ca-", calendar)) {
return false; return false;
@ -456,12 +452,6 @@ static bool ApplyUnicodeExtensionToTag(JSContext* cx, LanguageTag& tag,
} }
} }
// Normalize the case of the new keywords.
std::transform(newExtension.begin() + startNewKeywords, newExtension.end(),
newExtension.begin() + startNewKeywords, [](char c) {
return mozilla::IsAsciiUppercaseAlpha(c) ? (c | 0x20) : c;
});
// Append the remaining keywords from the previous Unicode extension subtag. // Append the remaining keywords from the previous Unicode extension subtag.
if (unicodeExtensionKeywords) { if (unicodeExtensionKeywords) {
if (!newExtension.append(unicodeExtensionKeywords, unicodeExtensionEnd)) { if (!newExtension.append(unicodeExtensionKeywords, unicodeExtensionEnd)) {
@ -847,18 +837,18 @@ static BaseNamePartsResult BaseNameParts(const CharT* baseName, size_t length) {
} }
IndexAndLength language{0, languageLength}; IndexAndLength language{0, languageLength};
MOZ_ASSERT(intl::IsStructurallyValidLanguageTag(language.rangeOf(baseName))); MOZ_ASSERT(intl::IsStructurallyValidLanguageTag(language.spanOf(baseName)));
mozilla::Maybe<IndexAndLength> script{}; mozilla::Maybe<IndexAndLength> script{};
if (scriptIndex) { if (scriptIndex) {
script.emplace(scriptIndex, ScriptLength); script.emplace(scriptIndex, ScriptLength);
MOZ_ASSERT(intl::IsStructurallyValidScriptTag(script->rangeOf(baseName))); MOZ_ASSERT(intl::IsStructurallyValidScriptTag(script->spanOf(baseName)));
} }
mozilla::Maybe<IndexAndLength> region{}; mozilla::Maybe<IndexAndLength> region{};
if (regionIndex) { if (regionIndex) {
region.emplace(regionIndex, regionLength); region.emplace(regionIndex, regionLength);
MOZ_ASSERT(intl::IsStructurallyValidRegionTag(region->rangeOf(baseName))); MOZ_ASSERT(intl::IsStructurallyValidRegionTag(region->spanOf(baseName)));
} }
return {language, script, region}; return {language, script, region};

4
js/src/builtin/intl/SharedIntlData.cpp
View file

@ -31,9 +31,7 @@ template<typename Char>
static constexpr Char static constexpr Char
ToUpperASCII(Char c) ToUpperASCII(Char c)
{ {
return ('a' <= c && c <= 'z') return mozilla::IsAsciiLowercaseAlpha(c) ? (c - 0x20) : c;
? (c & ~0x20)
: c;
} }
static_assert(ToUpperASCII('a') == 'A', "verifying 'a' uppercases correctly"); static_assert(ToUpperASCII('a') == 'A', "verifying 'a' uppercases correctly");

100
js/src/builtin/intl/make_intl_data.py
View file

@ -68,8 +68,8 @@ def writeMappingsVar(println, mapping, name, description, source, url):
println(u' "{0}": "{1}",'.format(key, value)) println(u' "{0}": "{1}",'.format(key, value))
println(u"};") println(u"};")
def writeMappingsBinarySearch(println, fn_name, type_name, name, validate_fn, mappings, def writeMappingsBinarySearch(println, fn_name, type_name, name, validate_fn, validate_case_fn,
tag_maxlength, description, source, url): mappings, tag_maxlength, description, source, url):
""" Emit code to perform a binary search on language tag subtags. """ Emit code to perform a binary search on language tag subtags.
Uses the contents of |mapping|, which can either be a dictionary or set, Uses the contents of |mapping|, which can either be a dictionary or set,
@ -79,8 +79,9 @@ def writeMappingsBinarySearch(println, fn_name, type_name, name, validate_fn, ma
writeMappingHeader(println, description, source, url) writeMappingHeader(println, description, source, url)
println(u""" println(u"""
bool js::intl::LanguageTag::{0}({1} {2}) {{ bool js::intl::LanguageTag::{0}({1} {2}) {{
MOZ_ASSERT({3}({2}.range())); MOZ_ASSERT({3}({2}.span()));
""".format(fn_name, type_name, name, validate_fn).strip()) MOZ_ASSERT({4}({2}.span()));
""".format(fn_name, type_name, name, validate_fn, validate_case_fn).strip())
def write_array(subtags, name, length, fixed): def write_array(subtags, name, length, fixed):
if fixed: if fixed:
@ -162,7 +163,7 @@ bool js::intl::LanguageTag::{0}({1} {2}) {{
println(u""" println(u"""
if (const char* replacement = SearchReplacement({0}s, aliases, {0})) {{ if (const char* replacement = SearchReplacement({0}s, aliases, {0})) {{
{0}.set(ConstCharRange(replacement, strlen(replacement))); {0}.set(mozilla::MakeCStringSpan(replacement));
return true; return true;
}} }}
return false; return false;
@ -190,7 +191,8 @@ def writeComplexLanguageTagMappings(println, complex_language_mappings,
writeMappingHeader(println, description, source, url) writeMappingHeader(println, description, source, url)
println(u""" println(u"""
void js::intl::LanguageTag::performComplexLanguageMappings() { void js::intl::LanguageTag::performComplexLanguageMappings() {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language().range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language().span()));
MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language().span()));
""".lstrip()) """.lstrip())
# Merge duplicate language entries. # Merge duplicate language entries.
@ -227,12 +229,12 @@ void js::intl::LanguageTag::performComplexLanguageMappings() {
if script is not None: if script is not None:
println(u""" println(u"""
if (script().length() == 0) {{ if (script().missing()) {{
setScript("{}"); setScript("{}");
}}""".format(script).strip("\n")) }}""".format(script).strip("\n"))
if region is not None: if region is not None:
println(u""" println(u"""
if (region().length() == 0) {{ if (region().missing()) {{
setRegion("{}"); setRegion("{}");
}}""".format(region).strip("\n")) }}""".format(region).strip("\n"))
println(u""" println(u"""
@ -249,8 +251,10 @@ def writeComplexRegionTagMappings(println, complex_region_mappings,
writeMappingHeader(println, description, source, url) writeMappingHeader(println, description, source, url)
println(u""" println(u"""
void js::intl::LanguageTag::performComplexRegionMappings() { void js::intl::LanguageTag::performComplexRegionMappings() {
MOZ_ASSERT(IsStructurallyValidLanguageTag(language().range())); MOZ_ASSERT(IsStructurallyValidLanguageTag(language().span()));
MOZ_ASSERT(IsStructurallyValidRegionTag(region().range())); MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language().span()));
MOZ_ASSERT(IsStructurallyValidRegionTag(region().span()));
MOZ_ASSERT(IsCanonicallyCasedRegionTag(region().span()));
""".lstrip()) """.lstrip())
# |non_default_replacements| is a list and hence not hashable. Convert it # |non_default_replacements| is a list and hence not hashable. Convert it
@ -360,14 +364,17 @@ bool js::intl::LanguageTag::updateGrandfatheredMappings(JSContext* cx) {
// no-nyn, zh-min, and zh-min-nan require BCP47's extlang subtag // no-nyn, zh-min, and zh-min-nan require BCP47's extlang subtag
// that |unicode_locale_id| doesn't support.) // that |unicode_locale_id| doesn't support.)
// * No RG tag contains |extensions| or |pu_extensions|. // * No RG tag contains |extensions| or |pu_extensions|.
if (script().length() != 0 || if (script().present() ||
region().length() != 0 || region().present() ||
variants().length() != 1 || variants().length() != 1 ||
extensions().length() != 0 || extensions().length() != 0 ||
privateuse()) { privateuse()) {
return true; return true;
} }
MOZ_ASSERT(IsCanonicallyCasedLanguageTag(language().span()));
MOZ_ASSERT(IsCanonicallyCasedVariantTag(mozilla::MakeCStringSpan(variants()[0].get())));
auto variantEqualTo = [this](const char* variant) { auto variantEqualTo = [this](const char* variant) {
return strcmp(variants()[0].get(), variant) == 0; return strcmp(variants()[0].get(), variant) == 0;
};""") };""")
@ -870,7 +877,7 @@ def writeCLDRLanguageTagData(println, data, url):
println(u""" println(u"""
#include "mozilla/Assertions.h" #include "mozilla/Assertions.h"
#include "mozilla/Range.h" #include "mozilla/Span.h"
#include "mozilla/TextUtils.h" #include "mozilla/TextUtils.h"
#include <algorithm> #include <algorithm>
@ -885,7 +892,6 @@ def writeCLDRLanguageTagData(println, data, url):
#include "builtin/intl/LanguageTag.h" #include "builtin/intl/LanguageTag.h"
using namespace js::intl::LanguageTagLimits; using namespace js::intl::LanguageTagLimits;
using ConstCharRange = mozilla::Range<const char>;
template <size_t Length, size_t TagLength, size_t SubtagLength> template <size_t Length, size_t TagLength, size_t SubtagLength>
static inline bool HasReplacement( static inline bool HasReplacement(
@ -894,7 +900,7 @@ static inline bool HasReplacement(
MOZ_ASSERT(subtag.length() == TagLength - 1, MOZ_ASSERT(subtag.length() == TagLength - 1,
"subtag must have the same length as the list of subtags"); "subtag must have the same length as the list of subtags");
const char* ptr = subtag.range().begin().get(); const char* ptr = subtag.span().data();
return std::binary_search(std::begin(subtags), std::end(subtags), ptr, return std::binary_search(std::begin(subtags), std::end(subtags), ptr,
[](const char* a, const char* b) { [](const char* a, const char* b) {
return memcmp(a, b, TagLength - 1) < 0; return memcmp(a, b, TagLength - 1) < 0;
@ -909,7 +915,7 @@ static inline const char* SearchReplacement(
MOZ_ASSERT(subtag.length() == TagLength - 1, MOZ_ASSERT(subtag.length() == TagLength - 1,
"subtag must have the same length as the list of subtags"); "subtag must have the same length as the list of subtags");
const char* ptr = subtag.range().begin().get(); const char* ptr = subtag.span().data();
auto p = std::lower_bound(std::begin(subtags), std::end(subtags), ptr, auto p = std::lower_bound(std::begin(subtags), std::end(subtags), ptr,
[](const char* a, const char* b) { [](const char* a, const char* b) {
return memcmp(a, b, TagLength - 1) < 0; return memcmp(a, b, TagLength - 1) < 0;
@ -919,6 +925,34 @@ static inline const char* SearchReplacement(
} }
return nullptr; return nullptr;
} }
#ifdef DEBUG
static bool IsCanonicallyCasedLanguageTag(mozilla::Span<const char> span) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
return std::all_of(span.begin(), span.end(), mozilla::IsAsciiLowercaseAlpha<char>);
}
static bool IsCanonicallyCasedRegionTag(mozilla::Span<const char> span) {
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
return std::all_of(span.begin(), span.end(), mozilla::IsAsciiUppercaseAlpha<char>) ||
std::all_of(span.begin(), span.end(), mozilla::IsAsciiDigit<char>);
}
static bool IsCanonicallyCasedVariantTag(mozilla::Span<const char> span) {
auto isAsciiLowercaseAlphaOrDigit = [](char c) {
return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c);
};
// Tell the analysis the |std::all_of| function can't GC.
JS::AutoSuppressGCAnalysis nogc;
return std::all_of(span.begin(), span.end(), isAsciiLowercaseAlphaOrDigit);
}
#endif
""".rstrip()) """.rstrip())
source = u"CLDR Supplemental Data, version {}".format(data["version"]) source = u"CLDR Supplemental Data, version {}".format(data["version"])
@ -938,21 +972,25 @@ static inline const char* SearchReplacement(
writeMappingsBinarySearch(println, "languageMapping", writeMappingsBinarySearch(println, "languageMapping",
"LanguageSubtag&", "language", "LanguageSubtag&", "language",
"IsStructurallyValidLanguageTag", "IsStructurallyValidLanguageTag",
"IsCanonicallyCasedLanguageTag",
language_mappings, language_maxlength, language_mappings, language_maxlength,
"Mappings from language subtags to preferred values.", source, url) "Mappings from language subtags to preferred values.", source, url)
writeMappingsBinarySearch(println, "complexLanguageMapping", writeMappingsBinarySearch(println, "complexLanguageMapping",
"const LanguageSubtag&", "language", "const LanguageSubtag&", "language",
"IsStructurallyValidLanguageTag", "IsStructurallyValidLanguageTag",
"IsCanonicallyCasedLanguageTag",
complex_language_mappings.keys(), language_maxlength, complex_language_mappings.keys(), language_maxlength,
"Language subtags with complex mappings.", source, url) "Language subtags with complex mappings.", source, url)
writeMappingsBinarySearch(println, "regionMapping", writeMappingsBinarySearch(println, "regionMapping",
"RegionSubtag&", "region", "RegionSubtag&", "region",
"IsStructurallyValidRegionTag", "IsStructurallyValidRegionTag",
"IsCanonicallyCasedRegionTag",
region_mappings, region_maxlength, region_mappings, region_maxlength,
"Mappings from region subtags to preferred values.", source, url) "Mappings from region subtags to preferred values.", source, url)
writeMappingsBinarySearch(println, "complexRegionMapping", writeMappingsBinarySearch(println, "complexRegionMapping",
"const RegionSubtag&", "region", "const RegionSubtag&", "region",
"IsStructurallyValidRegionTag", "IsStructurallyValidRegionTag",
"IsCanonicallyCasedRegionTag",
complex_region_mappings.keys(), region_maxlength, complex_region_mappings.keys(), region_maxlength,
"Region subtags with complex mappings.", source, url) "Region subtags with complex mappings.", source, url)
@ -1863,34 +1901,34 @@ def updateTzdata(topsrcdir, args):
def writeUnicodeExtensionsMappings(println, mapping): def writeUnicodeExtensionsMappings(println, mapping):
println(u""" println(u"""
template <size_t Length> template <size_t Length>
static inline bool IsUnicodeKey(const ConstCharRange& key, static inline bool IsUnicodeKey(mozilla::Span<const char> key,
const char (&str)[Length]) { const char (&str)[Length]) {
static_assert(Length == UnicodeKeyLength + 1, static_assert(Length == UnicodeKeyLength + 1,
"Unicode extension key is two characters long"); "Unicode extension key is two characters long");
return memcmp(key.begin().get(), str, Length - 1) == 0; return memcmp(key.data(), str, Length - 1) == 0;
} }
template <size_t Length> template <size_t Length>
static inline bool IsUnicodeType(const ConstCharRange& type, static inline bool IsUnicodeType(mozilla::Span<const char> type,
const char (&str)[Length]) { const char (&str)[Length]) {
static_assert(Length > UnicodeKeyLength + 1, static_assert(Length > UnicodeKeyLength + 1,
"Unicode extension type contains more than two characters"); "Unicode extension type contains more than two characters");
return type.length() == (Length - 1) && return type.size() == (Length - 1) &&
memcmp(type.begin().get(), str, Length - 1) == 0; memcmp(type.data(), str, Length - 1) == 0;
} }
static int32_t CompareUnicodeType(const char* a, const ConstCharRange& b) { static int32_t CompareUnicodeType(const char* a, mozilla::Span<const char> b) {
#ifdef DEBUG #ifdef DEBUG
auto isNull = [](char c) { auto isNull = [](char c) {
return c == '\\0'; return c == '\\0';
}; };
#endif #endif
MOZ_ASSERT(std::none_of(b.begin().get(), b.end().get(), isNull), MOZ_ASSERT(std::none_of(b.begin(), b.end(), isNull),
"unexpected null-character in string"); "unexpected null-character in string");
using UnsignedChar = unsigned char; using UnsignedChar = unsigned char;
for (size_t i = 0; i < b.length(); i++) { for (size_t i = 0; i < b.size(); i++) {
// |a| is zero-terminated and |b| doesn't contain a null-terminator. So if // |a| is zero-terminated and |b| doesn't contain a null-terminator. So if
// we've reached the end of |a|, the below if-statement will always be true. // we've reached the end of |a|, the below if-statement will always be true.
// That ensures we don't read past the end of |a|. // That ensures we don't read past the end of |a|.
@ -1901,13 +1939,13 @@ static int32_t CompareUnicodeType(const char* a, const ConstCharRange& b) {
// Return zero if both strings are equal or a negative number if |b| is a // Return zero if both strings are equal or a negative number if |b| is a
// prefix of |a|. // prefix of |a|.
return -int32_t(UnsignedChar(a[b.length()])); return -int32_t(UnsignedChar(a[b.size()]));
}; };
template <size_t Length> template <size_t Length>
static inline const char* SearchReplacement(const char* (&types)[Length], static inline const char* SearchReplacement(const char* (&types)[Length],
const char* (&aliases)[Length], const char* (&aliases)[Length],
const ConstCharRange& type) { mozilla::Span<const char> type) {
auto p = std::lower_bound(std::begin(types), std::end(types), type, auto p = std::lower_bound(std::begin(types), std::end(types), type,
[](const auto& a, const auto& b) { [](const auto& a, const auto& b) {
@ -1926,7 +1964,7 @@ static inline const char* SearchReplacement(const char* (&types)[Length],
* Spec: https://www.unicode.org/reports/tr35/#Unicode_Locale_Extension_Data_Files * Spec: https://www.unicode.org/reports/tr35/#Unicode_Locale_Extension_Data_Files
*/ */
const char* js::intl::LanguageTag::replaceUnicodeExtensionType( const char* js::intl::LanguageTag::replaceUnicodeExtensionType(
const ConstCharRange& key, const ConstCharRange& type) { mozilla::Span<const char> key, mozilla::Span<const char> type) {
#ifdef DEBUG #ifdef DEBUG
static auto isAsciiLowercaseAlphanumeric = [](char c) { static auto isAsciiLowercaseAlphanumeric = [](char c) {
return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c); return mozilla::IsAsciiLowercaseAlpha(c) || mozilla::IsAsciiDigit(c);
@ -1937,12 +1975,12 @@ const char* js::intl::LanguageTag::replaceUnicodeExtensionType(
}; };
#endif #endif
MOZ_ASSERT(key.length() == UnicodeKeyLength); MOZ_ASSERT(key.size() == UnicodeKeyLength);
MOZ_ASSERT(std::all_of(key.begin().get(), key.end().get(), MOZ_ASSERT(std::all_of(key.begin(), key.end(),
isAsciiLowercaseAlphanumeric)); isAsciiLowercaseAlphanumeric));
MOZ_ASSERT(type.length() > UnicodeKeyLength); MOZ_ASSERT(type.size() > UnicodeKeyLength);
MOZ_ASSERT(std::all_of(type.begin().get(), type.end().get(), MOZ_ASSERT(std::all_of(type.begin(), type.end(),
isAsciiLowercaseAlphanumericOrDash)); isAsciiLowercaseAlphanumericOrDash));
""") """)

1
js/src/js.msg
View file

@ -485,6 +485,7 @@ MSG_DEF(JSMSG_TRACELOGGER_ENABLE_FAIL, 1, JSEXN_ERR, "enabling tracelogger faile
// Intl // Intl
MSG_DEF(JSMSG_DATE_NOT_FINITE, 2, JSEXN_RANGEERR, "date value is not finite in {0}.{1}()") MSG_DEF(JSMSG_DATE_NOT_FINITE, 2, JSEXN_RANGEERR, "date value is not finite in {0}.{1}()")
MSG_DEF(JSMSG_DUPLICATE_VARIANT_SUBTAG, 1, JSEXN_RANGEERR, "duplicate variant subtag: {0}")
MSG_DEF(JSMSG_INTERNAL_INTL_ERROR, 0, JSEXN_ERR, "internal error while computing Intl data") MSG_DEF(JSMSG_INTERNAL_INTL_ERROR, 0, JSEXN_ERR, "internal error while computing Intl data")
MSG_DEF(JSMSG_INTL_OBJECT_NOT_INITED, 3, JSEXN_TYPEERR, "Intl.{0}.prototype.{1} called on value that's not an object initialized as a {2}") MSG_DEF(JSMSG_INTL_OBJECT_NOT_INITED, 3, JSEXN_TYPEERR, "Intl.{0}.prototype.{1} called on value that's not an object initialized as a {2}")
MSG_DEF(JSMSG_INVALID_CURRENCY_CODE, 1, JSEXN_RANGEERR, "invalid currency code in NumberFormat(): {0}") MSG_DEF(JSMSG_INVALID_CURRENCY_CODE, 1, JSEXN_RANGEERR, "invalid currency code in NumberFormat(): {0}")