Issue #2259 - process Unicode langtags and locale identifiers according to BCP 47

Major spec change: text references are to BCP47 (not the implementing RFCs) and
the single source of truth is now Unicode CLDR.

- Switch from IANA to CLDR for make_unicode
- Update grandfathered tag handling directly in the parser
- Don't support extlang, irregular, privateuse or 4-letter subtags
- Adjust comments to refer to Unicode BCP 47 locale identifiers, remove RFC 5646
- Canonicalize/order langtags correctly
- Tokenize BCP47 in reusable class

Based-on: m-c 1407674(partial), 1451082, 1530320, 1522070, 1531091

js/src/builtin/RegExp.cpp

@@ -974,8 +974,7 @@ IsTrailSurrogateWithLeadSurrogate(JSContext* cx, HandleLinearString input, int32
  */
 static RegExpRunStatus
 ExecuteRegExp(JSContext* cx, HandleObject regexp, HandleString string,
-              int32_t lastIndex,
+              int32_t lastIndex, MatchPairs* matches, size_t* endIndex)
-              MatchPairs* matches, size_t* endIndex, RegExpStaticsUpdate staticsUpdate)
 {
     /*
      * WARNING: Despite the presence of spec step comment numbers, this
@@ -990,14 +989,9 @@ ExecuteRegExp(JSContext* cx, HandleObject regexp, HandleString string,
     if (!RegExpObject::getShared(cx, reobj, &re))
         return RegExpRunStatus_Error;
 
-    RegExpStatics* res;
+    RegExpStatics* res = GlobalObject::getRegExpStatics(cx, cx->global());
-    if (staticsUpdate == UpdateRegExpStatics) {
+    if (!res)
-        res = GlobalObject::getRegExpStatics(cx, cx->global());
+        return RegExpRunStatus_Error;
-        if (!res)
-            return RegExpRunStatus_Error;
-    } else {
-        res = nullptr;
-    }
 
     RootedLinearString input(cx, string->ensureLinear(cx));
     if (!input)
@@ -1051,15 +1045,14 @@ ExecuteRegExp(JSContext* cx, HandleObject regexp, HandleString string,
  * steps 3, 9-25, except 12.a.i, 12.c.i.1, 15.
  */
 static bool
-RegExpMatcherImpl(JSContext* cx, HandleObject regexp, HandleString string,
+RegExpMatcherImpl(JSContext* cx, HandleObject regexp, HandleString string, int32_t lastIndex,
-                  int32_t lastIndex, RegExpStaticsUpdate staticsUpdate, MutableHandleValue rval)
+                  MutableHandleValue rval)
 {
     /* Execute regular expression and gather matches. */
     ScopedMatchPairs matches(&cx->tempLifoAlloc());
 
     /* Steps 3, 9-14, except 12.a.i, 12.c.i.1. */
-    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, lastIndex,
+    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, lastIndex, &matches, nullptr);
-                                           &matches, nullptr, staticsUpdate);
     if (status == RegExpRunStatus_Error)
         return false;
 
@@ -1099,8 +1092,7 @@ js::RegExpMatcher(JSContext* cx, unsigned argc, Value* vp)
         return false;
 
     /* Steps 3, 9-25, except 12.a.i, 12.c.i.1, 15. */
-    return RegExpMatcherImpl(cx, regexp, string, lastIndex,
+    return RegExpMatcherImpl(cx, regexp, string, lastIndex, args.rval());
-                             UpdateRegExpStatics, args.rval());
 }
 
 /*
@@ -1123,8 +1115,7 @@ js::RegExpMatcherRaw(JSContext* cx, HandleObject regexp, HandleString input,
           return false;
       return CreateRegExpMatchResult(cx, *shared, input, *maybeMatches, output);
     }
-    return RegExpMatcherImpl(cx, regexp, input, lastIndex,
+    return RegExpMatcherImpl(cx, regexp, input, lastIndex, output);
-                             UpdateRegExpStatics, output);
 }
 
 /*
@@ -1135,14 +1126,13 @@ js::RegExpMatcherRaw(JSContext* cx, HandleObject regexp, HandleString input,
  */
 static bool
 RegExpSearcherImpl(JSContext* cx, HandleObject regexp, HandleString string,
-                   int32_t lastIndex, RegExpStaticsUpdate staticsUpdate, int32_t* result)
+                   int32_t lastIndex, int32_t* result)
 {
     /* Execute regular expression and gather matches. */
     ScopedMatchPairs matches(&cx->tempLifoAlloc());
 
     /* Steps 3, 9-14, except 12.a.i, 12.c.i.1. */
-    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, lastIndex,
+    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, lastIndex, &matches, nullptr);
-                                           &matches, nullptr, staticsUpdate);
     if (status == RegExpRunStatus_Error)
         return false;
 
@@ -1180,7 +1170,7 @@ js::RegExpSearcher(JSContext* cx, unsigned argc, Value* vp)
 
     /* Steps 3, 9-25, except 12.a.i, 12.c.i.1, 15. */
     int32_t result = 0;
-    if (!RegExpSearcherImpl(cx, regexp, string, lastIndex, UpdateRegExpStatics, &result))
+    if (!RegExpSearcherImpl(cx, regexp, string, lastIndex, &result))
         return false;
 
     args.rval().setInt32(result);
@@ -1203,23 +1193,7 @@ js::RegExpSearcherRaw(JSContext* cx, HandleObject regexp, HandleString input,
         *result = CreateRegExpSearchResult(cx, *maybeMatches);
         return true;
     }
-    return RegExpSearcherImpl(cx, regexp, input, lastIndex,
+    return RegExpSearcherImpl(cx, regexp, input, lastIndex, result);
-                              UpdateRegExpStatics, result);
-}
-
-bool
-js::regexp_exec_no_statics(JSContext* cx, unsigned argc, Value* vp)
-{
-    CallArgs args = CallArgsFromVp(argc, vp);
-    MOZ_ASSERT(args.length() == 2);
-    MOZ_ASSERT(IsRegExpObject(args[0]));
-    MOZ_ASSERT(args[1].isString());
-
-    RootedObject regexp(cx, &args[0].toObject());
-    RootedString string(cx, args[1].toString());
-
-    return RegExpMatcherImpl(cx, regexp, string, 0,
-                             DontUpdateRegExpStatics, args.rval());
 }
 
 /*
@@ -1245,8 +1219,7 @@ js::RegExpTester(JSContext* cx, unsigned argc, Value* vp)
 
     /* Steps 3, 9-14, except 12.a.i, 12.c.i.1. */
     size_t endIndex = 0;
-    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, lastIndex,
+    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, lastIndex, nullptr, &endIndex);
-                                           nullptr, &endIndex, UpdateRegExpStatics);
 
     if (status == RegExpRunStatus_Error)
         return false;
@@ -1271,8 +1244,7 @@ js::RegExpTesterRaw(JSContext* cx, HandleObject regexp, HandleString input,
     MOZ_ASSERT(lastIndex >= 0);
 
     size_t endIndexTmp = 0;
-    RegExpRunStatus status = ExecuteRegExp(cx, regexp, input, lastIndex,
+    RegExpRunStatus status = ExecuteRegExp(cx, regexp, input, lastIndex, nullptr, &endIndexTmp);
-                                           nullptr, &endIndexTmp, UpdateRegExpStatics);
 
     if (status == RegExpRunStatus_Success) {
         MOZ_ASSERT(endIndexTmp <= INT32_MAX);
@@ -1287,24 +1259,6 @@ js::RegExpTesterRaw(JSContext* cx, HandleObject regexp, HandleString input,
     return false;
 }
 
-bool
-js::regexp_test_no_statics(JSContext* cx, unsigned argc, Value* vp)
-{
-    CallArgs args = CallArgsFromVp(argc, vp);
-    MOZ_ASSERT(args.length() == 2);
-    MOZ_ASSERT(IsRegExpObject(args[0]));
-    MOZ_ASSERT(args[1].isString());
-
-    RootedObject regexp(cx, &args[0].toObject());
-    RootedString string(cx, args[1].toString());
-
-    size_t ignored = 0;
-    RegExpRunStatus status = ExecuteRegExp(cx, regexp, string, 0,
-                                           nullptr, &ignored, DontUpdateRegExpStatics);
-    args.rval().setBoolean(status == RegExpRunStatus_Success);
-    return status != RegExpRunStatus_Error;
-}
-
 static void
 GetParen(JSLinearString* matched, const JS::Value& capture, JSSubString* out)
 {

js/src/builtin/RegExp.h

@@ -18,10 +18,6 @@ namespace js {
 JSObject*
 InitRegExpClass(JSContext* cx, HandleObject obj);
 
-// Whether RegExp statics should be updated with the input and results of a
-// regular expression execution.
-enum RegExpStaticsUpdate { UpdateRegExpStatics, DontUpdateRegExpStatics };
-
 /*
  * Legacy behavior of ExecuteRegExp(), which is baked into the JSAPI.
  *
@@ -71,22 +67,6 @@ intrinsic_GetStringDataProperty(JSContext* cx, unsigned argc, Value* vp);
  * The following functions are for use by self-hosted code.
  */
 
-/*
- * Behaves like regexp.exec(string), but doesn't set RegExp statics.
- *
- * Usage: match = regexp_exec_no_statics(regexp, string)
- */
-extern MOZ_MUST_USE bool
-regexp_exec_no_statics(JSContext* cx, unsigned argc, Value* vp);
-
-/*
- * Behaves like regexp.test(string), but doesn't set RegExp statics.
- *
- * Usage: does_match = regexp_test_no_statics(regexp, string)
- */
-extern MOZ_MUST_USE bool
-regexp_test_no_statics(JSContext* cx, unsigned argc, Value* vp);
-
 /*
  * Behaves like RegExp(pattern, flags).
  * |pattern| should be a RegExp object, |flags| should be a raw integer value.

js/src/builtin/Utilities.js

@@ -80,12 +80,6 @@ MakeConstructible(Record, {});
 /********** Abstract operations defined in ECMAScript Language Specification **********/
 
 
-/* Spec: ECMAScript Language Specification, 5.1 edition, 8.12.6 and 11.8.7 */
-function HasProperty(o, p) {
-    return p in o;
-}
-
-
 /* Spec: ECMAScript Language Specification, 5.1 edition, 9.2 and 11.4.9 */
 function ToBoolean(v) {
     return !!v;

js/src/builtin/intl/Collator.js

@@ -5,18 +5,6 @@
 /********** Intl.Collator **********/
 
 
-/**
- * Mapping from Unicode extension keys for collation to options properties,
- * their types and permissible values.
- *
- * Spec: ECMAScript Internationalization API Specification, 10.1.1.
- */
-var collatorKeyMappings = {
-    kn: {property: "numeric", type: "boolean"},
-    kf: {property: "caseFirst", type: "string", values: ["upper", "lower", "false"]}
-};
-
-
 /**
  * Compute an internal properties object from |lazyCollatorData|.
  */
@@ -26,60 +14,49 @@ function resolveCollatorInternals(lazyCollatorData)
 
     var internalProps = std_Object_create(null);
 
-    // Step 7.
-    internalProps.usage = lazyCollatorData.usage;
-
-    // Step 8.
     var Collator = collatorInternalProperties;
 
-    // Step 9.
+    // Step 5.
+    internalProps.usage = lazyCollatorData.usage;
+
+    // Steps 6-7.
     var collatorIsSorting = lazyCollatorData.usage === "sort";
     var localeData = collatorIsSorting
                      ? Collator.sortLocaleData
                      : Collator.searchLocaleData;
 
     // Compute effective locale.
-    // Step 14.
+    // Step 16.
     var relevantExtensionKeys = Collator.relevantExtensionKeys;
 
-    // Step 15.
+    // Step 17.
     var r = ResolveLocale(callFunction(Collator.availableLocales, Collator),
                           lazyCollatorData.requestedLocales,
                           lazyCollatorData.opt,
                           relevantExtensionKeys,
                           localeData);
 
-    // Step 16.
+    // Step 18.
     internalProps.locale = r.locale;
 
-    // Steps 17-19.
+    // Step 19.
-    var key, property, value, mapping;
+    var collation = r.co;
-    var i = 0, len = relevantExtensionKeys.length;
-    while (i < len) {
-        // Step 19.a.
-        key = relevantExtensionKeys[i];
-        if (key === "co") {
-            // Step 19.b.
-            property = "collation";
-            value = r.co === null ? "default" : r.co;
-        } else {
-            // Step 19.c.
-            mapping = collatorKeyMappings[key];
-            property = mapping.property;
-            value = r[key];
-            if (mapping.type === "boolean")
-                value = value === "true";
-        }
 
-        // Step 19.d.
+    // Step 20.
-        internalProps[property] = value;
+    if (collation === null)
+        collation = "default";
 
-        // Step 19.e.
+    // Step 21.
-        i++;
+    internalProps.collation = collation;
-    }
+
+    // Step 22.
+    internalProps.numeric = r.kn === "true";
+
+    // Step 23.
+    internalProps.caseFirst = r.kf;
 
     // Compute remaining collation options.
-    // Steps 21-22.
+    // Step 25.
     var s = lazyCollatorData.rawSensitivity;
     if (s === undefined) {
         // In theory the default sensitivity for the "search" collator is
@@ -88,14 +65,13 @@ function resolveCollatorInternals(lazyCollatorData)
         // both collation modes.
         s = "variant";
     }
+
+    // Step 26.
     internalProps.sensitivity = s;
 
-    // Step 24.
+    // Step 28.
     internalProps.ignorePunctuation = lazyCollatorData.ignorePunctuation;
 
-    // Step 25.
-    internalProps.boundFormat = undefined;
-
     // The caller is responsible for associating |internalProps| with the right
     // object using |setInternalProperties|.
     return internalProps;
@@ -139,9 +115,6 @@ function InitializeCollator(collator, locales, options) {
     assert(IsObject(collator), "InitializeCollator called with non-object");
     assert(IsCollator(collator), "InitializeCollator called with non-Collator");
 
-    // Steps 1-2 (These steps are no longer required and should be removed
-    // from the spec; https://github.com/tc39/ecma402/issues/115).;
-
     // Lazy Collator data has the following structure:
     //
     //   {
@@ -162,11 +135,11 @@ function InitializeCollator(collator, locales, options) {
     // subset of them.
     var lazyCollatorData = std_Object_create(null);
 
-    // Step 3.
+    // Step 1.
     var requestedLocales = CanonicalizeLocaleList(locales);
     lazyCollatorData.requestedLocales = requestedLocales;
 
-    // Steps 4-5.
+    // Steps 2-3.
     //
     // If we ever need more speed here at startup, we should try to detect the
     // case where |options === undefined| and Object.prototype hasn't been
@@ -179,38 +152,39 @@ function InitializeCollator(collator, locales, options) {
         options = ToObject(options);
 
     // Compute options that impact interpretation of locale.
-    // Step 6.
+    // Step 4.
     var u = GetOption(options, "usage", "string", ["sort", "search"], "sort");
     lazyCollatorData.usage = u;
 
-    // Step 10.
+    // Step 8.
     var opt = new Record();
     lazyCollatorData.opt = opt;
 
-    // Steps 11-12.
+    // Steps 9-10.
     var matcher = GetOption(options, "localeMatcher", "string", ["lookup", "best fit"], "best fit");
     opt.localeMatcher = matcher;
 
-    // Step 13, unrolled.
+    // Steps 11-13.
     var numericValue = GetOption(options, "numeric", "boolean", undefined, undefined);
     if (numericValue !== undefined)
         numericValue = numericValue ? 'true' : 'false';
     opt.kn = numericValue;
 
+    // Steps 14-15.
     var caseFirstValue = GetOption(options, "caseFirst", "string", ["upper", "lower", "false"], undefined);
     opt.kf = caseFirstValue;
 
     // Compute remaining collation options.
-    // Step 20.
+    // Step 24.
     var s = GetOption(options, "sensitivity", "string",
                       ["base", "accent", "case", "variant"], undefined);
     lazyCollatorData.rawSensitivity = s;
 
-    // Step 23.
+    // Step 27.
     var ip = GetOption(options, "ignorePunctuation", "boolean", undefined, false);
     lazyCollatorData.ignorePunctuation = ip;
 
-    // Step 26.
+    // Step 29.
     //
     // We've done everything that must be done now: mark the lazy data as fully
     // computed and install it.
@@ -228,9 +202,14 @@ function InitializeCollator(collator, locales, options) {
 function Intl_Collator_supportedLocalesOf(locales /*, options*/) {
     var options = arguments.length > 1 ? arguments[1] : undefined;
 
+    // Step 1.
     var availableLocales = callFunction(collatorInternalProperties.availableLocales,
                                         collatorInternalProperties);
+
+    // Step 2.
     var requestedLocales = CanonicalizeLocaleList(locales);
+
+    // Step 3.
     return SupportedLocales(availableLocales, requestedLocales, options);
 }
 
@@ -353,9 +332,9 @@ function collatorSearchLocaleData() {
 
 
 /**
- * Function to be bound and returned by Intl.Collator.prototype.format.
+ * Function to be bound and returned by Intl.Collator.prototype.compare.
  *
- * Spec: ECMAScript Internationalization API Specification, 12.3.2.
+ * Spec: ECMAScript Internationalization API Specification, 10.3.3.1.
  */
 function collatorCompareToBind(x, y) {
     // Steps 1.a.i-ii implemented by ECMAScript declaration binding instantiation,
@@ -375,26 +354,28 @@ function collatorCompareToBind(x, y) {
  * than 0 if x > y according to the sort order for the locale and collation
  * options of this Collator object.
  *
- * Spec: ECMAScript Internationalization API Specification, 10.3.2.
+ * Spec: ECMAScript Internationalization API Specification, 10.3.3.
  */
 function Intl_Collator_compare_get() {
-    // Check "this Collator object" per introduction of section 10.3.
+    // Step 1.
-    if (!IsObject(this) || !IsCollator(this))
+    var collator = this;
+
+    // Steps 2-3.
+    if (!IsObject(collator) || !IsCollator(collator))
         ThrowTypeError(JSMSG_INTL_OBJECT_NOT_INITED, "Collator", "compare", "Collator");
 
-    var internals = getCollatorInternals(this);
+    var internals = getCollatorInternals(collator);
 
-    // Step 1.
+    // Step 4.
     if (internals.boundCompare === undefined) {
-        // Step 1.a.
+        // Steps 4.a-b.
-        var F = collatorCompareToBind;
+        var F = callFunction(FunctionBind, collatorCompareToBind, collator);
 
-        // Steps 1.b-d.
+        // Step 4.c.
-        var bc = callFunction(FunctionBind, F, this);
+        internals.boundCompare = F;
-        internals.boundCompare = bc;
     }
 
-    // Step 2.
+    // Step 5.
     return internals.boundCompare;
 }
 _SetCanonicalName(Intl_Collator_compare_get, "get compare");
@@ -403,28 +384,30 @@ _SetCanonicalName(Intl_Collator_compare_get, "get compare");
 /**
  * Returns the resolved options for a Collator object.
  *
- * Spec: ECMAScript Internationalization API Specification, 10.3.3 and 10.4.
+ * Spec: ECMAScript Internationalization API Specification, 10.3.4.
  */
 function Intl_Collator_resolvedOptions() {
-    // Check "this Collator object" per introduction of section 10.3.
+    // Step 1.
-    if (!IsObject(this) || !IsCollator(this))
+    var collator = this;
+
+    // Steps 2-3.
+    if (!IsObject(collator) || !IsCollator(collator))
         ThrowTypeError(JSMSG_INTL_OBJECT_NOT_INITED, "Collator", "resolvedOptions", "Collator");
 
-    var internals = getCollatorInternals(this);
+    var internals = getCollatorInternals(collator);
 
+    // Steps 4-5.
     var result = {
         locale: internals.locale,
         usage: internals.usage,
         sensitivity: internals.sensitivity,
-        ignorePunctuation: internals.ignorePunctuation
+        ignorePunctuation: internals.ignorePunctuation,
+        collation: internals.collation,
+        numeric: internals.numeric,
+        caseFirst: internals.caseFirst,
     };
 
-    var relevantExtensionKeys = collatorInternalProperties.relevantExtensionKeys;
+    // Step 6.
-    for (var i = 0; i < relevantExtensionKeys.length; i++) {
-        var key = relevantExtensionKeys[i];
-        var property = (key === "co") ? "collation" : collatorKeyMappings[key].property;
-        _DefineDataProperty(result, property, internals[property]);
-    }
     return result;
 }
 

js/src/builtin/intl/DateTimeFormat.js

@@ -53,9 +53,10 @@ function resolveDateTimeFormatInternals(lazyDateTimeFormatData) {
     // never a subset of them.
 
     var internalProps = std_Object_create(null);
+    
+    var DateTimeFormat = dateTimeFormatInternalProperties;
 
     // Compute effective locale.
-    var DateTimeFormat = dateTimeFormatInternalProperties;
 
     // Step 10.
     var localeData = DateTimeFormat.localeData;
@@ -73,7 +74,7 @@ function resolveDateTimeFormatInternals(lazyDateTimeFormatData) {
     internalProps.numberingSystem = r.nu;
 
     // Compute formatting options.
-    // Step 16.
+    // Step 14.
     var dataLocale = r.dataLocale;
 
     // Steps 20.
@@ -119,8 +120,6 @@ function resolveDateTimeFormatInternals(lazyDateTimeFormatData) {
     // Step 31.
     internalProps.pattern = pattern;
 
-    internalProps.boundFormat = undefined;
-
     // The caller is responsible for associating |internalProps| with the right
     // object using |setInternalProperties|.
     return internalProps;
@@ -297,23 +296,25 @@ function DefaultTimeZone() {
 
 
 /**
- * UnwrapDateTimeFormat(dtf)
+ * 12.1.10 UnwrapDateTimeFormat( dtf )
  */
 function UnwrapDateTimeFormat(dtf, methodName) {
-    // Step 1.
+    // Step 1 (not applicable in our implementation).
+
+    // Step 2.
     if ((!IsObject(dtf) || !IsDateTimeFormat(dtf)) &&
         dtf instanceof GetDateTimeFormatConstructor())
     {
         dtf = dtf[intlFallbackSymbol()];
     }
 
-    // Step 2.
+    // Step 3.
     if (!IsObject(dtf) || !IsDateTimeFormat(dtf)) {
         ThrowTypeError(JSMSG_INTL_OBJECT_NOT_INITED, "DateTimeFormat", methodName,
                        "DateTimeFormat");
     }
 
-    // Step 3.
+    // Step 4.
     return dtf;
 }
 
@@ -334,9 +335,6 @@ function InitializeDateTimeFormat(dateTimeFormat, thisValue, locales, options, m
     assert(IsDateTimeFormat(dateTimeFormat),
            "InitializeDateTimeFormat called with non-DateTimeFormat");
 
-    // Steps 1-2 (These steps are no longer required and should be removed
-    // from the spec; https://github.com/tc39/ecma402/issues/115).
-
     // Lazy DateTimeFormat data has the following structure:
     //
     //   {
@@ -471,6 +469,8 @@ function InitializeDateTimeFormat(dateTimeFormat, thisValue, locales, options, m
     initializeIntlObject(dateTimeFormat, "DateTimeFormat", lazyDateTimeFormatData);
 
     // 12.2.1, steps 4-5.
+    // TODO: spec issue - The current spec doesn't have the IsObject check,
+    // which means |Intl.DateTimeFormat.call(null)| is supposed to throw here.
     if (dateTimeFormat !== thisValue && thisValue instanceof GetDateTimeFormatConstructor()) {
         if (!IsObject(thisValue))
             ThrowTypeError(JSMSG_NOT_NONNULL_OBJECT, typeof thisValue);
@@ -687,17 +687,19 @@ function ToDateTimeOptions(options, required, defaults) {
     assert(typeof required === "string", "ToDateTimeOptions");
     assert(typeof defaults === "string", "ToDateTimeOptions");
 
-    // Steps 1-3.
+    // Steps 1-2.
     if (options === undefined)
         options = null;
     else
         options = ToObject(options);
     options = std_Object_create(options);
 
-    // Step 4.
+    // Step 3.
     var needDefaults = true;
 
-    // Step 5.
+    // Step 4.
+    // TODO: spec issue - The spec requires to retrieve all options, so using
+    // the ||-operator with its lazy evaluation semantics is incorrect.
     if ((required === "date" || required === "any") &&
         (options.weekday !== undefined || options.year !== undefined ||
          options.month !== undefined || options.day !== undefined))
@@ -705,7 +707,9 @@ function ToDateTimeOptions(options, required, defaults) {
         needDefaults = false;
     }
 
-    // Step 6.
+    // Step 5.
+    // TODO: spec issue - The spec requires to retrieve all options, so using
+    // the ||-operator with its lazy evaluation semantics is incorrect.
     if ((required === "time" || required === "any") &&
         (options.hour !== undefined || options.minute !== undefined ||
          options.second !== undefined))
@@ -713,7 +717,7 @@ function ToDateTimeOptions(options, required, defaults) {
         needDefaults = false;
     }
 
-    // Step 7.
+    // Step 6.
     if (needDefaults && (defaults === "date" || defaults === "all")) {
         // The specification says to call [[DefineOwnProperty]] with false for
         // the Throw parameter, while Object.defineProperty uses true. For the
@@ -724,7 +728,7 @@ function ToDateTimeOptions(options, required, defaults) {
         _DefineDataProperty(options, "day", "numeric");
     }
 
-    // Step 8.
+    // Step 7.
     if (needDefaults && (defaults === "time" || defaults === "all")) {
         // See comment for step 7.
         _DefineDataProperty(options, "hour", "numeric");
@@ -732,7 +736,7 @@ function ToDateTimeOptions(options, required, defaults) {
         _DefineDataProperty(options, "second", "numeric");
     }
 
-    // Step 9.
+    // Step 8.
     return options;
 }
 
@@ -842,14 +846,19 @@ function BestFitFormatMatcher(options, formats) {
  * matching (possibly fallback) locale. Locales appear in the same order in the
  * returned list as in the input list.
  *
- * Spec: ECMAScript Internationalization API Specification, 12.2.2.
+ * Spec: ECMAScript Internationalization API Specification, 12.3.2.
  */
 function Intl_DateTimeFormat_supportedLocalesOf(locales /*, options*/) {
     var options = arguments.length > 1 ? arguments[1] : undefined;
 
+    // Step 1.
     var availableLocales = callFunction(dateTimeFormatInternalProperties.availableLocales,
                                         dateTimeFormatInternalProperties);
+
+    // Step 2.
     var requestedLocales = CanonicalizeLocaleList(locales);
+
+    // Step 3.
     return SupportedLocales(availableLocales, requestedLocales, options);
 }
 
@@ -857,7 +866,7 @@ function Intl_DateTimeFormat_supportedLocalesOf(locales /*, options*/) {
 /**
  * DateTimeFormat internal properties.
  *
- * Spec: ECMAScript Internationalization API Specification, 9.1 and 12.2.3.
+ * Spec: ECMAScript Internationalization API Specification, 9.1 and 12.3.3.
  */
 var dateTimeFormatInternalProperties = {
     localeData: dateTimeFormatLocaleData,
@@ -897,7 +906,7 @@ function dateTimeFormatLocaleData() {
 /**
  * Function to be bound and returned by Intl.DateTimeFormat.prototype.format.
  *
- * Spec: ECMAScript Internationalization API Specification, 12.3.2.
+ * Spec: ECMAScript Internationalization API Specification, 12.1.5.
  */
 function dateTimeFormatFormatToBind() {
     // Steps 1.a.i-ii
@@ -913,7 +922,7 @@ function dateTimeFormatFormatToBind() {
  * representing the result of calling ToNumber(date) according to the
  * effective locale and the formatting options of this DateTimeFormat.
  *
- * Spec: ECMAScript Internationalization API Specification, 12.3.2.
+ * Spec: ECMAScript Internationalization API Specification, 12.4.3.
  */
 function Intl_DateTimeFormat_format_get() {
     // Steps 1-3.
@@ -923,12 +932,11 @@ function Intl_DateTimeFormat_format_get() {
 
     // Step 4.
     if (internals.boundFormat === undefined) {
-        // Step 4.a.
+        // Steps 4.a-b.
-        var F = dateTimeFormatFormatToBind;
+        var F = callFunction(FunctionBind, dateTimeFormatFormatToBind, dtf);
 
-        // Steps 4.b-d.
+        // Step 4.c.
-        var bf = callFunction(FunctionBind, F, dtf);
+        internals.boundFormat = F;
-        internals.boundFormat = bf;
     }
 
     // Step 5.
@@ -937,6 +945,11 @@ function Intl_DateTimeFormat_format_get() {
 _SetCanonicalName(Intl_DateTimeFormat_format_get, "get format");
 
 
+/**
+ * Intl.DateTimeFormat.prototype.formatToParts ( date )
+ *
+ * Spec: ECMAScript Internationalization API Specification, 12.4.4.
+ */
 function Intl_DateTimeFormat_formatToParts() {
     // Steps 1-3.
     var dtf = UnwrapDateTimeFormat(this, "formatToParts");
@@ -956,14 +969,15 @@ function Intl_DateTimeFormat_formatToParts() {
 /**
  * Returns the resolved options for a DateTimeFormat object.
  *
- * Spec: ECMAScript Internationalization API Specification, 12.3.3 and 12.4.
+ * Spec: ECMAScript Internationalization API Specification, 12.4.5.
  */
 function Intl_DateTimeFormat_resolvedOptions() {
-    // Invoke |UnwrapDateTimeFormat| per introduction of section 12.3.
+    // Steps 1-3.
     var dtf = UnwrapDateTimeFormat(this, "resolvedOptions");
 
     var internals = getDateTimeFormatInternals(dtf);
 
+    // Steps 4-5.
     var result = {
         locale: internals.locale,
         calendar: internals.calendar,
@@ -981,6 +995,8 @@ function Intl_DateTimeFormat_resolvedOptions() {
     }
 
     resolveICUPattern(internals.pattern, result);
+
+    // Step 6.
     return result;
 }
 

js/src/builtin/intl/NumberFormat.js

@@ -8,7 +8,7 @@
 /**
  * NumberFormat internal properties.
  *
- * Spec: ECMAScript Internationalization API Specification, 9.1 and 11.2.3.
+ * Spec: ECMAScript Internationalization API Specification, 9.1 and 11.3.3.
  */
 var numberFormatInternalProperties = {
     localeData: numberFormatLocaleData,
@@ -35,44 +35,38 @@ function resolveNumberFormatInternals(lazyNumberFormatData) {
 
     var internalProps = std_Object_create(null);
 
-    // Step 3.
-    var requestedLocales = lazyNumberFormatData.requestedLocales;
-
-    // Compute options that impact interpretation of locale.
-    // Step 6.
-    var opt = lazyNumberFormatData.opt;
-
     var NumberFormat = numberFormatInternalProperties;
 
-    // Step 9.
+    // Compute effective locale.
+
+    // Step 7.
     var localeData = NumberFormat.localeData;
 
-    // Step 10.
+    // Step 8.
     var r = ResolveLocale(callFunction(NumberFormat.availableLocales, NumberFormat),
                           lazyNumberFormatData.requestedLocales,
                           lazyNumberFormatData.opt,
                           NumberFormat.relevantExtensionKeys,
                           localeData);
 
-    // Steps 11-12.  (Step 13 is not relevant to our implementation.)
+    // Steps 9-10. (Step 11 is not relevant to our implementation.)
     internalProps.locale = r.locale;
     internalProps.numberingSystem = r.nu;
 
     // Compute formatting options.
-    // Step 15.
+    // Step 13.
     var s = lazyNumberFormatData.style;
     internalProps.style = s;
 
-    // Steps 19, 21.
+    // Steps 17, 19.
     if (s === "currency") {
         internalProps.currency = lazyNumberFormatData.currency;
         internalProps.currencyDisplay = lazyNumberFormatData.currencyDisplay;
     }
 
+    // Step 22.
     internalProps.minimumIntegerDigits = lazyNumberFormatData.minimumIntegerDigits;
-
     internalProps.minimumFractionDigits = lazyNumberFormatData.minimumFractionDigits;
-
     internalProps.maximumFractionDigits = lazyNumberFormatData.maximumFractionDigits;
 
     if ("minimumSignificantDigits" in lazyNumberFormatData) {
@@ -83,12 +77,9 @@ function resolveNumberFormatInternals(lazyNumberFormatData) {
         internalProps.maximumSignificantDigits = lazyNumberFormatData.maximumSignificantDigits;
     }
 
-    // Step 27.
+    // Step 24.
     internalProps.useGrouping = lazyNumberFormatData.useGrouping;
 
-    // Step 34.
-    internalProps.boundFormat = undefined;
-
     // The caller is responsible for associating |internalProps| with the right
     // object using |setInternalProperties|.
     return internalProps;
@@ -118,19 +109,21 @@ function getNumberFormatInternals(obj) {
 
 
 /**
- * UnwrapNumberFormat(nf)
+ * 11.1.11 UnwrapNumberFormat( nf )
  */
 function UnwrapNumberFormat(nf, methodName) {
-    // Step 1.
+    // Step 1 (not applicable in our implementation).
+
+    // Step 2.
     if ((!IsObject(nf) || !IsNumberFormat(nf)) && nf instanceof GetNumberFormatConstructor()) {
         nf = nf[intlFallbackSymbol()];
     }
 
-    // Step 2.
+    // Step 3.
     if (!IsObject(nf) || !IsNumberFormat(nf))
         ThrowTypeError(JSMSG_INTL_OBJECT_NOT_INITED, "NumberFormat", methodName, "NumberFormat");
 
-    // Step 3.
+    // Step 4.
     return nf;
 }
 
@@ -141,18 +134,18 @@ function UnwrapNumberFormat(nf, methodName) {
  * Spec: ECMAScript Internationalization API Specification, 11.1.1.
  */
 function SetNumberFormatDigitOptions(lazyData, options, mnfdDefault) {
-    // We skip Step 1 because we set the properties on a lazyData object.
+    // We skip step 1 because we set the properties on a lazyData object.
 
-    // Step 2-3.
+    // Steps 2-4.
     assert(IsObject(options), "SetNumberFormatDigitOptions");
     assert(typeof mnfdDefault === "number", "SetNumberFormatDigitOptions");
 
-    // Steps 4-6.
+    // Steps 5-8.
     const mnid = GetNumberOption(options, "minimumIntegerDigits", 1, 21, 1);
     const mnfd = GetNumberOption(options, "minimumFractionDigits", 0, 20, mnfdDefault);
     const mxfd = GetNumberOption(options, "maximumFractionDigits", mnfd, 20);
 
-    // Steps 7-8.
+    // Steps 9-10.
     let mnsd = options.minimumSignificantDigits;
     let mxsd = options.maximumSignificantDigits;
 
@@ -196,17 +189,9 @@ function toASCIIUpperCase(s) {
  *
  * Spec: ECMAScript Internationalization API Specification, 6.3.1.
  */
-function getIsWellFormedCurrencyCodeRE() {
-    return internalIntlRegExps.isWellFormedCurrencyCodeRE ||
-           (internalIntlRegExps.isWellFormedCurrencyCodeRE = RegExpCreate("[^A-Z]"));
-}
-
 function IsWellFormedCurrencyCode(currency) {
-    var c = ToString(currency);
+    assert(typeof currency === "string", "currency is a string value");
-    var normalized = toASCIIUpperCase(c);
+    return currency.length === 3 && IsASCIIAlphaString(currency);
-    if (normalized.length !== 3)
-        return false;
-    return !regexp_test_no_statics(getIsWellFormedCurrencyCodeRE(), normalized);
 }
 
 /**
@@ -218,15 +203,12 @@ function IsWellFormedCurrencyCode(currency) {
  * This later work occurs in |resolveNumberFormatInternals|; steps not noted
  * here occur there.
  *
- * Spec: ECMAScript Internationalization API Specification, 11.1.1.
+ * Spec: ECMAScript Internationalization API Specification, 11.1.2.
  */
 function InitializeNumberFormat(numberFormat, thisValue, locales, options) {
     assert(IsObject(numberFormat), "InitializeNumberFormat called with non-object");
     assert(IsNumberFormat(numberFormat), "InitializeNumberFormat called with non-NumberFormat");
 
-    // Steps 1-2 (These steps are no longer required and should be removed
-    // from the spec; https://github.com/tc39/ecma402/issues/115).
-
     // Lazy NumberFormat data has the following structure:
     //
     //   {
@@ -258,11 +240,11 @@ function InitializeNumberFormat(numberFormat, thisValue, locales, options) {
     // subset of them.
     var lazyNumberFormatData = std_Object_create(null);
 
-    // Step 3.
+    // Step 1.
     var requestedLocales = CanonicalizeLocaleList(locales);
     lazyNumberFormatData.requestedLocales = requestedLocales;
 
-    // Steps 4-5.
+    // Steps 2-3.
     //
     // If we ever need more speed here at startup, we should try to detect the
     // case where |options === undefined| and Object.prototype hasn't been
@@ -275,20 +257,20 @@ function InitializeNumberFormat(numberFormat, thisValue, locales, options) {
         options = ToObject(options);
 
     // Compute options that impact interpretation of locale.
-    // Step 6.
+    // Step 4.
     var opt = new Record();
     lazyNumberFormatData.opt = opt;
 
-    // Steps 7-8.
+    // Steps 5-6.
     var matcher = GetOption(options, "localeMatcher", "string", ["lookup", "best fit"], "best fit");
     opt.localeMatcher = matcher;
 
     // Compute formatting options.
-    // Step 14.
+    // Step 12.
     var s = GetOption(options, "style", "string", ["decimal", "percent", "currency"], "decimal");
     lazyNumberFormatData.style = s;
 
-    // Steps 16-19.
+    // Steps 14-17.
     var c = GetOption(options, "currency", "string", undefined, undefined);
     if (c !== undefined && !IsWellFormedCurrencyCode(c))
         ThrowRangeError(JSMSG_INVALID_CURRENCY_CODE, c);
@@ -303,12 +285,12 @@ function InitializeNumberFormat(numberFormat, thisValue, locales, options) {
         cDigits = CurrencyDigits(c);
     }
 
-    // Step 20.
+    // Step 18.
     var cd = GetOption(options, "currencyDisplay", "string", ["code", "symbol", "name"], "symbol");
     if (s === "currency")
         lazyNumberFormatData.currencyDisplay = cd;
 
-    // Steps 22-24.
+    // Steps 20-22.
     SetNumberFormatDigitOptions(lazyNumberFormatData, options, s === "currency" ? cDigits: 0);
 
     // Step 25.
@@ -322,16 +304,19 @@ function InitializeNumberFormat(numberFormat, thisValue, locales, options) {
             std_Math_max(lazyNumberFormatData.minimumFractionDigits, mxfdDefault);
     }
 
-    // Step 26.
+    // Steps 23.
     var g = GetOption(options, "useGrouping", "boolean", undefined, true);
     lazyNumberFormatData.useGrouping = g;
 
-    // Steps 35-36.
+    // Step 31.
     //
     // We've done everything that must be done now: mark the lazy data as fully
     // computed and install it.
     initializeIntlObject(numberFormat, "NumberFormat", lazyNumberFormatData);
 
+    // 11.2.1, steps 4-5.
+    // TODO: spec issue - The current spec doesn't have the IsObject check,
+    // which means |Intl.NumberFormat.call(null)| is supposed to throw here.
     if (numberFormat !== thisValue && thisValue instanceof GetNumberFormatConstructor()) {
         if (!IsObject(thisValue))
             ThrowTypeError(JSMSG_NOT_NONNULL_OBJECT, typeof thisValue);
@@ -342,6 +327,7 @@ function InitializeNumberFormat(numberFormat, thisValue, locales, options) {
         return thisValue;
     }
 
+    // 11.2.1, step 6.
     return numberFormat;
 }
 
@@ -386,15 +372,12 @@ var currencyDigits = {
 /**
  * Returns the number of decimal digits to be used for the given currency.
  *
- * Spec: ECMAScript Internationalization API Specification, 11.1.1.
+ * Spec: ECMAScript Internationalization API Specification, 11.1.3.
  */
-function getCurrencyDigitsRE() {
-    return internalIntlRegExps.currencyDigitsRE ||
-           (internalIntlRegExps.currencyDigitsRE = RegExpCreate("^[A-Z]{3}$"));
-}
 function CurrencyDigits(currency) {
-    assert(typeof currency === "string", "CurrencyDigits");
+    assert(typeof currency === "string", "currency is a string value");
-    assert(regexp_test_no_statics(getCurrencyDigitsRE(), currency), "CurrencyDigits");
+    assert(IsWellFormedCurrencyCode(currency), "currency is well-formed");
+    assert(currency == toASCIIUpperCase(currency), "currency is all upper-case");
 
     if (hasOwn(currency, currencyDigits))
         return currencyDigits[currency];
@@ -407,14 +390,19 @@ function CurrencyDigits(currency) {
  * matching (possibly fallback) locale. Locales appear in the same order in the
  * returned list as in the input list.
  *
- * Spec: ECMAScript Internationalization API Specification, 11.2.2.
+ * Spec: ECMAScript Internationalization API Specification, 11.3.2.
  */
 function Intl_NumberFormat_supportedLocalesOf(locales /*, options*/) {
     var options = arguments.length > 1 ? arguments[1] : undefined;
 
+    // Step 1.
     var availableLocales = callFunction(numberFormatInternalProperties.availableLocales,
                                         numberFormatInternalProperties);
+
+    // Step 2.
     var requestedLocales = CanonicalizeLocaleList(locales);
+
+    // Step 3.
     return SupportedLocales(availableLocales, requestedLocales, options);
 }
 
@@ -427,8 +415,8 @@ function getNumberingSystems(locale) {
     // Algorithmic numbering systems are typically tied to one locale, so for
     // lack of information we don't offer them. To increase chances that
     // other software will process output correctly, we further restrict to
-    // those decimal numbering systems explicitly listed in table 2 of
+    // those decimal numbering systems explicitly listed in table 3 of
-    // the ECMAScript Internationalization API Specification, 11.3.2, which
+    // the ECMAScript Internationalization API Specification, 11.1.6, which
     // in turn are those with full specifications in version 21 of Unicode
     // Technical Standard #35 using digits that were defined in Unicode 5.0,
     // the Unicode version supported in Windows Vista.
@@ -459,7 +447,7 @@ function numberFormatLocaleData() {
 /**
  * Function to be bound and returned by Intl.NumberFormat.prototype.format.
  *
- * Spec: ECMAScript Internationalization API Specification, 11.3.2.
+ * Spec: ECMAScript Internationalization API Specification, 11.1.4.
  */
 function numberFormatFormatToBind(value) {
     // Steps 1.a.i implemented by ECMAScript declaration binding instantiation,
@@ -476,7 +464,7 @@ function numberFormatFormatToBind(value) {
  * representing the result of calling ToNumber(value) according to the
  * effective locale and the formatting options of this NumberFormat.
  *
- * Spec: ECMAScript Internationalization API Specification, 11.3.2.
+ * Spec: ECMAScript Internationalization API Specification, 11.4.3.
  */
 function Intl_NumberFormat_format_get() {
     // Steps 1-3.
@@ -486,12 +474,11 @@ function Intl_NumberFormat_format_get() {
 
     // Step 4.
     if (internals.boundFormat === undefined) {
-        // Step 4.a.
+        // Steps 4.a-b.
-        var F = numberFormatFormatToBind;
+        var F = callFunction(FunctionBind, numberFormatFormatToBind, nf);
 
-        // Steps 4.b-d.
+        // Step 4.c.
-        var bf = callFunction(FunctionBind, F, nf);
+        internals.boundFormat = F;
-        internals.boundFormat = bf;
     }
 
     // Step 5.
@@ -499,6 +486,9 @@ function Intl_NumberFormat_format_get() {
 }
 _SetCanonicalName(Intl_NumberFormat_format_get, "get format");
 
+/**
+ * 11.4.4 Intl.NumberFormat.prototype.formatToParts ( value )
+ */
 function Intl_NumberFormat_formatToParts(value) {
     // Steps 1-3.
     var nf = UnwrapNumberFormat(this, "formatToParts");
@@ -516,14 +506,15 @@ function Intl_NumberFormat_formatToParts(value) {
 /**
  * Returns the resolved options for a NumberFormat object.
  *
- * Spec: ECMAScript Internationalization API Specification, 11.3.3 and 11.4.
+ * Spec: ECMAScript Internationalization API Specification, 11.4.5.
  */
 function Intl_NumberFormat_resolvedOptions() {
-    // Invoke |UnwrapNumberFormat| per introduction of section 11.3.
+    // Steps 1-3.
     var nf = UnwrapNumberFormat(this, "resolvedOptions");
 
     var internals = getNumberFormatInternals(nf);
 
+    // Steps 4-5.
     var result = {
         locale: internals.locale,
         numberingSystem: internals.numberingSystem,
@@ -533,17 +524,31 @@ function Intl_NumberFormat_resolvedOptions() {
         maximumFractionDigits: internals.maximumFractionDigits,
         useGrouping: internals.useGrouping
     };
-    var optionalProperties = [
+
-        "currency",
+    // currency and currencyDisplay are only present for currency formatters.
-        "currencyDisplay",
+    assert(hasOwn("currency", internals) === (internals.style === "currency"),
-        "minimumSignificantDigits",
+           "currency is present iff style is 'currency'");
-        "maximumSignificantDigits"
+    assert(hasOwn("currencyDisplay", internals) === (internals.style === "currency"),
-    ];
+           "currencyDisplay is present iff style is 'currency'");
-    for (var i = 0; i < optionalProperties.length; i++) {
+
-        var p = optionalProperties[i];
+    if (hasOwn("currency", internals)) {
-        if (hasOwn(p, internals))
+        _DefineDataProperty(result, "currency", internals.currency);
-            _DefineDataProperty(result, p, internals[p]);
+        _DefineDataProperty(result, "currencyDisplay", internals.currencyDisplay);
     }
+
+    // Min/Max significant digits are either both present or not at all.
+    assert(hasOwn("minimumSignificantDigits", internals) ===
+           hasOwn("maximumSignificantDigits", internals),
+           "minimumSignificantDigits is present iff maximumSignificantDigits is present");
+
+    if (hasOwn("minimumSignificantDigits", internals)) {
+        _DefineDataProperty(result, "minimumSignificantDigits",
+                            internals.minimumSignificantDigits);
+        _DefineDataProperty(result, "maximumSignificantDigits",
+                            internals.maximumSignificantDigits);
+    }
+
+    // Step 6.
     return result;
 }
 

js/src/builtin/intl/PluralRules.cpp

@@ -79,7 +79,7 @@ static const JSFunctionSpec pluralRules_methods[] = {
 
 /**
  * PluralRules constructor.
- * Spec: ECMAScript 402 API, PluralRules, 1.1
+ * Spec: ECMAScript 402 API, PluralRules, 13.2.1
  */
 static bool
 PluralRules(JSContext* cx, const CallArgs& args, bool construct)

js/src/builtin/intl/PluralRules.js

@@ -7,7 +7,7 @@
 /**
  * PluralRules internal properties.
  *
- * Spec: ECMAScript 402 API, PluralRules, 1.3.3.
+ * Spec: ECMAScript 402 API, PluralRules, 13.3.3.
  */
 var pluralRulesInternalProperties = {
     localeData: pluralRulesLocaleData,
@@ -44,20 +44,25 @@ function resolvePluralRulesInternals(lazyPluralRulesData) {
 
     var PluralRules = pluralRulesInternalProperties;
 
-    // Step 13.
+    // Compute effective locale.
-    const r = ResolveLocale(callFunction(PluralRules.availableLocales, PluralRules),
-                          lazyPluralRulesData.requestedLocales,
-                          lazyPluralRulesData.opt,
-                          PluralRules.relevantExtensionKeys, PluralRules.localeData);
 
-    // Step 14.
+    // Step 10.
+    var localeData = PluralRules.localeData;
+
+    // Step 11.
+    const r = ResolveLocale(callFunction(PluralRules.availableLocales, PluralRules),
+                            lazyPluralRulesData.requestedLocales,
+                            lazyPluralRulesData.opt,
+                            PluralRules.relevantExtensionKeys,
+                            localeData);
+
+    // Step 12.
     internalProps.locale = r.locale;
+
+    // Step 8.
     internalProps.type = lazyPluralRulesData.type;
 
-    internalProps.pluralCategories = intl_GetPluralCategories(
+    // Step 9.
-        internalProps.locale,
-        internalProps.type);
-
     internalProps.minimumIntegerDigits = lazyPluralRulesData.minimumIntegerDigits;
     internalProps.minimumFractionDigits = lazyPluralRulesData.minimumFractionDigits;
     internalProps.maximumFractionDigits = lazyPluralRulesData.maximumFractionDigits;
@@ -68,6 +73,9 @@ function resolvePluralRulesInternals(lazyPluralRulesData) {
         internalProps.maximumSignificantDigits = lazyPluralRulesData.maximumSignificantDigits;
     }
 
+    // Step 13 (lazily computed on first access).
+    internalProps.pluralCategories = null;
+
     return internalProps;
 }
 
@@ -99,15 +107,12 @@ function getPluralRulesInternals(obj) {
  * This later work occurs in |resolvePluralRulesInternals|; steps not noted
  * here occur there.
  *
- * Spec: ECMAScript 402 API, PluralRules, 1.1.1.
+ * Spec: ECMAScript 402 API, PluralRules, 13.1.1.
  */
 function InitializePluralRules(pluralRules, locales, options) {
     assert(IsObject(pluralRules), "InitializePluralRules called with non-object");
     assert(IsPluralRules(pluralRules), "InitializePluralRules called with non-PluralRules");
 
-    // Steps 1-2 (These steps are no longer required and should be removed
-    // from the spec; https://github.com/tc39/ecma402/issues/115).
-
     // Lazy PluralRules data has the following structure:
     //
     //   {
@@ -133,30 +138,29 @@ function InitializePluralRules(pluralRules, locales, options) {
     // subset of them.
     const lazyPluralRulesData = std_Object_create(null);
 
-    // Step 3.
+    // Step 1.
     let requestedLocales = CanonicalizeLocaleList(locales);
     lazyPluralRulesData.requestedLocales = requestedLocales;
 
-    // Steps 4-5.
+    // Steps 2-3.
     if (options === undefined)
         options = {};
     else
         options = ToObject(options);
 
-    // Step 6.
+    // Step 4.
-    const type = GetOption(options, "type", "string", ["cardinal", "ordinal"], "cardinal");
-    lazyPluralRulesData.type = type;
-
-    // Step 8.
     let opt = new Record();
     lazyPluralRulesData.opt = opt;
 
-    // Steps 9-10.
+    // Steps 5-6.
     let matcher = GetOption(options, "localeMatcher", "string", ["lookup", "best fit"], "best fit");
     opt.localeMatcher = matcher;
 
+    // Step 7.
+    const type = GetOption(options, "type", "string", ["cardinal", "ordinal"], "cardinal");
+    lazyPluralRulesData.type = type;
 
-    // Step 11.
+    // Step 9.
     SetNumberFormatDigitOptions(lazyPluralRulesData, options, 0);
 
     // Step 12.
@@ -165,6 +169,10 @@ function InitializePluralRules(pluralRules, locales, options) {
            std_Math_max(lazyPluralRulesData.minimumFractionDigits, 3);
     }
 
+    // Step 15.
+    //
+    // We've done everything that must be done now: mark the lazy data as fully
+    // computed and install it.
     initializeIntlObject(pluralRules, "PluralRules", lazyPluralRulesData)
 }
 
@@ -173,7 +181,7 @@ function InitializePluralRules(pluralRules, locales, options) {
  * matching (possibly fallback) locale. Locales appear in the same order in the
  * returned list as in the input list.
  *
- * Spec: ECMAScript 402 API, PluralRules, 1.3.2.
+ * Spec: ECMAScript 402 API, PluralRules, 13.3.2.
  */
 function Intl_PluralRules_supportedLocalesOf(locales /*, options*/) {
     var options = arguments.length > 1 ? arguments[1] : undefined;
@@ -193,20 +201,20 @@ function Intl_PluralRules_supportedLocalesOf(locales /*, options*/) {
  * the number passed as value according to the
  * effective locale and the formatting options of this PluralRules.
  *
- * Spec: ECMAScript 402 API, PluralRules, 1.4.3.
+ * Spec: ECMAScript 402 API, PluralRules, 13.4.3.
  */
 function Intl_PluralRules_select(value) {
     // Step 1.
     let pluralRules = this;
 
-    // Step 2.
+    // Steps 2-3.
     if (!IsObject(pluralRules) || !IsPluralRules(pluralRules))
         ThrowTypeError(JSMSG_INTL_OBJECT_NOT_INITED, "PluralRules", "select", "PluralRules");
 
     // Ensure the PluralRules internals are resolved.
     getPluralRulesInternals(pluralRules);
 
-    // Steps 3-4.
+    // Step 4.
     let n = ToNumber(value);
 
     // Step 5.
@@ -216,17 +224,34 @@ function Intl_PluralRules_select(value) {
 /**
  * Returns the resolved options for a PluralRules object.
  *
- * Spec: ECMAScript 402 API, PluralRules, 1.4.4.
+ * Spec: ECMAScript 402 API, PluralRules, 13.4.4.
  */
 function Intl_PluralRules_resolvedOptions() {
-    // Check "this PluralRules object" per introduction of section 1.4.
+    // Step 1.
-    if (!IsObject(this) || !IsPluralRules(this)) {
+    var pluralRules = this;
+
+    // Steps 2-3.
+    if (!IsObject(pluralRules) || !IsPluralRules(pluralRules)) {
         ThrowTypeError(JSMSG_INTL_OBJECT_NOT_INITED, "PluralRules", "resolvedOptions",
                        "PluralRules");
     }
 
-    var internals = getPluralRulesInternals(this);
+    var internals = getPluralRulesInternals(pluralRules);
 
+    var internalsPluralCategories = internals.pluralCategories;
+    if (internalsPluralCategories === null) {
+        internalsPluralCategories = intl_GetPluralCategories(internals.locale, internals.type);
+        internals.pluralCategories = internalsPluralCategories;
+    }
+
+    // TODO: The current spec actually requires to return the internal array
+    // object and not a copy of it.
+    // <https://github.com/tc39/proposal-intl-plural-rules/issues/28#issuecomment-341557030>
+    var pluralCategories = [];
+    for (var i = 0; i < internalsPluralCategories.length; i++)
+        _DefineDataProperty(pluralCategories, i, internalsPluralCategories[i]);
+
+    // Steps 4-5.
     var result = {
         locale: internals.locale,
         type: internals.type,
@@ -236,16 +261,19 @@ function Intl_PluralRules_resolvedOptions() {
         maximumFractionDigits: internals.maximumFractionDigits,
     };
 
-    var optionalProperties = [
+    // Min/Max significant digits are either both present or not at all.
-        "minimumSignificantDigits",
+    assert(hasOwn("minimumSignificantDigits", internals) ===
-        "maximumSignificantDigits"
+           hasOwn("maximumSignificantDigits", internals),
-    ];
+           "minimumSignificantDigits is present iff maximumSignificantDigits is present");
 
-    for (var i = 0; i < optionalProperties.length; i++) {
+    if (hasOwn("minimumSignificantDigits", internals)) {
-        var p = optionalProperties[i];
+        _DefineDataProperty(result, "minimumSignificantDigits",
-        if (hasOwn(p, internals))
+                            internals.minimumSignificantDigits);
-            _DefineDataProperty(result, p, internals[p]);
+        _DefineDataProperty(result, "maximumSignificantDigits",
+                            internals.maximumSignificantDigits);
     }
+
+    // Step 6.
     return result;
 }
 

js/src/builtin/intl/make_intl_data.py

@@ -6,19 +6,14 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """ Usage:
-    make_intl_data.py langtags [language-subtag-registry.txt]
+    make_intl_data.py langtags [ldmlSupplemental.dtd supplementalMetadata.xml likelySubtags.xml]
     make_intl_data.py tzdata
 
     Target "langtags":
     This script extracts information about mappings between deprecated and
-    current BCP 47 language tags from the IANA Language Subtag Registry and
+    current Unicode BCP 47 locale identifiers from CLDR and converts it to
-    converts it to JavaScript object definitions in
+    JavaScript object definitions in LangTagMappingsGenerated.js. The
-    LangTagMappingsGenerated.js. The definitions are used in Intl.js.
+    definitions are used in Intl.js.
-
-    The IANA Language Subtag Registry is imported from
-    https://www.iana.org/assignments/language-subtag-registry
-    and uses the syntax specified in
-    https://tools.ietf.org/html/rfc5646#section-3
 
 
     Target "tzdata":
@@ -32,202 +27,714 @@ import os
 import re
 import io
 import codecs
+import shutil
+import subprocess
 import sys
 import tarfile
 import tempfile
 import urllib2
-import urlparse
+from contextlib import closing, contextmanager
-from contextlib import closing
 from functools import partial
 from itertools import chain, ifilter, ifilterfalse, imap, tee
 from operator import attrgetter, itemgetter
+from urlparse import urlsplit, urlunsplit
 
-def readRegistryRecord(registry):
+def writeMappingHeader(println, description, source, url):
-    """ Yields the records of the IANA Language Subtag Registry as dictionaries. """
+    if type(description) is not list:
-    record = {}
+        description = [description]
-    for line in registry:
+    for desc in description:
-        line = line.strip()
+        println(u"// {0}".format(desc))
-        if line == "":
+    println(u"// Derived from {0}.".format(source))
-            continue
+    println(u"// {0}".format(url))
-        if line == "%%":
+
-            yield record
+def writeMappingsVar(println, mapping, name, description, source, url):
-            record = {}
+    """ Writes a variable definition with a mapping table.
-        else:
+
-            if ":" in line:
+        Writes the contents of dictionary |mapping| through the |println|
-                key, value = line.split(":", 1)
+        function with the given variable name and a comment with description,
-                key, value = key.strip(), value.strip()
+        source, and URL.
-                record[key] = value
+    """
+    println(u"")
+    writeMappingHeader(println, description, source, url)
+    println(u"var {0} = {{".format(name))
+    for key in sorted(mapping):
+        if not isinstance(mapping[key], dict):
+            value = mapping[key]
+            if isinstance(value, bool):
+                value = "true" if value else "false"
             else:
-                # continuation line
+                value = '"{0}"'.format(value)
-                record[key] += " " + line
+        else:
-    if record:
+            preferred = mapping[key]["preferred"]
-        yield record
+            prefix = mapping[key]["prefix"]
-    return
+            if key != preferred:
+                raise Exception("Expected '{0}' matches preferred locale '{1}'".format(key, preferred))
+            value = '"{0}"'.format(prefix)
+        println(u'    "{0}": {1},'.format(key, value))
+    println(u"};")
+
+def writeUpdateLocaleIdMappingsFunction(println,
+                                        complex_language_mappings,
+                                        complex_region_mappings,
+                                        description, source, url):
+    """ Writes a function definition that performs language tag mapping. """
+    println(u"")
+    writeMappingHeader(println, description, source, url)
+    println(u"""\
+/* eslint-disable complexity */
+function updateLocaleIdMappings(tag) {
+    assert(IsObject(tag), "tag is an object");
+
+    // Replace deprecated language tags with their preferred values.
+    var language = tag.language;
+    if (hasOwn(language, languageMappings)) {
+        tag.language = languageMappings[language];
+    } else if (hasOwn(language, complexLanguageMappings)) {
+        switch (language) {""")
+
+    # Merge duplicate language entries.
+    language_aliases = {}
+    for (deprecated_language, (language, script, region)) in (
+        sorted(complex_language_mappings.items(), key=itemgetter(0))
+    ):
+        key = (language, script, region)
+        if key not in language_aliases:
+            language_aliases[key] = []
+        else:
+            language_aliases[key].append(deprecated_language)
+
+    for (deprecated_language, (language, script, region)) in (
+        sorted(complex_language_mappings.items(), key=itemgetter(0))
+    ):
+        key = (language, script, region)
+        if deprecated_language in language_aliases[key]:
+            continue
+
+        for lang in [deprecated_language] + language_aliases[key]:
+            println(u"""
+          case "{}":
+            """.format(lang).rstrip().strip("\n"))
+
+        println(u"""
+            tag.language = "{}";
+        """.format(language).rstrip().strip("\n"))
+        if script is not None:
+            println(u"""
+            if (tag.script === undefined)
+                tag.script = "{}";
+            """.format(script).rstrip().strip("\n"))
+        if region is not None:
+            println(u"""
+            if (tag.region === undefined)
+                tag.region = "{}";
+            """.format(region).rstrip().strip("\n"))
+        println(u"""
+            break;
+        """.rstrip().strip("\n"))
+
+    println(u"""
+          default:
+            assert(false, "language not handled: " + language);
+        }
+    }
+
+    // No script replacements are currently present.
+
+    // Replace deprecated subtags with their preferred values.
+    var region = tag.region;
+    if (region !== undefined) {
+        if (hasOwn(region, regionMappings)) {
+            tag.region = regionMappings[region];
+        } else if (hasOwn(region, complexRegionMappings)) {
+            switch (region) {""".lstrip("\n"))
+
+    # |non_default_replacements| is a list and hence not hashable. Convert it
+    # to a string to get a proper hashable value.
+    def hash_key(default, non_default_replacements):
+        return (default, str(sorted(str(v) for v in non_default_replacements)))
+
+    # Merge duplicate region entries.
+    region_aliases = {}
+    for (deprecated_region, (default, non_default_replacements)) in (
+        sorted(complex_region_mappings.items(), key=itemgetter(0))
+    ):
+        key = hash_key(default, non_default_replacements)
+        if key not in region_aliases:
+            region_aliases[key] = []
+        else:
+            region_aliases[key].append(deprecated_region)
+
+    for (deprecated_region, (default, non_default_replacements)) in (
+        sorted(complex_region_mappings.items(), key=itemgetter(0))
+    ):
+        key = hash_key(default, non_default_replacements)
+        if deprecated_region in region_aliases[key]:
+            continue
+
+        for region in [deprecated_region] + region_aliases[key]:
+            println(u"""
+              case "{}":
+            """.format(region).rstrip().strip("\n"))
+
+        for (language, script, region) in sorted(non_default_replacements, key=itemgetter(0)):
+            if script is None:
+                println(u"""
+                if (tag.language === "{}") {{
+                """.format(language).rstrip().strip("\n"))
+            else:
+                println(u"""
+                if (tag.language === "{}" && tag.script === "{}") {{
+                """.format(language, script).rstrip().strip("\n"))
+            println(u"""
+                    tag.region = "{}";
+                    break;
+                }}
+            """.format(region).rstrip().strip("\n"))
+
+        println(u"""
+                tag.region = "{}";
+                break;
+        """.format(default).rstrip().strip("\n"))
+
+    println(u"""
+              default:
+                assert(false, "region not handled: " + region);
+            }
+        }
+
+        // No variant replacements are currently present.
+        // No extension replacements are currently present.
+        // Private use sequences are left as is.
+
+    }
+}
+/* eslint-enable complexity */
+""".strip("\n"))
 
 
-def readRegistry(registry):
+def writeGrandfatheredMappingsFunction(println,
-    """ Reads IANA Language Subtag Registry and extracts information for Intl.js.
+                                       grandfathered_mappings,
+                                       description, source, url):
+    """ Writes a function definition that maps grandfathered language tags. """
+    println(u"")
+    writeMappingHeader(println, description, source, url)
+    println(u"""\
+function updateGrandfatheredMappings(tag) {
+    assert(IsObject(tag), "tag is an object");
+
+    // We're mapping regular grandfathered tags to non-grandfathered form here.
+    // Other tags remain unchanged.
+    //
+    // regular       = "art-lojban"
+    //               / "cel-gaulish"
+    //               / "no-bok"
+    //               / "no-nyn"
+    //               / "zh-guoyu"
+    //               / "zh-hakka"
+    //               / "zh-min"
+    //               / "zh-min-nan"
+    //               / "zh-xiang"
+    //
+    // Therefore we can quickly exclude most tags by checking every
+    // |unicode_locale_id| subcomponent for characteristics not shared by any of
+    // the regular grandfathered (RG) tags:
+    //
+    //   * Real-world |unicode_language_subtag|s are all two or three letters,
+    //     so don't waste time running a useless |language.length > 3| fast-path.
+    //   * No RG tag has a "script"-looking component.
+    //   * No RG tag has a "region"-looking component.
+    //   * The RG tags that match |unicode_locale_id| (art-lojban, cel-gaulish,
+    //     zh-guoyu, zh-hakka, zh-xiang) have exactly one "variant". (no-bok,
+    //     no-nyn, zh-min, and zh-min-nan require BCP47's extlang subtag
+    //     that |unicode_locale_id| doesn't support.)
+    //   * No RG tag contains |extensions| or |pu_extensions|.
+    if (tag.script !== undefined ||
+        tag.region !== undefined ||
+        tag.variants.length !== 1 ||
+        tag.extensions.length !== 0 ||
+        tag.privateuse !== undefined)
+    {
+        return;
+    }""")
+
+    # From Unicode BCP 47 locale identifier <https://unicode.org/reports/tr35/>.
+    #
+    # Doesn't allow any 'extensions' subtags.
+    re_unicode_locale_id = re.compile(
+        r"""
+        ^
+        # unicode_language_id = unicode_language_subtag
+        #     unicode_language_subtag = alpha{2,3} | alpha{5,8}
+        (?P<language>[a-z]{2,3}|[a-z]{5,8})
+
+        # (sep unicode_script_subtag)?
+        #     unicode_script_subtag = alpha{4}
+        (?:-(?P<script>[a-z]{4}))?
+
+        # (sep unicode_region_subtag)?
+        #     unicode_region_subtag = (alpha{2} | digit{3})
+        (?:-(?P<region>([a-z]{2}|[0-9]{3})))?
+
+        # (sep unicode_variant_subtag)*
+        #     unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3})
+        (?P<variants>(-([a-z0-9]{5,8}|[0-9][a-z0-9]{3}))+)?
+
+        # pu_extensions?
+        #     pu_extensions = sep [xX] (sep alphanum{1,8})+
+        (?:-(?P<privateuse>x(-[a-z0-9]{1,8})+))?
+        $
+        """, re.IGNORECASE | re.VERBOSE)
+
+    is_first = True
+
+    for (tag, modern) in sorted(grandfathered_mappings.items(), key=itemgetter(0)):
+        tag_match = re_unicode_locale_id.match(tag)
+        assert tag_match is not None
+
+        tag_language = tag_match.group("language")
+        assert tag_match.group("script") is None, (
+               "{} does not contain a script subtag".format(tag))
+        assert tag_match.group("region") is None, (
+               "{} does not contain a region subtag".format(tag))
+        tag_variants = tag_match.group("variants")
+        assert tag_variants is not None, (
+               "{} contains a variant subtag".format(tag))
+        assert tag_match.group("privateuse") is None, (
+               "{} does not contain a privateuse subtag".format(tag))
+
+        tag_variant = tag_variants[1:]
+        assert "-" not in tag_variant, (
+               "{} contains only a single variant".format(tag))
+
+        modern_match = re_unicode_locale_id.match(modern)
+        assert modern_match is not None
+
+        modern_language = modern_match.group("language")
+        modern_script = modern_match.group("script")
+        modern_region = modern_match.group("region")
+        modern_variants = modern_match.group("variants")
+        modern_privateuse = modern_match.group("privateuse")
+
+        println(u"""
+    // {} -> {}
+""".format(tag, modern).rstrip())
+
+        println(u"""
+    {}if (tag.language === "{}" && tag.variants[0] === "{}") {{
+        """.format("" if is_first else "else ", tag_language, tag_variant).rstrip().strip("\n"))
+
+        is_first = False
+
+        println(u"""
+        tag.language = "{}";
+        """.format(modern_language).rstrip().strip("\n"))
+
+        if modern_script is not None:
+            println(u"""
+        tag.script = "{}";
+        """.format(modern_script).rstrip().strip("\n"))
+
+        if modern_region is not None:
+            println(u"""
+        tag.region = "{}";
+        """.format(modern_region).rstrip().strip("\n"))
+
+        if modern_variants is not None:
+            println(u"""
+        tag.variants = {};
+        """.format(sorted(modern_variants[1:].split("-"))).rstrip().strip("\n"))
+        else:
+            println(u"""
+        tag.variants.length = 0;
+        """.rstrip().strip("\n"))
+
+        if modern_privateuse is not None:
+            println(u"""
+        tag.privateuse = "{}";
+        """.format(modern_privateuse).rstrip().strip("\n"))
+
+        println(u"""
+    }""".rstrip().strip("\n"))
+
+    println(u"""
+}""".lstrip("\n"))
+
+
+@contextmanager
+def TemporaryDirectory():
+    tmpDir = tempfile.mkdtemp()
+    try:
+        yield tmpDir
+    finally:
+        shutil.rmtree(tmpDir)
+
+
+def readSupplementalData(supplemental_dtd_file, supplemental_metadata_file, likely_subtags_file):
+    """ Reads CLDR Supplemental Data and extracts information for Intl.js.
 
         Information extracted:
-        - langTagMappings: mappings from complete language tags to preferred
+        - grandfatheredMappings: mappings from grandfathered tags to preferred
           complete language tags
-        - langSubtagMappings: mappings from subtags to preferred subtags
+        - languageMappings: mappings from language subtags to preferred subtags
-        - extlangMappings: mappings from extlang subtags to preferred subtags,
+        - complexLanguageMappings: mappings from language subtags with complex rules
-          with prefix to be removed
+        - regionMappings: mappings from region subtags to preferred subtags
-        Returns these three mappings as dictionaries, along with the registry's
+        - complexRegionMappings: mappings from region subtags with complex rules
-        file date.
+        Returns these five mappings as dictionaries.
-
-        We also check that mappings for language subtags don't affect extlang
-        subtags and vice versa, so that CanonicalizeLanguageTag doesn't have
-        to separate them for processing. Region codes are separated by case,
-        and script codes by length, so they're unproblematic.
     """
-    langTagMappings = {}
+    import xml.etree.ElementTree as ET
-    langSubtagMappings = {}
-    extlangMappings = {}
-    languageSubtags = set()
-    extlangSubtags = set()
 
-    for record in readRegistryRecord(registry):
+    # <!ATTLIST version cldrVersion CDATA #FIXED "36" >
-        if "File-Date" in record:
+    re_cldr_version = re.compile(
-            fileDate = record["File-Date"]
+        r"""<!ATTLIST version cldrVersion CDATA #FIXED "(?P<version>[\d|\.]+)" >""")
+
+    with io.open(supplemental_dtd_file, mode="r", encoding="utf-8") as f:
+        version_match = re_cldr_version.search(f.read())
+        assert version_match is not None, "CLDR version string not found"
+        cldr_version = version_match.group("version")
+
+    # From Unicode BCP 47 locale identifier <https://unicode.org/reports/tr35/>.
+    re_unicode_language_id = re.compile(
+        r"""
+        ^
+        # unicode_language_id = unicode_language_subtag
+        #     unicode_language_subtag = alpha{2,3} | alpha{5,8}
+        (?P<language>[a-z]{2,3}|[a-z]{5,8})
+
+        # (sep unicode_script_subtag)?
+        #     unicode_script_subtag = alpha{4}
+        (?:-(?P<script>[a-z]{4}))?
+
+        # (sep unicode_region_subtag)?
+        #     unicode_region_subtag = (alpha{2} | digit{3})
+        (?:-(?P<region>([a-z]{2}|[0-9]{3})))?
+
+        # (sep unicode_variant_subtag)*
+        #     unicode_variant_subtag = (alphanum{5,8} | digit alphanum{3})
+        (?P<variants>(-([a-z0-9]{5,8}|[0-9][a-z0-9]{3}))+)?
+        $
+        """, re.IGNORECASE | re.VERBOSE)
+
+    re_unicode_language_subtag = re.compile(
+        r"""
+        ^
+        # unicode_language_subtag = alpha{2,3} | alpha{5,8}
+        ([a-z]{2,3}|[a-z]{5,8})
+        $
+        """, re.IGNORECASE | re.VERBOSE)
+
+    re_unicode_region_subtag = re.compile(
+        r"""
+        ^
+        # unicode_region_subtag = (alpha{2} | digit{3})
+        ([a-z]{2}|[0-9]{3})
+        $
+        """, re.IGNORECASE | re.VERBOSE)
+
+    # The fixed list of BCP 47 grandfathered language tags.
+    grandfathered_tags = (
+        "art-lojban",
+        "cel-gaulish",
+        "en-GB-oed",
+        "i-ami",
+        "i-bnn",
+        "i-default",
+        "i-enochian",
+        "i-hak",
+        "i-klingon",
+        "i-lux",
+        "i-mingo",
+        "i-navajo",
+        "i-pwn",
+        "i-tao",
+        "i-tay",
+        "i-tsu",
+        "no-bok",
+        "no-nyn",
+        "sgn-BE-FR",
+        "sgn-BE-NL",
+        "sgn-CH-DE",
+        "zh-guoyu",
+        "zh-hakka",
+        "zh-min",
+        "zh-min-nan",
+        "zh-xiang",
+    )
+
+    # The list of grandfathered tags which are valid Unicode BCP 47 locale identifiers.
+    unicode_bcp47_grandfathered_tags = {tag for tag in grandfathered_tags
+                                        if re_unicode_language_id.match(tag)}
+
+    # Dictionary of simple language subtag mappings, e.g. "in" -> "id".
+    language_mappings = {}
+
+    # Dictionary of complex language subtag mappings, modifying more than one
+    # subtag, e.g. "sh" -> ("sr", "Latn", None) and "cnr" -> ("sr", None, "ME").
+    complex_language_mappings = {}
+
+    # Dictionary of simple region subtag mappings, e.g. "DD" -> "DE".
+    region_mappings = {}
+
+    # Dictionary of complex region subtag mappings, containing more than one
+    # replacement, e.g. "SU" -> ("RU", ["AM",complex_region_mappings[type] = replacements "AZ", "BY", ...]).
+    complex_region_mappings = {}
+
+    # Dictionary of grandfathered mappings to preferred values.
+    grandfathered_mappings = {}
+
+    # CLDR uses "_" as the separator for some elements. Replace it with "-".
+    def bcp47_id(cldr_id):
+        return cldr_id.replace("_", "-")
+
+    # CLDR uses the canonical case for most entries, but there are some
+    # exceptions, like:
+    #   <languageAlias type="drw" replacement="fa_af" reason="deprecated"/>
+    # Therefore canonicalize all tags to be on the safe side.
+    def bcp47_canonical(language, script, region):
+        # Canonical case for language subtags is lower case.
+        # Canonical case for script subtags is title case.
+        # Canonical case for region subtags is upper case.
+        return (language.lower() if language else None,
+                script.title() if script else None,
+                region.upper() if region else None)
+
+    tree = ET.parse(supplemental_metadata_file)
+
+    for language_alias in tree.iterfind(".//languageAlias"):
+        type = bcp47_id(language_alias.get("type"))
+        replacement = bcp47_id(language_alias.get("replacement"))
+
+        # Handle grandfathered mappings first.
+        if type in unicode_bcp47_grandfathered_tags:
+            grandfathered_mappings[type] = replacement
             continue
 
-        if record["Type"] == "grandfathered":
+        # We're only interested in language subtag matches, so ignore any
-            # Grandfathered tags don't use standard syntax, so
+        # entries which have additional subtags.
-            # CanonicalizeLanguageTag expects the mapping table to provide
+        if re_unicode_language_subtag.match(type) is None:
-            # the final form for all.
+            continue
-            # For langTagMappings, keys must be in lower case; values in
+
-            # the case used in the registry.
+        if re_unicode_language_subtag.match(replacement) is not None:
-            tag = record["Tag"]
+            # Canonical case for language subtags is lower-case.
-            if "Preferred-Value" in record:
+            language_mappings[type] = replacement.lower()
-                langTagMappings[tag.lower()] = record["Preferred-Value"]
-            else:
-                langTagMappings[tag.lower()] = tag
-        elif record["Type"] == "redundant":
-            # For langTagMappings, keys must be in lower case; values in
-            # the case used in the registry.
-            if "Preferred-Value" in record:
-                langTagMappings[record["Tag"].lower()] = record["Preferred-Value"]
-        elif record["Type"] in ("language", "script", "region", "variant"):
-            # For langSubtagMappings, keys and values must be in the case used
-            # in the registry.
-            subtag = record["Subtag"]
-            if record["Type"] == "language":
-                languageSubtags.add(subtag)
-            if "Preferred-Value" in record:
-                if subtag == "heploc":
-                    # The entry for heploc is unique in its complexity; handle
-                    # it as special case below.
-                    continue
-                if "Prefix" in record:
-                    # This might indicate another heploc-like complex case.
-                    raise Exception("Please evaluate: subtag mapping with prefix value.")
-                langSubtagMappings[subtag] = record["Preferred-Value"]
-        elif record["Type"] == "extlang":
-            # For extlangMappings, keys must be in the case used in the
-            # registry; values are records with the preferred value and the
-            # prefix to be removed.
-            subtag = record["Subtag"]
-            extlangSubtags.add(subtag)
-            if "Preferred-Value" in record:
-                preferred = record["Preferred-Value"]
-                prefix = record["Prefix"]
-                extlangMappings[subtag] = {"preferred": preferred, "prefix": prefix}
         else:
-            # No other types are allowed by
+            replacement_match = re_unicode_language_id.match(replacement)
-            # https://tools.ietf.org/html/rfc5646#section-3.1.3
+            assert replacement_match is not None, (
-            assert False, "Unrecognized Type: {0}".format(record["Type"])
+                   "{} invalid Unicode BCP 47 locale identifier".format(replacement))
+            assert replacement_match.group("variants") is None, (
+                   "{}: unexpected variant subtags in {}".format(type, replacement))
 
-    # Check that mappings for language subtags and extlang subtags don't affect
+            complex_language_mappings[type] = bcp47_canonical(replacement_match.group("language"),
-    # each other.
+                                                              replacement_match.group("script"),
-    for lang in languageSubtags:
+                                                              replacement_match.group("region"))
-        if lang in extlangMappings and extlangMappings[lang]["preferred"] != lang:
-            raise Exception("Conflict: lang with extlang mapping: " + lang)
-    for extlang in extlangSubtags:
-        if extlang in langSubtagMappings:
-            raise Exception("Conflict: extlang with lang mapping: " + extlang)
 
-    # Special case for heploc.
+    for territory_alias in tree.iterfind(".//territoryAlias"):
-    langTagMappings["ja-latn-hepburn-heploc"] = "ja-Latn-alalc97"
+        type = territory_alias.get("type")
+        replacement = territory_alias.get("replacement")
 
-    # ValidateAndCanonicalizeLanguageTag in Intl.js expects langTagMappings
+        # We're only interested in region subtag matches, so ignore any entries
-    # contains no 2*3ALPHA.
+        # which contain legacy formats, e.g. three letter region codes.
-    assert all(len(lang) > 3 for lang in langTagMappings.iterkeys())
+        if re_unicode_region_subtag.match(type) is None:
+            continue
 
-    return {"fileDate": fileDate,
+        if re_unicode_region_subtag.match(replacement) is not None:
-            "langTagMappings": langTagMappings,
+            # Canonical case for region subtags is upper-case.
-            "langSubtagMappings": langSubtagMappings,
+            region_mappings[type] = replacement.upper()
-            "extlangMappings": extlangMappings}
-
-
-def writeMappingsVar(intlData, dict, name, description, fileDate, url):
-    """ Writes a variable definition with a mapping table to file intlData.
-
-        Writes the contents of dictionary dict to file intlData with the given
-        variable name and a comment with description, fileDate, and URL.
-    """
-    intlData.write("\n")
-    intlData.write("// {0}.\n".format(description))
-    intlData.write("// Derived from IANA Language Subtag Registry, file date {0}.\n".format(fileDate))
-    intlData.write("// {0}\n".format(url))
-    intlData.write("var {0} = {{\n".format(name))
-    keys = sorted(dict)
-    for key in keys:
-        if isinstance(dict[key], basestring):
-            value = '"{0}"'.format(dict[key])
         else:
-            preferred = dict[key]["preferred"]
+            # Canonical case for region subtags is upper-case.
-            prefix = dict[key]["prefix"]
+            replacements = [r.upper() for r in replacement.split(" ")]
-            value = '{{preferred: "{0}", prefix: "{1}"}}'.format(preferred, prefix)
+            assert all(
-        intlData.write('    "{0}": {1},\n'.format(key, value))
+                re_unicode_region_subtag.match(loc) is not None for loc in replacements
-    intlData.write("};\n")
+            ), "{} invalid region subtags".format(replacement)
+            complex_region_mappings[type] = replacements
 
+    tree = ET.parse(likely_subtags_file)
 
-def writeLanguageTagData(intlData, fileDate, url, langTagMappings, langSubtagMappings, extlangMappings):
+    likely_subtags = {}
+
+    for likely_subtag in tree.iterfind(".//likelySubtag"):
+        from_tag = bcp47_id(likely_subtag.get("from"))
+        from_match = re_unicode_language_id.match(from_tag)
+        assert from_match is not None, (
+               "{} invalid Unicode BCP 47 locale identifier".format(from_tag))
+        assert from_match.group("variants") is None, (
+               "unexpected variant subtags in {}".format(from_tag))
+
+        to_tag = bcp47_id(likely_subtag.get("to"))
+        to_match = re_unicode_language_id.match(to_tag)
+        assert to_match is not None, (
+               "{} invalid Unicode BCP 47 locale identifier".format(to_tag))
+        assert to_match.group("variants") is None, (
+               "unexpected variant subtags in {}".format(to_tag))
+
+        from_canonical = bcp47_canonical(from_match.group("language"),
+                                         from_match.group("script"),
+                                         from_match.group("region"))
+
+        to_canonical = bcp47_canonical(to_match.group("language"),
+                                       to_match.group("script"),
+                                       to_match.group("region"))
+
+        likely_subtags[from_canonical] = to_canonical
+
+    complex_region_mappings_final = {}
+
+    for (deprecated_region, replacements) in complex_region_mappings.items():
+        # Find all likely subtag entries which don't already contain a region
+        # subtag and whose target region is in the list of replacement regions.
+        region_likely_subtags = [(from_language, from_script, to_region)
+                                 for ((from_language, from_script, from_region),
+                                      (_, _, to_region)) in likely_subtags.items()
+                                 if from_region is None and to_region in replacements]
+
+        # The first replacement entry is the default region.
+        default = replacements[0]
+
+        # Find all likely subtag entries whose region matches the default region.
+        default_replacements = {(language, script)
+                                for (language, script, region) in region_likely_subtags
+                                if region == default}
+
+        # And finally find those entries which don't use the default region.
+        # These are the entries we're actually interested in, because those need
+        # to be handled specially when selecting the correct preferred region.
+        non_default_replacements = [(language, script, region)
+                                    for (language, script, region) in region_likely_subtags
+                                    if (language, script) not in default_replacements]
+
+        # If there are no non-default replacements, we can handle the region as
+        # part of the simple region mapping.
+        if non_default_replacements:
+            complex_region_mappings_final[deprecated_region] = (default, non_default_replacements)
+        else:
+            region_mappings[deprecated_region] = default
+
+    return {"version": cldr_version,
+            "grandfatheredMappings": grandfathered_mappings,
+            "languageMappings": language_mappings,
+            "complexLanguageMappings": complex_language_mappings,
+            "regionMappings": region_mappings,
+            "complexRegionMappings": complex_region_mappings_final,
+            }
+
+def writeCLDRLanguageTagData(println, data, url):
     """ Writes the language tag data to the Intl data file. """
-    writeMappingsVar(intlData, langTagMappings, "langTagMappings",
-                     "Mappings from complete tags to preferred values", fileDate, url)
-    writeMappingsVar(intlData, langSubtagMappings, "langSubtagMappings",
-                     "Mappings from non-extlang subtags to preferred values", fileDate, url)
-    writeMappingsVar(intlData, extlangMappings, "extlangMappings",
-                     "Mappings from extlang subtags to preferred values", fileDate, url)
 
-def updateLangTags(args):
+    source = u"CLDR Supplemental Data, version {}".format(data["version"])
-    """ Update the LangTagMappingsGenerated.js file. """
+    grandfathered_mappings = data["grandfatheredMappings"]
+    language_mappings = data["languageMappings"]
+    complex_language_mappings = data["complexLanguageMappings"]
+    region_mappings = data["regionMappings"]
+    complex_region_mappings = data["complexRegionMappings"]
+
+    writeMappingsVar(println, grandfathered_mappings, "grandfatheredMappings",
+                     "Mappings from grandfathered tags to preferred values.", source, url)
+    writeMappingsVar(println, language_mappings, "languageMappings",
+                     "Mappings from language subtags to preferred values.", source, url)
+    writeMappingsVar(println, {key: True for key in complex_language_mappings},
+                     "complexLanguageMappings",
+                     "Language subtags with complex mappings.", source, url)
+    writeMappingsVar(println, region_mappings, "regionMappings",
+                     "Mappings from region subtags to preferred values.", source, url)
+    writeMappingsVar(println, {key: True for key in complex_region_mappings},
+                     "complexRegionMappings",
+                     "Region subtags with complex mappings.", source, url)
+
+    writeUpdateLocaleIdMappingsFunction(println, complex_language_mappings,
+                                        complex_region_mappings,
+                                        "Canonicalize Unicode BCP 47 locale identifiers.",
+                                        source, url)
+    writeGrandfatheredMappingsFunction(println, grandfathered_mappings,
+                                       "Canonicalize grandfathered locale identifiers.",
+                                       source, url)
+
+
+def updateCLDRLangTags(args):
+    """ Update the LangTagMappingsCLDRGenerated.js file. """
     url = args.url
+    branch = args.branch
+    revision = args.revision
     out = args.out
-    filename = args.file
+    files = args.files
 
     print("Arguments:")
     print("\tDownload url: %s" % url)
-    print("\tLocal registry: %s" % filename)
+    print("\tBranch: %s" % branch)
+    print("\tRevision: %s" % revision)
+    print("\tLocal supplemental data and likely subtags: %s" % files)
     print("\tOutput file: %s" % out)
     print("")
 
-    if filename is not None:
+    if files:
-        print("Always make sure you have the newest language-subtag-registry.txt!")
+        if len(files) != 3:
-        registry = codecs.open(filename, "r", encoding="utf-8")
+            raise Exception("Expected three files, but got: {}".format(files))
-    else:
-        print("Downloading IANA Language Subtag Registry...")
-        with closing(urllib2.urlopen(url)) as reader:
-            text = reader.read().decode("utf-8")
-        registry = codecs.open("language-subtag-registry.txt", "w+", encoding="utf-8")
-        registry.write(text)
-        registry.seek(0)
 
-    print("Processing IANA Language Subtag Registry...")
+        print(("Always make sure you have the newest ldmlSupplemental.dtd, "
-    with closing(registry) as reg:
+               "supplementalMetadata.xml, and likelySubtags.xml!"))
-        data = readRegistry(reg)
+
-    fileDate = data["fileDate"]
+        supplemental_dtd_file = files[0]
-    langTagMappings = data["langTagMappings"]
+        supplemental_metadata_file = files[1]
-    langSubtagMappings = data["langSubtagMappings"]
+        likely_subtags_file = files[2]
-    extlangMappings = data["extlangMappings"]
+    else:
+        print("Downloading CLDR supplemental data...")
+
+        supplemental_dtd_filename = "ldmlSupplemental.dtd"
+        supplemental_dtd_path = "common/dtd/{}".format(supplemental_dtd_filename)
+        supplemental_dtd_file = os.path.join(os.getcwd(), supplemental_dtd_filename)
+
+        supplemental_metadata_filename = "supplementalMetadata.xml"
+        supplemental_metadata_path = "common/supplemental/{}".format(
+            supplemental_metadata_filename)
+        supplemental_metadata_file = os.path.join(os.getcwd(), supplemental_metadata_filename)
+
+        likely_subtags_filename = "likelySubtags.xml"
+        likely_subtags_path = "common/supplemental/{}".format(likely_subtags_filename)
+        likely_subtags_file = os.path.join(os.getcwd(), likely_subtags_filename)
+
+        # Try to download the raw file directly from GitHub if possible.
+        split = urlsplit(url)
+        if split.netloc == "github.com" and split.path.endswith(".git") and revision == "HEAD":
+            def download(path, file):
+                urlpath = "{}/raw/{}/{}".format(urlsplit(url).path[:-4], branch, path)
+                raw_url = urlunsplit((split.scheme, split.netloc, urlpath, split.query,
+                                      split.fragment))
+
+                with closing(urllib2.urlopen(raw_url)) as reader:
+                    text = reader.read().decode("utf-8")
+                with io.open(file, "w", encoding="utf-8") as saved_file:
+                    saved_file.write(text)
+
+            download(supplemental_dtd_path, supplemental_dtd_file)
+            download(supplemental_metadata_path, supplemental_metadata_file)
+            download(likely_subtags_path, likely_subtags_file)
+        else:
+            # Download the requested branch in a temporary directory.
+            with TemporaryDirectory() as inDir:
+                if revision == "HEAD":
+                    subprocess.check_call(["git", "clone", "--depth=1",
+                                           "--branch=%s" % branch, url, inDir])
+                else:
+                    subprocess.check_call(["git", "clone", "--single-branch",
+                                           "--branch=%s" % branch, url, inDir])
+                    subprocess.check_call(["git", "-C", inDir, "reset", "--hard", revision])
+
+                    shutil.copyfile(os.path.join(inDir, supplemental_dtd_path),
+                                    supplemental_dtd_file)
+                    shutil.copyfile(os.path.join(inDir, supplemental_metadata_path),
+                                    supplemental_metadata_file)
+                    shutil.copyfile(os.path.join(inDir, likely_subtags_path), likely_subtags_file)
+
+    print("Processing CLDR supplemental data...")
+    data = readSupplementalData(supplemental_dtd_file,
+                                supplemental_metadata_file,
+                                likely_subtags_file)
 
     print("Writing Intl data...")
-    with codecs.open(out, "w", encoding="utf-8") as intlData:
+    with io.open(out, mode="w", encoding="utf-8", newline="") as f:
-        intlData.write("// Generated by make_intl_data.py. DO NOT EDIT.\n")
+        println = partial(print, file=f)
-        writeLanguageTagData(intlData, fileDate, url, langTagMappings, langSubtagMappings, extlangMappings)
+
+        println(u"// Generated by make_intl_data.py. DO NOT EDIT.")
+        writeCLDRLanguageTagData(println, data, url)
+
 
 def flines(filepath, encoding="utf-8"):
     """ Open filepath and iterate over its content. """
@@ -707,11 +1214,11 @@ def processTimeZones(tzdataDir, icuDir, icuTzDir, version, ignoreBackzone, ignor
 
         println(u"// Format:")
         println(u'// "LinkName", "Target" // ICU-Target [time zone file]')
-        println(u"struct LinkAndTarget");
+        println(u"struct LinkAndTarget")
-        println(u"{");
+        println(u"{")
-        println(u"    const char* const link;");
+        println(u"    const char* const link;")
-        println(u"    const char* const target;");
+        println(u"    const char* const target;")
-        println(u"};");
+        println(u"};")
         println(u"")
         println(u"const LinkAndTarget ianaLinksCanonicalizedDifferentlyByICU[] = {")
         for (zone, target, icuTarget) in incorrectLinks:
@@ -932,7 +1439,7 @@ def updateTzdata(topsrcdir, args):
     if tzDir is None:
         print("Downloading tzdata file...")
         with closing(urllib2.urlopen(url)) as tzfile:
-            fname = urlparse.urlsplit(tzfile.geturl()).path.split("/")[-1]
+            fname = urlsplit(tzfile.geturl()).path.split("/")[-1]
             with tempfile.NamedTemporaryFile(suffix=fname) as tztmpfile:
                 print("File stored in %s" % tztmpfile.name)
                 tztmpfile.write(tzfile.read())
@@ -959,20 +1466,24 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Update intl data.")
     subparsers = parser.add_subparsers(help="Select update mode")
 
-    parser_tags = subparsers.add_parser("langtags",
+    parser_cldr_tags = subparsers.add_parser("langtags",
-                                        help="Update language-subtag-registry")
+                                             help="Update CLDR language tags data")
-    parser_tags.add_argument("--url",
+    parser_cldr_tags.add_argument("--url",
-                             metavar="URL",
+                                  metavar="URL",
-                             default="https://www.iana.org/assignments/language-subtag-registry",
+                                  default="https://github.com/unicode-org/cldr.git",
-                             type=EnsureHttps,
+                                  help="URL to git repository (default: %(default)s)")
-                             help="Download url for language-subtag-registry.txt (default: %(default)s)")
+    parser_cldr_tags.add_argument("--branch", default="latest",
-    parser_tags.add_argument("--out",
+                                  help="Git branch (default: %(default)s)")
-                             default="LangTagMappingsGenerated.js",
+    parser_cldr_tags.add_argument("--revision", default="HEAD",
-                             help="Output file (default: %(default)s)")
+                                  help="Git revision (default: %(default)s)")
-    parser_tags.add_argument("file",
+    parser_cldr_tags.add_argument("--out",
-                             nargs="?",
+                                  default="LangTagMappingsGenerated.js",
-                             help="Local language-subtag-registry.txt file, if omitted uses <URL>")
+                                  help="Output file (default: %(default)s)")
-    parser_tags.set_defaults(func=updateLangTags)
+    parser_cldr_tags.add_argument("files",
+                                  nargs="*",
+                                  help="Local ldmlSupplemental.dtd, supplementalMetadata.xml, "
+                                       "and likelySubtags.xml files, if omitted uses <URL>")
+    parser_cldr_tags.set_defaults(func=updateCLDRLangTags)
 
     parser_tz = subparsers.add_parser("tzdata", help="Update tzdata")
     parser_tz.add_argument("--tz",

js/src/vm/SelfHosting.cpp

@@ -2545,8 +2545,6 @@ static const JSFunctionSpec intrinsic_functions[] = {
     JS_FN("StringSplitStringLimit", intrinsic_StringSplitStringLimit, 3, 0),
 
     // See builtin/RegExp.h for descriptions of the regexp_* functions.
-    JS_FN("regexp_exec_no_statics", regexp_exec_no_statics, 2,0),
-    JS_FN("regexp_test_no_statics", regexp_test_no_statics, 2,0),
     JS_FN("regexp_construct_raw_flags", regexp_construct_raw_flags, 2,0),
     JS_FN("regexp_clone", regexp_clone, 1,0),