import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

third_party/rust/unicode-normalization/.cargo-checksum.json

@@ -0,0 +1 @@
+{"files":{".cargo-ok":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",".gitignore":"2282653731b0b47b955075a5a366b16c4ea092dbe7de4f9bb68a0e36abbb62f3",".travis.yml":"493704d6a0d0f27ad2ad6e950f8bce5f42d9ec4081daeb3c5a48066f1030f467","COPYRIGHT":"23860c2a7b5d96b21569afedf033469bab9fe14a1b24a35068b8641c578ce24d","Cargo.toml":"098ff7b940aedf372f29c54788e1a58173a54846c88fec5c627b3d74d013298c","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"7b63ecd5f1902af1b63729947373683c32745c16a10e8e6292e2e2dcd7e90ae0","README.md":"8a937b4305a42bfa5a2faf7baaaae8156d055537934a53eed2d3bc9a3f9da40a","scripts/unicode.py":"64a1b919ab0e251fdb1db8b9c5363d84227fca33ac8375268bb88b74c4462f8f","scripts/unicode_gen_normtests.py":"da891d433fa58068747a1cd121774435b7d486394ce5c85c8079b227d20ea507","src/DerivedNormalizationProps.txt":"6cc76100d402d7dd12bd769ac61158b91e71aad58c8333abb848f1b9c9fdde2d","src/ReadMe.txt":"e4dc6045cb89ddaca938ff7ffffc2c2aafc5d4fc85023fffb9b36d4aee47986b","src/UnicodeData.txt":"38b17e1118206489a7e0ab5d29d7932212d38838df7d3ec025ecb58e8798ec20","src/decompose.rs":"04818a6b0271412ec58508e44535b18c58c80384a5836fe5b2c24e489a5ab4cc","src/lib.rs":"d3cab5f68a71b640da82859d06d348a0c3050d611c17cc9cb66989492437a98c","src/normalize.rs":"6bcc92e3f2ee9922a017a7b7adc8dfd88de4bcf777974e07da3688559d0ec008","src/recompose.rs":"96b8aea91e09f1fa439467378f2d1fa2aa4c81c86e597d3d36a2a35d7750a0d6","src/tables.rs":"bc930c04ed78307e624a225ec27679cf3fad00ba13cb2f283963476cd449e84e","src/test.rs":"e47de49aeef9231f5ff177eeb9946618577fc67ed798e889b99710c877e92e1b","src/testdata.rs":"6a67ae0a072c7d65fca0d101f4f5831592ff24ab137f8e1c4f9524f2bdc8d63e"},"package":"26643a2f83bac55f1976fb716c10234485f9202dcd65cfbdf9da49867b271172"}

third_party/rust/unicode-normalization/.gitignore

@@ -0,0 +1,4 @@
+target
+Cargo.lock
+scripts/tmp
+*.pyc

third_party/rust/unicode-normalization/.travis.yml

@@ -0,0 +1,20 @@
+language: rust
+sudo: false
+script:
+  - cargo build --verbose
+  - cargo test --verbose
+  - rustdoc --test README.md -L target/debug -L target/debug/deps
+  - cargo doc
+after_success: |
+  [ $TRAVIS_BRANCH = master ] &&
+  [ $TRAVIS_PULL_REQUEST = false ] &&
+  echo '<meta http-equiv=refresh content=0;url=unicode_normalization/index.html>' > target/doc/index.html &&
+  pip install ghp-import --user $USER &&
+  $HOME/.local/bin/ghp-import -n target/doc &&
+  git push -qf https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages
+env:
+  global:
+    secure: Te91dtDN8uv5OBxVuMy+nvQ5GtnLU9r6amS9p6IbblVXyzXgXPQdFfAND+GXXfZNnsjAyS2LnZL4NcNOR4JF63H0KxTTDIGXLSUAmc0C98UhqWWvv5bjz4mY0YKs9MwuTBX9P0LdTZjvRTd+yJ2PYH7ORGen+ZuOmlPNE7lpzrg=
+notifications:
+  email:
+    on_success: never

third_party/rust/unicode-normalization/COPYRIGHT

@@ -0,0 +1,7 @@
+Licensed under the Apache License, Version 2.0
+<LICENSE-APACHE or
+http://www.apache.org/licenses/LICENSE-2.0> or the MIT
+license <LICENSE-MIT or http://opensource.org/licenses/MIT>,
+at your option. All files in the project carrying such
+notice may not be copied, modified, or distributed except
+according to those terms.

third_party/rust/unicode-normalization/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+
+name = "unicode-normalization"
+version = "0.1.2"
+authors = ["kwantam <kwantam@gmail.com>"]
+
+homepage = "https://github.com/unicode-rs/unicode-normalization"
+repository = "https://github.com/unicode-rs/unicode-normalization"
+documentation = "https://unicode-rs.github.io/unicode-normalization"
+
+license = "MIT/Apache-2.0"
+keywords = ["text", "unicode", "normalization", "decomposition", "recomposition"]
+readme = "README.md"
+description = """
+This crate provides functions for normalization of
+Unicode strings, including Canonical and Compatible
+Decomposition and Recomposition, as described in
+Unicode Standard Annex #15.
+"""
+
+exclude = [ "target/*", "Cargo.lock", "scripts/tmp" ]

third_party/rust/unicode-normalization/LICENSE-APACHE

@@ -0,0 +1,201 @@
+                              Apache License
+                        Version 2.0, January 2004
+                     http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+   "License" shall mean the terms and conditions for use, reproduction,
+   and distribution as defined by Sections 1 through 9 of this document.
+
+   "Licensor" shall mean the copyright owner or entity authorized by
+   the copyright owner that is granting the License.
+
+   "Legal Entity" shall mean the union of the acting entity and all
+   other entities that control, are controlled by, or are under common
+   control with that entity. For the purposes of this definition,
+   "control" means (i) the power, direct or indirect, to cause the
+   direction or management of such entity, whether by contract or
+   otherwise, or (ii) ownership of fifty percent (50%) or more of the
+   outstanding shares, or (iii) beneficial ownership of such entity.
+
+   "You" (or "Your") shall mean an individual or Legal Entity
+   exercising permissions granted by this License.
+
+   "Source" form shall mean the preferred form for making modifications,
+   including but not limited to software source code, documentation
+   source, and configuration files.
+
+   "Object" form shall mean any form resulting from mechanical
+   transformation or translation of a Source form, including but
+   not limited to compiled object code, generated documentation,
+   and conversions to other media types.
+
+   "Work" shall mean the work of authorship, whether in Source or
+   Object form, made available under the License, as indicated by a
+   copyright notice that is included in or attached to the work
+   (an example is provided in the Appendix below).
+
+   "Derivative Works" shall mean any work, whether in Source or Object
+   form, that is based on (or derived from) the Work and for which the
+   editorial revisions, annotations, elaborations, or other modifications
+   represent, as a whole, an original work of authorship. For the purposes
+   of this License, Derivative Works shall not include works that remain
+   separable from, or merely link (or bind by name) to the interfaces of,
+   the Work and Derivative Works thereof.
+
+   "Contribution" shall mean any work of authorship, including
+   the original version of the Work and any modifications or additions
+   to that Work or Derivative Works thereof, that is intentionally
+   submitted to Licensor for inclusion in the Work by the copyright owner
+   or by an individual or Legal Entity authorized to submit on behalf of
+   the copyright owner. For the purposes of this definition, "submitted"
+   means any form of electronic, verbal, or written communication sent
+   to the Licensor or its representatives, including but not limited to
+   communication on electronic mailing lists, source code control systems,
+   and issue tracking systems that are managed by, or on behalf of, the
+   Licensor for the purpose of discussing and improving the Work, but
+   excluding communication that is conspicuously marked or otherwise
+   designated in writing by the copyright owner as "Not a Contribution."
+
+   "Contributor" shall mean Licensor and any individual or Legal Entity
+   on behalf of whom a Contribution has been received by Licensor and
+   subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   copyright license to reproduce, prepare Derivative Works of,
+   publicly display, publicly perform, sublicense, and distribute the
+   Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+   this License, each Contributor hereby grants to You a perpetual,
+   worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+   (except as stated in this section) patent license to make, have made,
+   use, offer to sell, sell, import, and otherwise transfer the Work,
+   where such license applies only to those patent claims licensable
+   by such Contributor that are necessarily infringed by their
+   Contribution(s) alone or by combination of their Contribution(s)
+   with the Work to which such Contribution(s) was submitted. If You
+   institute patent litigation against any entity (including a
+   cross-claim or counterclaim in a lawsuit) alleging that the Work
+   or a Contribution incorporated within the Work constitutes direct
+   or contributory patent infringement, then any patent licenses
+   granted to You under this License for that Work shall terminate
+   as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+   Work or Derivative Works thereof in any medium, with or without
+   modifications, and in Source or Object form, provided that You
+   meet the following conditions:
+
+   (a) You must give any other recipients of the Work or
+       Derivative Works a copy of this License; and
+
+   (b) You must cause any modified files to carry prominent notices
+       stating that You changed the files; and
+
+   (c) You must retain, in the Source form of any Derivative Works
+       that You distribute, all copyright, patent, trademark, and
+       attribution notices from the Source form of the Work,
+       excluding those notices that do not pertain to any part of
+       the Derivative Works; and
+
+   (d) If the Work includes a "NOTICE" text file as part of its
+       distribution, then any Derivative Works that You distribute must
+       include a readable copy of the attribution notices contained
+       within such NOTICE file, excluding those notices that do not
+       pertain to any part of the Derivative Works, in at least one
+       of the following places: within a NOTICE text file distributed
+       as part of the Derivative Works; within the Source form or
+       documentation, if provided along with the Derivative Works; or,
+       within a display generated by the Derivative Works, if and
+       wherever such third-party notices normally appear. The contents
+       of the NOTICE file are for informational purposes only and
+       do not modify the License. You may add Your own attribution
+       notices within Derivative Works that You distribute, alongside
+       or as an addendum to the NOTICE text from the Work, provided
+       that such additional attribution notices cannot be construed
+       as modifying the License.
+
+   You may add Your own copyright statement to Your modifications and
+   may provide additional or different license terms and conditions
+   for use, reproduction, or distribution of Your modifications, or
+   for any such Derivative Works as a whole, provided Your use,
+   reproduction, and distribution of the Work otherwise complies with
+   the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+   any Contribution intentionally submitted for inclusion in the Work
+   by You to the Licensor shall be under the terms and conditions of
+   this License, without any additional terms or conditions.
+   Notwithstanding the above, nothing herein shall supersede or modify
+   the terms of any separate license agreement you may have executed
+   with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+   names, trademarks, service marks, or product names of the Licensor,
+   except as required for reasonable and customary use in describing the
+   origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+   agreed to in writing, Licensor provides the Work (and each
+   Contributor provides its Contributions) on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied, including, without limitation, any warranties or conditions
+   of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+   PARTICULAR PURPOSE. You are solely responsible for determining the
+   appropriateness of using or redistributing the Work and assume any
+   risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+   whether in tort (including negligence), contract, or otherwise,
+   unless required by applicable law (such as deliberate and grossly
+   negligent acts) or agreed to in writing, shall any Contributor be
+   liable to You for damages, including any direct, indirect, special,
+   incidental, or consequential damages of any character arising as a
+   result of this License or out of the use or inability to use the
+   Work (including but not limited to damages for loss of goodwill,
+   work stoppage, computer failure or malfunction, or any and all
+   other commercial damages or losses), even if such Contributor
+   has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+   the Work or Derivative Works thereof, You may choose to offer,
+   and charge a fee for, acceptance of support, warranty, indemnity,
+   or other liability obligations and/or rights consistent with this
+   License. However, in accepting such obligations, You may act only
+   on Your own behalf and on Your sole responsibility, not on behalf
+   of any other Contributor, and only if You agree to indemnify,
+   defend, and hold each Contributor harmless for any liability
+   incurred by, or claims asserted against, such Contributor by reason
+   of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+APPENDIX: How to apply the Apache License to your work.
+
+   To apply the Apache License to your work, attach the following
+   boilerplate notice, with the fields enclosed by brackets "[]"
+   replaced with your own identifying information. (Don't include
+   the brackets!)  The text should be enclosed in the appropriate
+   comment syntax for the file format. We also recommend that a
+   file or class name and description of purpose be included on the
+   same "printed page" as the copyright notice for easier
+   identification within third-party archives.
+
+Copyright [yyyy] [name of copyright owner]
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

third_party/rust/unicode-normalization/LICENSE-MIT

@@ -0,0 +1,25 @@
+Copyright (c) 2015 The Rust Project Developers
+
+Permission is hereby granted, free of charge, to any
+person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the
+Software without restriction, including without
+limitation the rights to use, copy, modify, merge,
+publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software
+is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions
+of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
+TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
+IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

third_party/rust/unicode-normalization/README.md

@@ -0,0 +1,32 @@
+Unicode character composition and decomposition utilities
+as described in
+[Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/).
+
+[![Build Status](https://travis-ci.org/unicode-rs/unicode-normalization.svg)](https://travis-ci.org/unicode-rs/unicode-normalization)
+
+[Documentation](https://unicode-rs.github.io/unicode-normalization/unicode_normalization/index.html)
+
+```rust
+extern crate unicode_normalization;
+
+use unicode_normalization::char::compose;
+use unicode_normalization::UnicodeNormalization;
+
+fn main() {
+    assert_eq!(compose('A','\u{30a}'), Some('Å'));
+    
+    let s = "ÅΩ";
+    let c = s.nfc().collect::<String>();
+    assert_eq!(c, "ÅΩ");
+}
+```
+
+# crates.io
+
+You can use this package in your project by adding the following
+to your `Cargo.toml`:
+
+```toml
+[dependencies]
+unicode-normalization = "0.1.1"
+```

third_party/rust/unicode-normalization/scripts/unicode.py

@@ -0,0 +1,372 @@
+#!/usr/bin/env python
+#
+# Copyright 2011-2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This script uses the following Unicode tables:
+# - DerivedNormalizationProps.txt
+# - ReadMe.txt
+# - UnicodeData.txt
+#
+# Since this should not require frequent updates, we just store this
+# out-of-line and check the unicode.rs file into git.
+
+import fileinput, re, os, sys
+
+preamble = '''// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
+
+#![allow(missing_docs, non_upper_case_globals, non_snake_case)]
+'''
+
+# Mapping taken from Table 12 from:
+# http://www.unicode.org/reports/tr44/#General_Category_Values
+expanded_categories = {
+    'Lu': ['LC', 'L'], 'Ll': ['LC', 'L'], 'Lt': ['LC', 'L'],
+    'Lm': ['L'], 'Lo': ['L'],
+    'Mn': ['M'], 'Mc': ['M'], 'Me': ['M'],
+    'Nd': ['N'], 'Nl': ['N'], 'No': ['No'],
+    'Pc': ['P'], 'Pd': ['P'], 'Ps': ['P'], 'Pe': ['P'],
+    'Pi': ['P'], 'Pf': ['P'], 'Po': ['P'],
+    'Sm': ['S'], 'Sc': ['S'], 'Sk': ['S'], 'So': ['S'],
+    'Zs': ['Z'], 'Zl': ['Z'], 'Zp': ['Z'],
+    'Cc': ['C'], 'Cf': ['C'], 'Cs': ['C'], 'Co': ['C'], 'Cn': ['C'],
+}
+
+# these are the surrogate codepoints, which are not valid rust characters
+surrogate_codepoints = (0xd800, 0xdfff)
+
+def fetch(f):
+    if not os.path.exists(os.path.basename(f)):
+        os.system("curl -O http://www.unicode.org/Public/UNIDATA/%s"
+                  % f)
+
+    if not os.path.exists(os.path.basename(f)):
+        sys.stderr.write("cannot load %s" % f)
+        exit(1)
+
+def is_surrogate(n):
+    return surrogate_codepoints[0] <= n <= surrogate_codepoints[1]
+
+def load_unicode_data(f):
+    fetch(f)
+    combines = {}
+    canon_decomp = {}
+    compat_decomp = {}
+    general_category_mark = []
+
+    udict = {};
+    range_start = -1;
+    for line in fileinput.input(f):
+        data = line.split(';');
+        if len(data) != 15:
+            continue
+        cp = int(data[0], 16);
+        if is_surrogate(cp):
+            continue
+        if range_start >= 0:
+            for i in xrange(range_start, cp):
+                udict[i] = data;
+            range_start = -1;
+        if data[1].endswith(", First>"):
+            range_start = cp;
+            continue;
+        udict[cp] = data;
+
+    for code in udict:
+        [code_org, name, gencat, combine, bidi,
+         decomp, deci, digit, num, mirror,
+         old, iso, upcase, lowcase, titlecase ] = udict[code];
+
+        # store decomposition, if given
+        if decomp != "":
+            if decomp.startswith('<'):
+                seq = []
+                for i in decomp.split()[1:]:
+                    seq.append(int(i, 16))
+                compat_decomp[code] = seq
+            else:
+                seq = []
+                for i in decomp.split():
+                    seq.append(int(i, 16))
+                canon_decomp[code] = seq
+
+        # record combining class, if any
+        if combine != "0":
+            if combine not in combines:
+                combines[combine] = []
+            combines[combine].append(code)
+
+        if 'M' in [gencat] + expanded_categories.get(gencat, []):
+            general_category_mark.append(code)
+    general_category_mark = group_cat(general_category_mark)
+
+    combines = to_combines(group_cats(combines))
+
+    return (canon_decomp, compat_decomp, combines, general_category_mark)
+
+def group_cats(cats):
+    cats_out = {}
+    for cat in cats:
+        cats_out[cat] = group_cat(cats[cat])
+    return cats_out
+
+def group_cat(cat):
+    cat_out = []
+    letters = sorted(set(cat))
+    cur_start = letters.pop(0)
+    cur_end = cur_start
+    for letter in letters:
+        assert letter > cur_end, \
+            "cur_end: %s, letter: %s" % (hex(cur_end), hex(letter))
+        if letter == cur_end + 1:
+            cur_end = letter
+        else:
+            cat_out.append((cur_start, cur_end))
+            cur_start = cur_end = letter
+    cat_out.append((cur_start, cur_end))
+    return cat_out
+
+def ungroup_cat(cat):
+    cat_out = []
+    for (lo, hi) in cat:
+        while lo <= hi:
+            cat_out.append(lo)
+            lo += 1
+    return cat_out
+
+def to_combines(combs):
+    combs_out = []
+    for comb in combs:
+        for (lo, hi) in combs[comb]:
+            combs_out.append((lo, hi, comb))
+    combs_out.sort(key=lambda comb: comb[0])
+    return combs_out
+
+def format_table_content(f, content, indent):
+    line = " "*indent
+    first = True
+    for chunk in content.split(","):
+        if len(line) + len(chunk) < 98:
+            if first:
+                line += chunk
+            else:
+                line += ", " + chunk
+            first = False
+        else:
+            f.write(line + ",\n")
+            line = " "*indent + chunk
+    f.write(line)
+
+def load_properties(f, interestingprops):
+    fetch(f)
+    props = {}
+    re1 = re.compile("^ *([0-9A-F]+) *; *(\w+)")
+    re2 = re.compile("^ *([0-9A-F]+)\.\.([0-9A-F]+) *; *(\w+)")
+
+    for line in fileinput.input(os.path.basename(f)):
+        prop = None
+        d_lo = 0
+        d_hi = 0
+        m = re1.match(line)
+        if m:
+            d_lo = m.group(1)
+            d_hi = m.group(1)
+            prop = m.group(2)
+        else:
+            m = re2.match(line)
+            if m:
+                d_lo = m.group(1)
+                d_hi = m.group(2)
+                prop = m.group(3)
+            else:
+                continue
+        if interestingprops and prop not in interestingprops:
+            continue
+        d_lo = int(d_lo, 16)
+        d_hi = int(d_hi, 16)
+        if prop not in props:
+            props[prop] = []
+        props[prop].append((d_lo, d_hi))
+
+    # optimize if possible
+    for prop in props:
+        props[prop] = group_cat(ungroup_cat(props[prop]))
+
+    return props
+
+def escape_char(c):
+    return "'\\u{%x}'" % c
+
+def emit_table(f, name, t_data, t_type = "&'static [(char, char)]", is_pub=True,
+        pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1]))):
+    pub_string = ""
+    if is_pub:
+        pub_string = "pub "
+    f.write("    %sconst %s: %s = &[\n" % (pub_string, name, t_type))
+    data = ""
+    first = True
+    for dat in t_data:
+        if not first:
+            data += ","
+        first = False
+        data += pfun(dat)
+    format_table_content(f, data, 8)
+    f.write("\n    ];\n\n")
+
+def emit_norm_module(f, canon, compat, combine, norm_props, general_category_mark):
+    canon_keys = canon.keys()
+    canon_keys.sort()
+
+    compat_keys = compat.keys()
+    compat_keys.sort()
+
+    canon_comp = {}
+    comp_exclusions = norm_props["Full_Composition_Exclusion"]
+    for char in canon_keys:
+        if True in map(lambda (lo, hi): lo <= char <= hi, comp_exclusions):
+            continue
+        decomp = canon[char]
+        if len(decomp) == 2:
+            if not canon_comp.has_key(decomp[0]):
+                canon_comp[decomp[0]] = []
+            canon_comp[decomp[0]].append( (decomp[1], char) )
+    canon_comp_keys = canon_comp.keys()
+    canon_comp_keys.sort()
+
+    f.write("pub mod normalization {\n")
+
+    def mkdata_fun(table):
+        def f(char):
+            data = "(%s,&[" % escape_char(char)
+            first = True
+            for d in table[char]:
+                if not first:
+                    data += ","
+                first = False
+                data += escape_char(d)
+            data += "])"
+            return data
+        return f
+
+    f.write("    // Canonical decompositions\n")
+    emit_table(f, "canonical_table", canon_keys, "&'static [(char, &'static [char])]",
+        pfun=mkdata_fun(canon))
+
+    f.write("    // Compatibility decompositions\n")
+    emit_table(f, "compatibility_table", compat_keys, "&'static [(char, &'static [char])]",
+        pfun=mkdata_fun(compat))
+
+    def comp_pfun(char):
+        data = "(%s,&[" % escape_char(char)
+        canon_comp[char].sort(lambda x, y: x[0] - y[0])
+        first = True
+        for pair in canon_comp[char]:
+            if not first:
+                data += ","
+            first = False
+            data += "(%s,%s)" % (escape_char(pair[0]), escape_char(pair[1]))
+        data += "])"
+        return data
+
+    f.write("    // Canonical compositions\n")
+    emit_table(f, "composition_table", canon_comp_keys,
+        "&'static [(char, &'static [(char, char)])]", pfun=comp_pfun)
+
+    f.write("""
+    fn bsearch_range_value_table(c: char, r: &'static [(char, char, u8)]) -> u8 {
+        use std::cmp::Ordering::{Equal, Less, Greater};
+        match r.binary_search_by(|&(lo, hi, _)| {
+            if lo <= c && c <= hi { Equal }
+            else if hi < c { Less }
+            else { Greater }
+        }) {
+            Ok(idx) => {
+                let (_, _, result) = r[idx];
+                result
+            }
+            Err(_) => 0
+        }
+    }\n
+""")
+
+    emit_table(f, "combining_class_table", combine, "&'static [(char, char, u8)]", is_pub=False,
+            pfun=lambda x: "(%s,%s,%s)" % (escape_char(x[0]), escape_char(x[1]), x[2]))
+
+    f.write("    pub fn canonical_combining_class(c: char) -> u8 {\n"
+        + "        bsearch_range_value_table(c, combining_class_table)\n"
+        + "    }\n")
+
+    f.write("""
+    fn bsearch_range_table(c: char, r: &'static [(char, char)]) -> bool {
+        use std::cmp::Ordering::{Equal, Less, Greater};
+        r.binary_search_by(|&(lo, hi)| {
+             if lo <= c && c <= hi {
+                 Equal
+             } else if hi < c {
+                 Less
+             } else {
+                 Greater
+             }
+         })
+         .is_ok()
+    }
+
+    /// Return whether the given character is a combining mark (`General_Category=Mark`)
+    pub fn is_combining_mark(c: char) -> bool {
+        bsearch_range_table(c, general_category_mark)
+    }
+
+""")
+
+    emit_table(f, "general_category_mark", combine, "&'static [(char, char)]", is_pub=False,
+            pfun=lambda x: "(%s,%s)" % (escape_char(x[0]), escape_char(x[1])))
+
+    f.write("""
+}
+
+""")
+
+if __name__ == "__main__":
+    r = "tables.rs"
+    if os.path.exists(r):
+        os.remove(r)
+    with open(r, "w") as rf:
+        # write the file's preamble
+        rf.write(preamble)
+
+        # download and parse all the data
+        fetch("ReadMe.txt")
+        with open("ReadMe.txt") as readme:
+            pattern = "for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
+            unicode_version = re.search(pattern, readme.read()).groups()
+        rf.write("""
+/// The version of [Unicode](http://www.unicode.org/)
+/// that this version of unicode-normalization is based on.
+pub const UNICODE_VERSION: (u64, u64, u64) = (%s, %s, %s);
+
+""" % unicode_version)
+        (canon_decomp, compat_decomp, combines, general_category_mark) = \
+            load_unicode_data("UnicodeData.txt")
+        norm_props = load_properties("DerivedNormalizationProps.txt",
+                     ["Full_Composition_Exclusion"])
+
+        # normalizations and conversions module
+        emit_norm_module(rf, canon_decomp, compat_decomp, combines, norm_props,
+                         general_category_mark)

third_party/rust/unicode-normalization/scripts/unicode_gen_normtests.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+#
+# Copyright 2015 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# This script uses the following Unicode tables:
+# - NormalizationTest.txt
+#
+# Since this should not require frequent updates, we just store this
+# out-of-line and check the unicode.rs file into git.
+
+import unicode, re, os, fileinput
+
+def load_test_data(f):
+    outls = []
+    testRe = re.compile("^(.*?);(.*?);(.*?);(.*?);(.*?);\s+#.*$")
+
+    unicode.fetch(f)
+    for line in fileinput.input(os.path.basename(f)):
+        # comment and header lines start with # and @ respectively
+        if len(line) < 1 or line[0:1] == '#' or line[0:1] == '@':
+            continue
+
+        m = testRe.match(line)
+        groups = []
+        if not m:
+            print "error: no match on line where test was expected: %s" % line
+            continue
+
+        has_surrogates = False
+        for i in range(1, 6):
+            group = []
+            chs = m.group(i).split()
+            for ch in chs:
+                intch = int(ch,16)
+                if unicode.is_surrogate(intch):
+                    has_surrogates = True
+                    break
+                group.append(intch)
+
+            if has_surrogates:
+                break
+            groups.append(group)
+
+        if has_surrogates:
+            continue
+        outls.append(groups)
+
+    return outls
+
+def showfun(gs):
+    outstr = '('
+    gfirst = True
+    for g in gs:
+        if not gfirst:
+            outstr += ','
+        gfirst = False
+
+        outstr += '"'
+        for ch in g:
+            outstr += "\\u{%x}" % ch
+        outstr += '"'
+    outstr += ')'
+    return outstr
+
+if __name__ == "__main__":
+    d = load_test_data("NormalizationTest.txt")
+    ntype = "&'static [(&'static str, &'static str, &'static str, &'static str, &'static str)]"
+    with open("testdata.rs", "w") as nf:
+        nf.write(unicode.preamble)
+        nf.write("\n")
+        nf.write("    // official Unicode test data\n")
+        nf.write("    // http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt\n")
+        unicode.emit_table(nf, "TEST_NORM", d, ntype, True, showfun)

third_party/rust/unicode-normalization/src/ReadMe.txt

@@ -0,0 +1,17 @@
+# Date: 2015-06-16, 20:24:00 GMT [KW]
+#
+# Unicode Character Database
+# Copyright (c) 1991-2015 Unicode, Inc.
+# For terms of use, see http://www.unicode.org/terms_of_use.html
+#
+# For documentation, see the following:
+# NamesList.html
+# UAX #38, "Unicode Han Database (Unihan)"
+# UAX #44, "Unicode Character Database."
+#
+# The UAXes can be accessed at http://www.unicode.org/versions/Unicode8.0.0/
+
+This directory contains the final data files
+for the Unicode Character Database, for Version 8.0.0 of the Unicode
+Standard.
+

third_party/rust/unicode-normalization/src/decompose.rs

@@ -0,0 +1,135 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+
+// Helper functions used for Unicode normalization
+fn canonical_sort(comb: &mut [(char, u8)]) {
+    let len = comb.len();
+    for i in 0..len {
+        let mut swapped = false;
+        for j in 1..len-i {
+            let class_a = comb[j-1].1;
+            let class_b = comb[j].1;
+            if class_a != 0 && class_b != 0 && class_a > class_b {
+                comb.swap(j-1, j);
+                swapped = true;
+            }
+        }
+        if !swapped { break; }
+    }
+}
+
+#[derive(Clone)]
+enum DecompositionType {
+    Canonical,
+    Compatible
+}
+
+/// External iterator for a string decomposition's characters.
+#[derive(Clone)]
+pub struct Decompositions<I> {
+    kind: DecompositionType,
+    iter: I,
+    buffer: Vec<(char, u8)>,
+    sorted: bool
+}
+
+#[inline]
+pub fn new_canonical<I: Iterator<Item=char>>(iter: I) -> Decompositions<I> {
+    Decompositions {
+        iter: iter,
+        buffer: Vec::new(),
+        sorted: false,
+        kind: self::DecompositionType::Canonical,
+    }
+}
+
+#[inline]
+pub fn new_compatible<I: Iterator<Item=char>>(iter: I) -> Decompositions<I> {
+    Decompositions {
+        iter: iter,
+        buffer: Vec::new(),
+        sorted: false,
+        kind: self::DecompositionType::Compatible,
+    }
+}
+
+impl<I: Iterator<Item=char>> Iterator for Decompositions<I> {
+    type Item = char;
+
+    #[inline]
+    fn next(&mut self) -> Option<char> {
+        use self::DecompositionType::*;
+
+        match self.buffer.first() {
+            Some(&(c, 0)) => {
+                self.sorted = false;
+                self.buffer.remove(0);
+                return Some(c);
+            }
+            Some(&(c, _)) if self.sorted => {
+                self.buffer.remove(0);
+                return Some(c);
+            }
+            _ => self.sorted = false
+        }
+
+        if !self.sorted {
+            for ch in self.iter.by_ref() {
+                let buffer = &mut self.buffer;
+                let sorted = &mut self.sorted;
+                {
+                    let callback = |d| {
+                        let class =
+                            super::char::canonical_combining_class(d);
+                        if class == 0 && !*sorted {
+                            canonical_sort(buffer);
+                            *sorted = true;
+                        }
+                        buffer.push((d, class));
+                    };
+                    match self.kind {
+                        Canonical => {
+                            super::char::decompose_canonical(ch, callback)
+                        }
+                        Compatible => {
+                            super::char::decompose_compatible(ch, callback)
+                        }
+                    }
+                }
+                if *sorted {
+                    break
+                }
+            }
+        }
+
+        if !self.sorted {
+            canonical_sort(&mut self.buffer);
+            self.sorted = true;
+        }
+
+        if self.buffer.is_empty() {
+            None
+        } else {
+            match self.buffer.remove(0) {
+                (c, 0) => {
+                    self.sorted = false;
+                    Some(c)
+                }
+                (c, _) => Some(c),
+            }
+        }
+    }
+
+    fn size_hint(&self) -> (usize, Option<usize>) {
+        let (lower, _) = self.iter.size_hint();
+        (lower, None)
+    }
+}

third_party/rust/unicode-normalization/src/lib.rs

@@ -0,0 +1,138 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Unicode character composition and decomposition utilities
+//! as described in
+//! [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/).
+//!
+//! ```rust
+//! extern crate unicode_normalization;
+//!
+//! use unicode_normalization::char::compose;
+//! use unicode_normalization::UnicodeNormalization;
+//!
+//! fn main() {
+//!     assert_eq!(compose('A','\u{30a}'), Some('Å'));
+//!
+//!     let s = "ÅΩ";
+//!     let c = s.nfc().collect::<String>();
+//!     assert_eq!(c, "ÅΩ");
+//! }
+//! ```
+//!
+//! # crates.io
+//!
+//! You can use this package in your project by adding the following
+//! to your `Cargo.toml`:
+//!
+//! ```toml
+//! [dependencies]
+//! unicode-normalization = "0.1.0"
+//! ```
+
+#![deny(missing_docs, unsafe_code)]
+#![doc(html_logo_url = "https://unicode-rs.github.io/unicode-rs_sm.png",
+       html_favicon_url = "https://unicode-rs.github.io/unicode-rs_sm.png")]
+
+pub use tables::UNICODE_VERSION;
+pub use decompose::Decompositions;
+pub use recompose::Recompositions;
+use std::str::Chars;
+
+mod decompose;
+mod normalize;
+mod recompose;
+mod tables;
+
+#[cfg(test)]
+mod test;
+#[cfg(test)]
+mod testdata;
+
+/// Methods for composing and decomposing characters.
+pub mod char {
+    pub use normalize::{decompose_canonical, decompose_compatible, compose};
+
+    /// Look up the canonical combining class of a character.
+    pub use tables::normalization::canonical_combining_class;
+
+    /// Return whether the given character is a combining mark (`General_Category=Mark`)
+    pub use tables::normalization::is_combining_mark;
+}
+
+
+/// Methods for iterating over strings while applying Unicode normalizations
+/// as described in
+/// [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/).
+pub trait UnicodeNormalization<I: Iterator<Item=char>> {
+    /// Returns an iterator over the string in Unicode Normalization Form D
+    /// (canonical decomposition).
+    #[inline]
+    fn nfd(self) -> Decompositions<I>;
+
+    /// Returns an iterator over the string in Unicode Normalization Form KD
+    /// (compatibility decomposition).
+    #[inline]
+    fn nfkd(self) -> Decompositions<I>;
+
+    /// An Iterator over the string in Unicode Normalization Form C
+    /// (canonical decomposition followed by canonical composition).
+    #[inline]
+    fn nfc(self) -> Recompositions<I>;
+
+    /// An Iterator over the string in Unicode Normalization Form KC
+    /// (compatibility decomposition followed by canonical composition).
+    #[inline]
+    fn nfkc(self) -> Recompositions<I>;
+}
+
+impl<'a> UnicodeNormalization<Chars<'a>> for &'a str {
+    #[inline]
+    fn nfd(self) -> Decompositions<Chars<'a>> {
+        decompose::new_canonical(self.chars())
+    }
+
+    #[inline]
+    fn nfkd(self) -> Decompositions<Chars<'a>> {
+        decompose::new_compatible(self.chars())
+    }
+
+    #[inline]
+    fn nfc(self) -> Recompositions<Chars<'a>> {
+        recompose::new_canonical(self.chars())
+    }
+
+    #[inline]
+    fn nfkc(self) -> Recompositions<Chars<'a>> {
+        recompose::new_compatible(self.chars())
+    }
+}
+
+impl<I: Iterator<Item=char>> UnicodeNormalization<I> for I {
+    #[inline]
+    fn nfd(self) -> Decompositions<I> {
+        decompose::new_canonical(self)
+    }
+
+    #[inline]
+    fn nfkd(self) -> Decompositions<I> {
+        decompose::new_compatible(self)
+    }
+
+    #[inline]
+    fn nfc(self) -> Recompositions<I> {
+        recompose::new_canonical(self)
+    }
+
+    #[inline]
+    fn nfkc(self) -> Recompositions<I> {
+        recompose::new_compatible(self)
+    }
+}

third_party/rust/unicode-normalization/src/normalize.rs

@@ -0,0 +1,158 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Functions for computing canonical and compatible decompositions for Unicode characters.
+
+use std::cmp::Ordering::{Equal, Less, Greater};
+use std::ops::FnMut;
+use tables::normalization::{canonical_table, compatibility_table, composition_table};
+
+fn bsearch_table<T>(c: char, r: &'static [(char, &'static [T])]) -> Option<&'static [T]> {
+    match r.binary_search_by(|&(val, _)| {
+        if c == val { Equal }
+        else if val < c { Less }
+        else { Greater }
+    }) {
+        Ok(idx) => {
+            let (_, result) = r[idx];
+            Some(result)
+        }
+        Err(_) => None
+    }
+}
+
+/// Compute canonical Unicode decomposition for character.
+/// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
+/// for more information.
+pub fn decompose_canonical<F>(c: char, mut i: F) where F: FnMut(char) { d(c, &mut i, false); }
+
+/// Compute canonical or compatible Unicode decomposition for character.
+/// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
+/// for more information.
+pub fn decompose_compatible<F>(c: char, mut i: F) where F: FnMut(char) { d(c, &mut i, true); }
+
+// FIXME(#19596) This is a workaround, we should use `F` instead of `&mut F`
+fn d<F>(c: char, i: &mut F, k: bool) where F: FnMut(char) {
+    // 7-bit ASCII never decomposes
+    if c <= '\x7f' { (*i)(c); return; }
+
+    // Perform decomposition for Hangul
+    if (c as u32) >= S_BASE && (c as u32) < (S_BASE + S_COUNT) {
+        decompose_hangul(c, i);
+        return;
+    }
+
+    // First check the canonical decompositions
+    match bsearch_table(c, canonical_table) {
+        Some(canon) => {
+            for x in canon {
+                d(*x, i, k);
+            }
+            return;
+        }
+        None => ()
+    }
+
+    // Bottom out if we're not doing compat.
+    if !k { (*i)(c); return; }
+
+    // Then check the compatibility decompositions
+    match bsearch_table(c, compatibility_table) {
+        Some(compat) => {
+            for x in compat {
+                d(*x, i, k);
+            }
+            return;
+        }
+        None => ()
+    }
+
+    // Finally bottom out.
+    (*i)(c);
+}
+
+/// Compose two characters into a single character, if possible.
+/// See [Unicode Standard Annex #15](http://www.unicode.org/reports/tr15/)
+/// for more information.
+pub fn compose(a: char, b: char) -> Option<char> {
+    compose_hangul(a, b).or_else(|| {
+        match bsearch_table(a, composition_table) {
+            None => None,
+            Some(candidates) => {
+                match candidates.binary_search_by(|&(val, _)| {
+                    if b == val { Equal }
+                    else if val < b { Less }
+                    else { Greater }
+                }) {
+                    Ok(idx) => {
+                        let (_, result) = candidates[idx];
+                        Some(result)
+                    }
+                    Err(_) => None
+                }
+            }
+        }
+    })
+}
+
+// Constants from Unicode 7.0.0 Section 3.12 Conjoining Jamo Behavior
+const S_BASE: u32 = 0xAC00;
+const L_BASE: u32 = 0x1100;
+const V_BASE: u32 = 0x1161;
+const T_BASE: u32 = 0x11A7;
+const L_COUNT: u32 = 19;
+const V_COUNT: u32 = 21;
+const T_COUNT: u32 = 28;
+const N_COUNT: u32 = (V_COUNT * T_COUNT);
+const S_COUNT: u32 = (L_COUNT * N_COUNT);
+
+// FIXME(#19596) This is a workaround, we should use `F` instead of `&mut F`
+// Decompose a precomposed Hangul syllable
+#[allow(unsafe_code)]
+#[inline(always)]
+fn decompose_hangul<F>(s: char, f: &mut F) where F: FnMut(char) {
+    use std::mem::transmute;
+
+    let si = s as u32 - S_BASE;
+
+    let li = si / N_COUNT;
+    unsafe {
+        (*f)(transmute(L_BASE + li));
+
+        let vi = (si % N_COUNT) / T_COUNT;
+        (*f)(transmute(V_BASE + vi));
+
+        let ti = si % T_COUNT;
+        if ti > 0 {
+            (*f)(transmute(T_BASE + ti));
+        }
+    }
+}
+
+// Compose a pair of Hangul Jamo
+#[allow(unsafe_code)]
+#[inline(always)]
+fn compose_hangul(a: char, b: char) -> Option<char> {
+    use std::mem::transmute;
+
+    let l = a as u32;
+    let v = b as u32;
+    // Compose an LPart and a VPart
+    if L_BASE <= l && l < (L_BASE + L_COUNT) && V_BASE <= v && v < (V_BASE + V_COUNT) {
+        let r = S_BASE + (l - L_BASE) * N_COUNT + (v - V_BASE) * T_COUNT;
+        return unsafe { Some(transmute(r)) };
+    }
+    // Compose an LVPart and a TPart
+    if S_BASE <= l && l <= (S_BASE+S_COUNT-T_COUNT) && T_BASE <= v && v < (T_BASE+T_COUNT) {
+        let r = l + (v - T_BASE);
+        return unsafe { Some(transmute(r)) };
+    }
+    None
+}

third_party/rust/unicode-normalization/src/recompose.rs

@@ -0,0 +1,137 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::collections::VecDeque;
+use decompose::Decompositions;
+
+#[derive(Clone)]
+enum RecompositionState {
+    Composing,
+    Purging,
+    Finished
+}
+
+/// External iterator for a string recomposition's characters.
+#[derive(Clone)]
+pub struct Recompositions<I> {
+    iter: Decompositions<I>,
+    state: RecompositionState,
+    buffer: VecDeque<char>,
+    composee: Option<char>,
+    last_ccc: Option<u8>
+}
+
+#[inline]
+pub fn new_canonical<I: Iterator<Item=char>>(iter: I) -> Recompositions<I> {
+    Recompositions {
+        iter: super::decompose::new_canonical(iter),
+        state: self::RecompositionState::Composing,
+        buffer: VecDeque::new(),
+        composee: None,
+        last_ccc: None,
+    }
+}
+
+#[inline]
+pub fn new_compatible<I: Iterator<Item=char>>(iter: I) -> Recompositions<I> {
+    Recompositions {
+        iter: super::decompose::new_compatible(iter),
+        state : self::RecompositionState::Composing,
+        buffer: VecDeque::new(),
+        composee: None,
+        last_ccc: None,
+    }
+}
+
+impl<I: Iterator<Item=char>> Iterator for Recompositions<I> {
+    type Item = char;
+
+    #[inline]
+    fn next(&mut self) -> Option<char> {
+        use self::RecompositionState::*;
+
+        loop {
+            match self.state {
+                Composing => {
+                    for ch in self.iter.by_ref() {
+                        let ch_class = super::char::canonical_combining_class(ch);
+                        if self.composee.is_none() {
+                            if ch_class != 0 {
+                                return Some(ch);
+                            }
+                            self.composee = Some(ch);
+                            continue;
+                        }
+                        let k = self.composee.clone().unwrap();
+
+                        match self.last_ccc {
+                            None => {
+                                match super::char::compose(k, ch) {
+                                    Some(r) => {
+                                        self.composee = Some(r);
+                                        continue;
+                                    }
+                                    None => {
+                                        if ch_class == 0 {
+                                            self.composee = Some(ch);
+                                            return Some(k);
+                                        }
+                                        self.buffer.push_back(ch);
+                                        self.last_ccc = Some(ch_class);
+                                    }
+                                }
+                            }
+                            Some(l_class) => {
+                                if l_class >= ch_class {
+                                    // `ch` is blocked from `composee`
+                                    if ch_class == 0 {
+                                        self.composee = Some(ch);
+                                        self.last_ccc = None;
+                                        self.state = Purging;
+                                        return Some(k);
+                                    }
+                                    self.buffer.push_back(ch);
+                                    self.last_ccc = Some(ch_class);
+                                    continue;
+                                }
+                                match super::char::compose(k, ch) {
+                                    Some(r) => {
+                                        self.composee = Some(r);
+                                        continue;
+                                    }
+                                    None => {
+                                        self.buffer.push_back(ch);
+                                        self.last_ccc = Some(ch_class);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    self.state = Finished;
+                    if self.composee.is_some() {
+                        return self.composee.take();
+                    }
+                }
+                Purging => {
+                    match self.buffer.pop_front() {
+                        None => self.state = Composing,
+                        s => return s
+                    }
+                }
+                Finished => {
+                    match self.buffer.pop_front() {
+                        None => return self.composee.take(),
+                        s => return s
+                    }
+                }
+            }
+        }
+    }
+}

third_party/rust/unicode-normalization/src/test.rs

@@ -0,0 +1,155 @@
+// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use UnicodeNormalization;
+
+#[test]
+fn test_nfd() {
+    macro_rules! t {
+        ($input: expr, $expected: expr) => {
+            assert_eq!($input.nfd().collect::<String>(), $expected);
+            // A dummy iterator that is not std::str::Chars directly:
+            assert_eq!($input.chars().map(|c| c).nfd().collect::<String>(), $expected);
+        }
+    }
+    t!("abc", "abc");
+    t!("\u{1e0b}\u{1c4}", "d\u{307}\u{1c4}");
+    t!("\u{2026}", "\u{2026}");
+    t!("\u{2126}", "\u{3a9}");
+    t!("\u{1e0b}\u{323}", "d\u{323}\u{307}");
+    t!("\u{1e0d}\u{307}", "d\u{323}\u{307}");
+    t!("a\u{301}", "a\u{301}");
+    t!("\u{301}a", "\u{301}a");
+    t!("\u{d4db}", "\u{1111}\u{1171}\u{11b6}");
+    t!("\u{ac1c}", "\u{1100}\u{1162}");
+}
+
+#[test]
+fn test_nfkd() {
+    macro_rules! t {
+        ($input: expr, $expected: expr) => {
+            assert_eq!($input.nfkd().collect::<String>(), $expected);
+        }
+    }
+    t!("abc", "abc");
+    t!("\u{1e0b}\u{1c4}", "d\u{307}DZ\u{30c}");
+    t!("\u{2026}", "...");
+    t!("\u{2126}", "\u{3a9}");
+    t!("\u{1e0b}\u{323}", "d\u{323}\u{307}");
+    t!("\u{1e0d}\u{307}", "d\u{323}\u{307}");
+    t!("a\u{301}", "a\u{301}");
+    t!("\u{301}a", "\u{301}a");
+    t!("\u{d4db}", "\u{1111}\u{1171}\u{11b6}");
+    t!("\u{ac1c}", "\u{1100}\u{1162}");
+}
+
+#[test]
+fn test_nfc() {
+    macro_rules! t {
+        ($input: expr, $expected: expr) => {
+            assert_eq!($input.nfc().collect::<String>(), $expected);
+        }
+    }
+    t!("abc", "abc");
+    t!("\u{1e0b}\u{1c4}", "\u{1e0b}\u{1c4}");
+    t!("\u{2026}", "\u{2026}");
+    t!("\u{2126}", "\u{3a9}");
+    t!("\u{1e0b}\u{323}", "\u{1e0d}\u{307}");
+    t!("\u{1e0d}\u{307}", "\u{1e0d}\u{307}");
+    t!("a\u{301}", "\u{e1}");
+    t!("\u{301}a", "\u{301}a");
+    t!("\u{d4db}", "\u{d4db}");
+    t!("\u{ac1c}", "\u{ac1c}");
+    t!("a\u{300}\u{305}\u{315}\u{5ae}b", "\u{e0}\u{5ae}\u{305}\u{315}b");
+}
+
+#[test]
+fn test_nfkc() {
+    macro_rules! t {
+        ($input: expr, $expected: expr) => {
+            assert_eq!($input.nfkc().collect::<String>(), $expected);
+        }
+    }
+    t!("abc", "abc");
+    t!("\u{1e0b}\u{1c4}", "\u{1e0b}D\u{17d}");
+    t!("\u{2026}", "...");
+    t!("\u{2126}", "\u{3a9}");
+    t!("\u{1e0b}\u{323}", "\u{1e0d}\u{307}");
+    t!("\u{1e0d}\u{307}", "\u{1e0d}\u{307}");
+    t!("a\u{301}", "\u{e1}");
+    t!("\u{301}a", "\u{301}a");
+    t!("\u{d4db}", "\u{d4db}");
+    t!("\u{ac1c}", "\u{ac1c}");
+    t!("a\u{300}\u{305}\u{315}\u{5ae}b", "\u{e0}\u{5ae}\u{305}\u{315}b");
+}
+
+#[test]
+fn test_official() {
+    use testdata::TEST_NORM;
+    macro_rules! normString {
+        ($method: ident, $input: expr) => { $input.$method().collect::<String>() }
+    }
+
+    for &(s1, s2, s3, s4, s5) in TEST_NORM {
+        // these invariants come from the CONFORMANCE section of
+        // http://www.unicode.org/Public/UNIDATA/NormalizationTest.txt
+        {
+            let r1 = normString!(nfc, s1);
+            let r2 = normString!(nfc, s2);
+            let r3 = normString!(nfc, s3);
+            let r4 = normString!(nfc, s4);
+            let r5 = normString!(nfc, s5);
+            assert_eq!(s2, &r1[..]);
+            assert_eq!(s2, &r2[..]);
+            assert_eq!(s2, &r3[..]);
+            assert_eq!(s4, &r4[..]);
+            assert_eq!(s4, &r5[..]);
+        }
+
+        {
+            let r1 = normString!(nfd, s1);
+            let r2 = normString!(nfd, s2);
+            let r3 = normString!(nfd, s3);
+            let r4 = normString!(nfd, s4);
+            let r5 = normString!(nfd, s5);
+            assert_eq!(s3, &r1[..]);
+            assert_eq!(s3, &r2[..]);
+            assert_eq!(s3, &r3[..]);
+            assert_eq!(s5, &r4[..]);
+            assert_eq!(s5, &r5[..]);
+        }
+
+        {
+            let r1 = normString!(nfkc, s1);
+            let r2 = normString!(nfkc, s2);
+            let r3 = normString!(nfkc, s3);
+            let r4 = normString!(nfkc, s4);
+            let r5 = normString!(nfkc, s5);
+            assert_eq!(s4, &r1[..]);
+            assert_eq!(s4, &r2[..]);
+            assert_eq!(s4, &r3[..]);
+            assert_eq!(s4, &r4[..]);
+            assert_eq!(s4, &r5[..]);
+        }
+
+        {
+            let r1 = normString!(nfkd, s1);
+            let r2 = normString!(nfkd, s2);
+            let r3 = normString!(nfkd, s3);
+            let r4 = normString!(nfkd, s4);
+            let r5 = normString!(nfkd, s5);
+            assert_eq!(s5, &r1[..]);
+            assert_eq!(s5, &r2[..]);
+            assert_eq!(s5, &r3[..]);
+            assert_eq!(s5, &r4[..]);
+            assert_eq!(s5, &r5[..]);
+        }
+    }
+}