import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

python/mozbuild/mozpack/test/data/test_data

@@ -0,0 +1 @@
+test_data

python/mozbuild/mozpack/test/support/minify_js_verify.py

@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+import sys
+
+
+if len(sys.argv) != 4:
+    raise Exception('Usage: minify_js_verify <exitcode> <orig> <minified>')
+
+retcode = int(sys.argv[1])
+
+if retcode:
+    print('Error message', file=sys.stderr)
+
+sys.exit(retcode)

python/mozbuild/mozpack/test/test_archive.py

@@ -0,0 +1,190 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import hashlib
+import os
+import shutil
+import stat
+import tarfile
+import tempfile
+import unittest
+
+from mozpack.archive import (
+    DEFAULT_MTIME,
+    create_tar_from_files,
+    create_tar_gz_from_files,
+    create_tar_bz2_from_files,
+)
+
+from mozunit import main
+
+
+MODE_STANDARD = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
+
+
+def file_hash(path):
+    h = hashlib.sha1()
+    with open(path, 'rb') as fh:
+        while True:
+            data = fh.read(8192)
+            if not data:
+                break
+            h.update(data)
+
+    return h.hexdigest()
+
+
+class TestArchive(unittest.TestCase):
+    def _create_files(self, root):
+        files = {}
+        for i in range(10):
+            p = os.path.join(root, b'file%d' % i)
+            with open(p, 'wb') as fh:
+                fh.write(b'file%d' % i)
+            # Need to set permissions or umask may influence testing.
+            os.chmod(p, MODE_STANDARD)
+            files[b'file%d' % i] = p
+
+        return files
+
+    def _verify_basic_tarfile(self, tf):
+        self.assertEqual(len(tf.getmembers()), 10)
+
+        names = ['file%d' % i for i in range(10)]
+        self.assertEqual(tf.getnames(), names)
+
+        for ti in tf.getmembers():
+            self.assertEqual(ti.uid, 0)
+            self.assertEqual(ti.gid, 0)
+            self.assertEqual(ti.uname, '')
+            self.assertEqual(ti.gname, '')
+            self.assertEqual(ti.mode, MODE_STANDARD)
+            self.assertEqual(ti.mtime, DEFAULT_MTIME)
+
+    def test_dirs_refused(self):
+        d = tempfile.mkdtemp()
+        try:
+            tp = os.path.join(d, 'test.tar')
+            with open(tp, 'wb') as fh:
+                with self.assertRaisesRegexp(ValueError, 'not a regular'):
+                    create_tar_from_files(fh, {'test': d})
+        finally:
+            shutil.rmtree(d)
+
+    def test_setuid_setgid_refused(self):
+        d = tempfile.mkdtemp()
+        try:
+            uid = os.path.join(d, 'setuid')
+            gid = os.path.join(d, 'setgid')
+            with open(uid, 'a'):
+                pass
+            with open(gid, 'a'):
+                pass
+
+            os.chmod(uid, MODE_STANDARD | stat.S_ISUID)
+            os.chmod(gid, MODE_STANDARD | stat.S_ISGID)
+
+            tp = os.path.join(d, 'test.tar')
+            with open(tp, 'wb') as fh:
+                with self.assertRaisesRegexp(ValueError, 'cannot add file with setuid'):
+                    create_tar_from_files(fh, {'test': uid})
+                with self.assertRaisesRegexp(ValueError, 'cannot add file with setuid'):
+                    create_tar_from_files(fh, {'test': gid})
+        finally:
+            shutil.rmtree(d)
+
+    def test_create_tar_basic(self):
+        d = tempfile.mkdtemp()
+        try:
+            files = self._create_files(d)
+
+            tp = os.path.join(d, 'test.tar')
+            with open(tp, 'wb') as fh:
+                create_tar_from_files(fh, files)
+
+            # Output should be deterministic.
+            self.assertEqual(file_hash(tp), 'cd16cee6f13391abd94dfa435d2633b61ed727f1')
+
+            with tarfile.open(tp, 'r') as tf:
+                self._verify_basic_tarfile(tf)
+
+        finally:
+            shutil.rmtree(d)
+
+    def test_executable_preserved(self):
+        d = tempfile.mkdtemp()
+        try:
+            p = os.path.join(d, 'exec')
+            with open(p, 'wb') as fh:
+                fh.write('#!/bin/bash\n')
+            os.chmod(p, MODE_STANDARD | stat.S_IXUSR)
+
+            tp = os.path.join(d, 'test.tar')
+            with open(tp, 'wb') as fh:
+                create_tar_from_files(fh, {'exec': p})
+
+            self.assertEqual(file_hash(tp), '357e1b81c0b6cfdfa5d2d118d420025c3c76ee93')
+
+            with tarfile.open(tp, 'r') as tf:
+                m = tf.getmember('exec')
+                self.assertEqual(m.mode, MODE_STANDARD | stat.S_IXUSR)
+
+        finally:
+            shutil.rmtree(d)
+
+    def test_create_tar_gz_basic(self):
+        d = tempfile.mkdtemp()
+        try:
+            files = self._create_files(d)
+
+            gp = os.path.join(d, 'test.tar.gz')
+            with open(gp, 'wb') as fh:
+                create_tar_gz_from_files(fh, files)
+
+            self.assertEqual(file_hash(gp), 'acb602239c1aeb625da5e69336775609516d60f5')
+
+            with tarfile.open(gp, 'r:gz') as tf:
+                self._verify_basic_tarfile(tf)
+
+        finally:
+            shutil.rmtree(d)
+
+    def test_tar_gz_name(self):
+        d = tempfile.mkdtemp()
+        try:
+            files = self._create_files(d)
+
+            gp = os.path.join(d, 'test.tar.gz')
+            with open(gp, 'wb') as fh:
+                create_tar_gz_from_files(fh, files, filename='foobar', compresslevel=1)
+
+            self.assertEqual(file_hash(gp), 'fd099f96480cc1100f37baa8e89a6b820dbbcbd3')
+
+            with tarfile.open(gp, 'r:gz') as tf:
+                self._verify_basic_tarfile(tf)
+
+        finally:
+            shutil.rmtree(d)
+
+    def test_create_tar_bz2_basic(self):
+        d = tempfile.mkdtemp()
+        try:
+            files = self._create_files(d)
+
+            bp = os.path.join(d, 'test.tar.bz2')
+            with open(bp, 'wb') as fh:
+                create_tar_bz2_from_files(fh, files)
+
+            self.assertEqual(file_hash(bp), '1827ad00dfe7acf857b7a1c95ce100361e3f6eea')
+
+            with tarfile.open(bp, 'r:bz2') as tf:
+                self._verify_basic_tarfile(tf)
+        finally:
+            shutil.rmtree(d)
+
+
+if __name__ == '__main__':
+    main()

python/mozbuild/mozpack/test/test_chrome_flags.py

@@ -0,0 +1,148 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+from mozpack.chrome.flags import (
+    Flag,
+    StringFlag,
+    VersionFlag,
+    Flags,
+)
+from mozpack.errors import ErrorMessage
+
+
+class TestFlag(unittest.TestCase):
+    def test_flag(self):
+        flag = Flag('flag')
+        self.assertEqual(str(flag), '')
+        self.assertTrue(flag.matches(False))
+        self.assertTrue(flag.matches('false'))
+        self.assertFalse(flag.matches('true'))
+        self.assertRaises(ErrorMessage, flag.add_definition, 'flag=')
+        self.assertRaises(ErrorMessage, flag.add_definition, 'flag=42')
+        self.assertRaises(ErrorMessage, flag.add_definition, 'flag!=false')
+
+        flag.add_definition('flag=1')
+        self.assertEqual(str(flag), 'flag=1')
+        self.assertTrue(flag.matches(True))
+        self.assertTrue(flag.matches('1'))
+        self.assertFalse(flag.matches('no'))
+
+        flag.add_definition('flag=true')
+        self.assertEqual(str(flag), 'flag=true')
+        self.assertTrue(flag.matches(True))
+        self.assertTrue(flag.matches('true'))
+        self.assertFalse(flag.matches('0'))
+
+        flag.add_definition('flag=no')
+        self.assertEqual(str(flag), 'flag=no')
+        self.assertTrue(flag.matches('false'))
+        self.assertFalse(flag.matches('1'))
+
+        flag.add_definition('flag')
+        self.assertEqual(str(flag), 'flag')
+        self.assertFalse(flag.matches('false'))
+        self.assertTrue(flag.matches('true'))
+        self.assertFalse(flag.matches(False))
+
+    def test_string_flag(self):
+        flag = StringFlag('flag')
+        self.assertEqual(str(flag), '')
+        self.assertTrue(flag.matches('foo'))
+        self.assertRaises(ErrorMessage, flag.add_definition, 'flag>=2')
+
+        flag.add_definition('flag=foo')
+        self.assertEqual(str(flag), 'flag=foo')
+        self.assertTrue(flag.matches('foo'))
+        self.assertFalse(flag.matches('bar'))
+
+        flag.add_definition('flag=bar')
+        self.assertEqual(str(flag), 'flag=foo flag=bar')
+        self.assertTrue(flag.matches('foo'))
+        self.assertTrue(flag.matches('bar'))
+        self.assertFalse(flag.matches('baz'))
+
+        flag = StringFlag('flag')
+        flag.add_definition('flag!=bar')
+        self.assertEqual(str(flag), 'flag!=bar')
+        self.assertTrue(flag.matches('foo'))
+        self.assertFalse(flag.matches('bar'))
+
+    def test_version_flag(self):
+        flag = VersionFlag('flag')
+        self.assertEqual(str(flag), '')
+        self.assertTrue(flag.matches('1.0'))
+        self.assertRaises(ErrorMessage, flag.add_definition, 'flag!=2')
+
+        flag.add_definition('flag=1.0')
+        self.assertEqual(str(flag), 'flag=1.0')
+        self.assertTrue(flag.matches('1.0'))
+        self.assertFalse(flag.matches('2.0'))
+
+        flag.add_definition('flag=2.0')
+        self.assertEqual(str(flag), 'flag=1.0 flag=2.0')
+        self.assertTrue(flag.matches('1.0'))
+        self.assertTrue(flag.matches('2.0'))
+        self.assertFalse(flag.matches('3.0'))
+
+        flag = VersionFlag('flag')
+        flag.add_definition('flag>=2.0')
+        self.assertEqual(str(flag), 'flag>=2.0')
+        self.assertFalse(flag.matches('1.0'))
+        self.assertTrue(flag.matches('2.0'))
+        self.assertTrue(flag.matches('3.0'))
+
+        flag.add_definition('flag<1.10')
+        self.assertEqual(str(flag), 'flag>=2.0 flag<1.10')
+        self.assertTrue(flag.matches('1.0'))
+        self.assertTrue(flag.matches('1.9'))
+        self.assertFalse(flag.matches('1.10'))
+        self.assertFalse(flag.matches('1.20'))
+        self.assertTrue(flag.matches('2.0'))
+        self.assertTrue(flag.matches('3.0'))
+        self.assertRaises(Exception, flag.add_definition, 'flag<')
+        self.assertRaises(Exception, flag.add_definition, 'flag>')
+        self.assertRaises(Exception, flag.add_definition, 'flag>=')
+        self.assertRaises(Exception, flag.add_definition, 'flag<=')
+        self.assertRaises(Exception, flag.add_definition, 'flag!=1.0')
+
+
+class TestFlags(unittest.TestCase):
+    def setUp(self):
+        self.flags = Flags('contentaccessible=yes',
+                           'appversion>=3.5',
+                           'application=foo',
+                           'application=bar',
+                           'appversion<2.0',
+                           'platform',
+                           'abi!=Linux_x86-gcc3')
+
+    def test_flags_str(self):
+        self.assertEqual(str(self.flags), 'contentaccessible=yes ' +
+                         'appversion>=3.5 appversion<2.0 application=foo ' +
+                         'application=bar platform abi!=Linux_x86-gcc3')
+
+    def test_flags_match_unset(self):
+        self.assertTrue(self.flags.match(os='WINNT'))
+
+    def test_flags_match(self):
+        self.assertTrue(self.flags.match(application='foo'))
+        self.assertFalse(self.flags.match(application='qux'))
+
+    def test_flags_match_different(self):
+        self.assertTrue(self.flags.match(abi='WINNT_x86-MSVC'))
+        self.assertFalse(self.flags.match(abi='Linux_x86-gcc3'))
+
+    def test_flags_match_version(self):
+        self.assertTrue(self.flags.match(appversion='1.0'))
+        self.assertTrue(self.flags.match(appversion='1.5'))
+        self.assertFalse(self.flags.match(appversion='2.0'))
+        self.assertFalse(self.flags.match(appversion='3.0'))
+        self.assertTrue(self.flags.match(appversion='3.5'))
+        self.assertTrue(self.flags.match(appversion='3.10'))
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_chrome_manifest.py

@@ -0,0 +1,149 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+import os
+from mozpack.chrome.manifest import (
+    ManifestContent,
+    ManifestLocale,
+    ManifestSkin,
+    Manifest,
+    ManifestResource,
+    ManifestOverride,
+    ManifestComponent,
+    ManifestContract,
+    ManifestInterfaces,
+    ManifestBinaryComponent,
+    ManifestCategory,
+    ManifestStyle,
+    ManifestOverlay,
+    MANIFESTS_TYPES,
+    parse_manifest,
+    parse_manifest_line,
+)
+from mozpack.errors import errors, AccumulatedErrors
+from test_errors import TestErrors
+
+
+class TestManifest(unittest.TestCase):
+    def test_parse_manifest(self):
+        manifest = [
+            'content global content/global/',
+            'content global content/global/ application=foo application=bar' +
+            ' platform',
+            'locale global en-US content/en-US/',
+            'locale global en-US content/en-US/ application=foo',
+            'skin global classic/1.0 content/skin/classic/',
+            'skin global classic/1.0 content/skin/classic/ application=foo' +
+            ' os=WINNT',
+            '',
+            'manifest pdfjs/chrome.manifest',
+            'resource gre-resources toolkit/res/',
+            'override chrome://global/locale/netError.dtd' +
+            ' chrome://browser/locale/netError.dtd',
+            '# Comment',
+            'component {b2bba4df-057d-41ea-b6b1-94a10a8ede68} foo.js',
+            'contract @mozilla.org/foo;1' +
+            ' {b2bba4df-057d-41ea-b6b1-94a10a8ede68}',
+            'interfaces foo.xpt',
+            'binary-component bar.so',
+            'category command-line-handler m-browser' +
+            ' @mozilla.org/browser/clh;1' +
+            ' application={ec8030f7-c20a-464f-9b0e-13a3a9e97384}',
+            'style chrome://global/content/customizeToolbar.xul' +
+            ' chrome://browser/skin/',
+            'overlay chrome://global/content/viewSource.xul' +
+            ' chrome://browser/content/viewSourceOverlay.xul',
+        ]
+        other_manifest = [
+            'content global content/global/'
+        ]
+        expected_result = [
+            ManifestContent('', 'global', 'content/global/'),
+            ManifestContent('', 'global', 'content/global/', 'application=foo',
+                            'application=bar', 'platform'),
+            ManifestLocale('', 'global', 'en-US', 'content/en-US/'),
+            ManifestLocale('', 'global', 'en-US', 'content/en-US/',
+                           'application=foo'),
+            ManifestSkin('', 'global', 'classic/1.0', 'content/skin/classic/'),
+            ManifestSkin('', 'global', 'classic/1.0', 'content/skin/classic/',
+                         'application=foo', 'os=WINNT'),
+            Manifest('', 'pdfjs/chrome.manifest'),
+            ManifestResource('', 'gre-resources', 'toolkit/res/'),
+            ManifestOverride('', 'chrome://global/locale/netError.dtd',
+                             'chrome://browser/locale/netError.dtd'),
+            ManifestComponent('', '{b2bba4df-057d-41ea-b6b1-94a10a8ede68}',
+                              'foo.js'),
+            ManifestContract('', '@mozilla.org/foo;1',
+                             '{b2bba4df-057d-41ea-b6b1-94a10a8ede68}'),
+            ManifestInterfaces('', 'foo.xpt'),
+            ManifestBinaryComponent('', 'bar.so'),
+            ManifestCategory('', 'command-line-handler', 'm-browser',
+                             '@mozilla.org/browser/clh;1', 'application=' +
+                             '{ec8030f7-c20a-464f-9b0e-13a3a9e97384}'),
+            ManifestStyle('', 'chrome://global/content/customizeToolbar.xul',
+                          'chrome://browser/skin/'),
+            ManifestOverlay('', 'chrome://global/content/viewSource.xul',
+                            'chrome://browser/content/viewSourceOverlay.xul'),
+        ]
+        with mozunit.MockedOpen({'manifest': '\n'.join(manifest),
+                                 'other/manifest': '\n'.join(other_manifest)}):
+            # Ensure we have tests for all types of manifests.
+            self.assertEqual(set(type(e) for e in expected_result),
+                             set(MANIFESTS_TYPES.values()))
+            self.assertEqual(list(parse_manifest(os.curdir, 'manifest')),
+                             expected_result)
+            self.assertEqual(list(parse_manifest(os.curdir, 'other/manifest')),
+                             [ManifestContent('other', 'global',
+                                              'content/global/')])
+
+    def test_manifest_rebase(self):
+        m = parse_manifest_line('chrome', 'content global content/global/')
+        m = m.rebase('')
+        self.assertEqual(str(m), 'content global chrome/content/global/')
+        m = m.rebase('chrome')
+        self.assertEqual(str(m), 'content global content/global/')
+
+        m = parse_manifest_line('chrome/foo', 'content global content/global/')
+        m = m.rebase('chrome')
+        self.assertEqual(str(m), 'content global foo/content/global/')
+        m = m.rebase('chrome/foo')
+        self.assertEqual(str(m), 'content global content/global/')
+
+        m = parse_manifest_line('modules/foo', 'resource foo ./')
+        m = m.rebase('modules')
+        self.assertEqual(str(m), 'resource foo foo/')
+        m = m.rebase('modules/foo')
+        self.assertEqual(str(m), 'resource foo ./')
+
+        m = parse_manifest_line('chrome', 'content browser browser/content/')
+        m = m.rebase('chrome/browser').move('jar:browser.jar!').rebase('')
+        self.assertEqual(str(m), 'content browser jar:browser.jar!/content/')
+
+
+class TestManifestErrors(TestErrors, unittest.TestCase):
+    def test_parse_manifest_errors(self):
+        manifest = [
+            'skin global classic/1.0 content/skin/classic/ platform',
+            '',
+            'binary-component bar.so',
+            'unsupported foo',
+        ]
+        with mozunit.MockedOpen({'manifest': '\n'.join(manifest)}):
+            with self.assertRaises(AccumulatedErrors):
+                with errors.accumulate():
+                    list(parse_manifest(os.curdir, 'manifest'))
+            out = self.get_output()
+            # Expecting 2 errors
+            self.assertEqual(len(out), 2)
+            path = os.path.abspath('manifest')
+            # First on line 1
+            self.assertTrue(out[0].startswith('Error: %s:1: ' % path))
+            # Second on line 4
+            self.assertTrue(out[1].startswith('Error: %s:4: ' % path))
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_copier.py

@@ -0,0 +1,529 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+    FileRegistrySubtree,
+    Jarrer,
+)
+from mozpack.files import (
+    GeneratedFile,
+    ExistingFile,
+)
+from mozpack.mozjar import JarReader
+import mozpack.path as mozpath
+import unittest
+import mozunit
+import os
+import stat
+from mozpack.errors import ErrorMessage
+from mozpack.test.test_files import (
+    MockDest,
+    MatchTestTemplate,
+    TestWithTmpDir,
+)
+
+
+class BaseTestFileRegistry(MatchTestTemplate):
+    def add(self, path):
+        self.registry.add(path, GeneratedFile(path))
+
+    def do_check(self, pattern, result):
+        self.checked = True
+        if result:
+            self.assertTrue(self.registry.contains(pattern))
+        else:
+            self.assertFalse(self.registry.contains(pattern))
+        self.assertEqual(self.registry.match(pattern), result)
+
+    def do_test_file_registry(self, registry):
+        self.registry = registry
+        self.registry.add('foo', GeneratedFile('foo'))
+        bar = GeneratedFile('bar')
+        self.registry.add('bar', bar)
+        self.assertEqual(self.registry.paths(), ['foo', 'bar'])
+        self.assertEqual(self.registry['bar'], bar)
+
+        self.assertRaises(ErrorMessage, self.registry.add, 'foo',
+                          GeneratedFile('foo2'))
+
+        self.assertRaises(ErrorMessage, self.registry.remove, 'qux')
+
+        self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
+                          GeneratedFile('foobar'))
+        self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar/baz',
+                          GeneratedFile('foobar'))
+
+        self.assertEqual(self.registry.paths(), ['foo', 'bar'])
+
+        self.registry.remove('foo')
+        self.assertEqual(self.registry.paths(), ['bar'])
+        self.registry.remove('bar')
+        self.assertEqual(self.registry.paths(), [])
+
+        self.prepare_match_test()
+        self.do_match_test()
+        self.assertTrue(self.checked)
+        self.assertEqual(self.registry.paths(), [
+            'bar',
+            'foo/bar',
+            'foo/baz',
+            'foo/qux/1',
+            'foo/qux/bar',
+            'foo/qux/2/test',
+            'foo/qux/2/test2',
+        ])
+
+        self.registry.remove('foo/qux')
+        self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz'])
+
+        self.registry.add('foo/qux', GeneratedFile('fooqux'))
+        self.assertEqual(self.registry.paths(), ['bar', 'foo/bar', 'foo/baz',
+                                                 'foo/qux'])
+        self.registry.remove('foo/b*')
+        self.assertEqual(self.registry.paths(), ['bar', 'foo/qux'])
+
+        self.assertEqual([f for f, c in self.registry], ['bar', 'foo/qux'])
+        self.assertEqual(len(self.registry), 2)
+
+        self.add('foo/.foo')
+        self.assertTrue(self.registry.contains('foo/.foo'))
+
+    def do_test_registry_paths(self, registry):
+        self.registry = registry
+
+        # Can't add a file if it requires a directory in place of a
+        # file we also require.
+        self.registry.add('foo', GeneratedFile('foo'))
+        self.assertRaises(ErrorMessage, self.registry.add, 'foo/bar',
+                          GeneratedFile('foobar'))
+
+        # Can't add a file if we already have a directory there.
+        self.registry.add('bar/baz', GeneratedFile('barbaz'))
+        self.assertRaises(ErrorMessage, self.registry.add, 'bar',
+                          GeneratedFile('bar'))
+
+        # Bump the count of things that require bar/ to 2.
+        self.registry.add('bar/zot', GeneratedFile('barzot'))
+        self.assertRaises(ErrorMessage, self.registry.add, 'bar',
+                          GeneratedFile('bar'))
+
+        # Drop the count of things that require bar/ to 1.
+        self.registry.remove('bar/baz')
+        self.assertRaises(ErrorMessage, self.registry.add, 'bar',
+                          GeneratedFile('bar'))
+
+        # Drop the count of things that require bar/ to 0.
+        self.registry.remove('bar/zot')
+        self.registry.add('bar/zot', GeneratedFile('barzot'))
+
+class TestFileRegistry(BaseTestFileRegistry, unittest.TestCase):
+    def test_partial_paths(self):
+        cases = {
+            'foo/bar/baz/zot': ['foo/bar/baz', 'foo/bar', 'foo'],
+            'foo/bar': ['foo'],
+            'bar': [],
+        }
+        reg = FileRegistry()
+        for path, parts in cases.iteritems():
+            self.assertEqual(reg._partial_paths(path), parts)
+
+    def test_file_registry(self):
+        self.do_test_file_registry(FileRegistry())
+
+    def test_registry_paths(self):
+        self.do_test_registry_paths(FileRegistry())
+
+    def test_required_directories(self):
+        self.registry = FileRegistry()
+
+        self.registry.add('foo', GeneratedFile('foo'))
+        self.assertEqual(self.registry.required_directories(), set())
+
+        self.registry.add('bar/baz', GeneratedFile('barbaz'))
+        self.assertEqual(self.registry.required_directories(), {'bar'})
+
+        self.registry.add('bar/zot', GeneratedFile('barzot'))
+        self.assertEqual(self.registry.required_directories(), {'bar'})
+
+        self.registry.add('bar/zap/zot', GeneratedFile('barzapzot'))
+        self.assertEqual(self.registry.required_directories(), {'bar', 'bar/zap'})
+
+        self.registry.remove('bar/zap/zot')
+        self.assertEqual(self.registry.required_directories(), {'bar'})
+
+        self.registry.remove('bar/baz')
+        self.assertEqual(self.registry.required_directories(), {'bar'})
+
+        self.registry.remove('bar/zot')
+        self.assertEqual(self.registry.required_directories(), set())
+
+        self.registry.add('x/y/z', GeneratedFile('xyz'))
+        self.assertEqual(self.registry.required_directories(), {'x', 'x/y'})
+
+
+class TestFileRegistrySubtree(BaseTestFileRegistry, unittest.TestCase):
+    def test_file_registry_subtree_base(self):
+        registry = FileRegistry()
+        self.assertEqual(registry, FileRegistrySubtree('', registry))
+        self.assertNotEqual(registry, FileRegistrySubtree('base', registry))
+
+    def create_registry(self):
+        registry = FileRegistry()
+        registry.add('foo/bar', GeneratedFile('foo/bar'))
+        registry.add('baz/qux', GeneratedFile('baz/qux'))
+        return FileRegistrySubtree('base/root', registry)
+
+    def test_file_registry_subtree(self):
+        self.do_test_file_registry(self.create_registry())
+
+    def test_registry_paths_subtree(self):
+        registry = FileRegistry()
+        self.do_test_registry_paths(self.create_registry())
+
+
+class TestFileCopier(TestWithTmpDir):
+    def all_dirs(self, base):
+        all_dirs = set()
+        for root, dirs, files in os.walk(base):
+            if not dirs:
+                all_dirs.add(mozpath.relpath(root, base))
+        return all_dirs
+
+    def all_files(self, base):
+        all_files = set()
+        for root, dirs, files in os.walk(base):
+            for f in files:
+                all_files.add(
+                    mozpath.join(mozpath.relpath(root, base), f))
+        return all_files
+
+    def test_file_copier(self):
+        copier = FileCopier()
+        copier.add('foo/bar', GeneratedFile('foobar'))
+        copier.add('foo/qux', GeneratedFile('fooqux'))
+        copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
+        copier.add('bar', GeneratedFile('bar'))
+        copier.add('qux/foo', GeneratedFile('quxfoo'))
+        copier.add('qux/bar', GeneratedFile(''))
+
+        result = copier.copy(self.tmpdir)
+        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
+        self.assertEqual(self.all_dirs(self.tmpdir),
+                         set(['foo/deep/nested/directory', 'qux']))
+
+        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in
+            self.all_files(self.tmpdir)))
+        self.assertEqual(result.existing_files, set())
+        self.assertEqual(result.removed_files, set())
+        self.assertEqual(result.removed_directories, set())
+
+        copier.remove('foo')
+        copier.add('test', GeneratedFile('test'))
+        result = copier.copy(self.tmpdir)
+        self.assertEqual(self.all_files(self.tmpdir), set(copier.paths()))
+        self.assertEqual(self.all_dirs(self.tmpdir), set(['qux']))
+        self.assertEqual(result.removed_files, set(self.tmppath(p) for p in
+            ('foo/bar', 'foo/qux', 'foo/deep/nested/directory/file')))
+
+    def test_symlink_directory_replaced(self):
+        """Directory symlinks in destination are replaced if they need to be
+        real directories."""
+        if not self.symlink_supported:
+            return
+
+        dest = self.tmppath('dest')
+
+        copier = FileCopier()
+        copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))
+
+        os.makedirs(self.tmppath('dest/foo'))
+        dummy = self.tmppath('dummy')
+        os.mkdir(dummy)
+        link = self.tmppath('dest/foo/bar')
+        os.symlink(dummy, link)
+
+        result = copier.copy(dest)
+
+        st = os.lstat(link)
+        self.assertFalse(stat.S_ISLNK(st.st_mode))
+        self.assertTrue(stat.S_ISDIR(st.st_mode))
+
+        self.assertEqual(self.all_files(dest), set(copier.paths()))
+
+        self.assertEqual(result.removed_directories, set())
+        self.assertEqual(len(result.updated_files), 1)
+
+    def test_remove_unaccounted_directory_symlinks(self):
+        """Directory symlinks in destination that are not in the way are
+        deleted according to remove_unaccounted and
+        remove_all_directory_symlinks.
+        """
+        if not self.symlink_supported:
+            return
+
+        dest = self.tmppath('dest')
+
+        copier = FileCopier()
+        copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))
+
+        os.makedirs(self.tmppath('dest/foo'))
+        dummy = self.tmppath('dummy')
+        os.mkdir(dummy)
+
+        os.mkdir(self.tmppath('dest/zot'))
+        link = self.tmppath('dest/zot/zap')
+        os.symlink(dummy, link)
+
+        # If not remove_unaccounted but remove_empty_directories, then
+        # the symlinked directory remains (as does its containing
+        # directory).
+        result = copier.copy(dest, remove_unaccounted=False,
+            remove_empty_directories=True,
+            remove_all_directory_symlinks=False)
+
+        st = os.lstat(link)
+        self.assertTrue(stat.S_ISLNK(st.st_mode))
+        self.assertFalse(stat.S_ISDIR(st.st_mode))
+
+        self.assertEqual(self.all_files(dest), set(copier.paths()))
+        self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
+
+        self.assertEqual(result.removed_directories, set())
+        self.assertEqual(len(result.updated_files), 1)
+
+        # If remove_unaccounted but not remove_empty_directories, then
+        # only the symlinked directory is removed.
+        result = copier.copy(dest, remove_unaccounted=True,
+            remove_empty_directories=False,
+            remove_all_directory_symlinks=False)
+
+        st = os.lstat(self.tmppath('dest/zot'))
+        self.assertFalse(stat.S_ISLNK(st.st_mode))
+        self.assertTrue(stat.S_ISDIR(st.st_mode))
+
+        self.assertEqual(result.removed_files, set([link]))
+        self.assertEqual(result.removed_directories, set())
+
+        self.assertEqual(self.all_files(dest), set(copier.paths()))
+        self.assertEqual(self.all_dirs(dest), set(['foo/bar', 'zot']))
+
+        # If remove_unaccounted and remove_empty_directories, then
+        # both the symlink and its containing directory are removed.
+        link = self.tmppath('dest/zot/zap')
+        os.symlink(dummy, link)
+
+        result = copier.copy(dest, remove_unaccounted=True,
+            remove_empty_directories=True,
+            remove_all_directory_symlinks=False)
+
+        self.assertEqual(result.removed_files, set([link]))
+        self.assertEqual(result.removed_directories, set([self.tmppath('dest/zot')]))
+
+        self.assertEqual(self.all_files(dest), set(copier.paths()))
+        self.assertEqual(self.all_dirs(dest), set(['foo/bar']))
+
+    def test_permissions(self):
+        """Ensure files without write permission can be deleted."""
+        with open(self.tmppath('dummy'), 'a'):
+            pass
+
+        p = self.tmppath('no_perms')
+        with open(p, 'a'):
+            pass
+
+        # Make file and directory unwritable. Reminder: making a directory
+        # unwritable prevents modifications (including deletes) from the list
+        # of files in that directory.
+        os.chmod(p, 0o400)
+        os.chmod(self.tmpdir, 0o400)
+
+        copier = FileCopier()
+        copier.add('dummy', GeneratedFile('content'))
+        result = copier.copy(self.tmpdir)
+        self.assertEqual(result.removed_files_count, 1)
+        self.assertFalse(os.path.exists(p))
+
+    def test_no_remove(self):
+        copier = FileCopier()
+        copier.add('foo', GeneratedFile('foo'))
+
+        with open(self.tmppath('bar'), 'a'):
+            pass
+
+        os.mkdir(self.tmppath('emptydir'))
+        d = self.tmppath('populateddir')
+        os.mkdir(d)
+
+        with open(self.tmppath('populateddir/foo'), 'a'):
+            pass
+
+        result = copier.copy(self.tmpdir, remove_unaccounted=False)
+
+        self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar',
+            'populateddir/foo']))
+        self.assertEqual(self.all_dirs(self.tmpdir), set(['populateddir']))
+        self.assertEqual(result.removed_files, set())
+        self.assertEqual(result.removed_directories,
+            set([self.tmppath('emptydir')]))
+
+    def test_no_remove_empty_directories(self):
+        copier = FileCopier()
+        copier.add('foo', GeneratedFile('foo'))
+
+        with open(self.tmppath('bar'), 'a'):
+            pass
+
+        os.mkdir(self.tmppath('emptydir'))
+        d = self.tmppath('populateddir')
+        os.mkdir(d)
+
+        with open(self.tmppath('populateddir/foo'), 'a'):
+            pass
+
+        result = copier.copy(self.tmpdir, remove_unaccounted=False,
+            remove_empty_directories=False)
+
+        self.assertEqual(self.all_files(self.tmpdir), set(['foo', 'bar',
+            'populateddir/foo']))
+        self.assertEqual(self.all_dirs(self.tmpdir), set(['emptydir',
+            'populateddir']))
+        self.assertEqual(result.removed_files, set())
+        self.assertEqual(result.removed_directories, set())
+
+    def test_optional_exists_creates_unneeded_directory(self):
+        """Demonstrate that a directory not strictly required, but specified
+        as the path to an optional file, will be unnecessarily created.
+
+        This behaviour is wrong; fixing it is tracked by Bug 972432;
+        and this test exists to guard against unexpected changes in
+        behaviour.
+        """
+
+        dest = self.tmppath('dest')
+
+        copier = FileCopier()
+        copier.add('foo/bar', ExistingFile(required=False))
+
+        result = copier.copy(dest)
+
+        st = os.lstat(self.tmppath('dest/foo'))
+        self.assertFalse(stat.S_ISLNK(st.st_mode))
+        self.assertTrue(stat.S_ISDIR(st.st_mode))
+
+        # What's worse, we have no record that dest was created.
+        self.assertEquals(len(result.updated_files), 0)
+
+        # But we do have an erroneous record of an optional file
+        # existing when it does not.
+        self.assertIn(self.tmppath('dest/foo/bar'), result.existing_files)
+
+    def test_remove_unaccounted_file_registry(self):
+        """Test FileCopier.copy(remove_unaccounted=FileRegistry())"""
+
+        dest = self.tmppath('dest')
+
+        copier = FileCopier()
+        copier.add('foo/bar/baz', GeneratedFile('foobarbaz'))
+        copier.add('foo/bar/qux', GeneratedFile('foobarqux'))
+        copier.add('foo/hoge/fuga', GeneratedFile('foohogefuga'))
+        copier.add('foo/toto/tata', GeneratedFile('footototata'))
+
+        os.makedirs(os.path.join(dest, 'bar'))
+        with open(os.path.join(dest, 'bar', 'bar'), 'w') as fh:
+            fh.write('barbar');
+        os.makedirs(os.path.join(dest, 'foo', 'toto'))
+        with open(os.path.join(dest, 'foo', 'toto', 'toto'), 'w') as fh:
+            fh.write('foototototo');
+
+        result = copier.copy(dest, remove_unaccounted=False)
+
+        self.assertEqual(self.all_files(dest),
+                         set(copier.paths()) | { 'foo/toto/toto', 'bar/bar'})
+        self.assertEqual(self.all_dirs(dest),
+                         {'foo/bar', 'foo/hoge', 'foo/toto', 'bar'})
+
+        copier2 = FileCopier()
+        copier2.add('foo/hoge/fuga', GeneratedFile('foohogefuga'))
+
+        # We expect only files copied from the first copier to be removed,
+        # not the extra file that was there beforehand.
+        result = copier2.copy(dest, remove_unaccounted=copier)
+
+        self.assertEqual(self.all_files(dest),
+                         set(copier2.paths()) | { 'foo/toto/toto', 'bar/bar'})
+        self.assertEqual(self.all_dirs(dest),
+                         {'foo/hoge', 'foo/toto', 'bar'})
+        self.assertEqual(result.updated_files,
+                         {self.tmppath('dest/foo/hoge/fuga')})
+        self.assertEqual(result.existing_files, set())
+        self.assertEqual(result.removed_files, {self.tmppath(p) for p in
+            ('dest/foo/bar/baz', 'dest/foo/bar/qux', 'dest/foo/toto/tata')})
+        self.assertEqual(result.removed_directories,
+                         {self.tmppath('dest/foo/bar')})
+
+
+class TestJarrer(unittest.TestCase):
+    def check_jar(self, dest, copier):
+        jar = JarReader(fileobj=dest)
+        self.assertEqual([f.filename for f in jar], copier.paths())
+        for f in jar:
+            self.assertEqual(f.uncompressed_data.read(),
+                             copier[f.filename].content)
+
+    def test_jarrer(self):
+        copier = Jarrer()
+        copier.add('foo/bar', GeneratedFile('foobar'))
+        copier.add('foo/qux', GeneratedFile('fooqux'))
+        copier.add('foo/deep/nested/directory/file', GeneratedFile('fooz'))
+        copier.add('bar', GeneratedFile('bar'))
+        copier.add('qux/foo', GeneratedFile('quxfoo'))
+        copier.add('qux/bar', GeneratedFile(''))
+
+        dest = MockDest()
+        copier.copy(dest)
+        self.check_jar(dest, copier)
+
+        copier.remove('foo')
+        copier.add('test', GeneratedFile('test'))
+        copier.copy(dest)
+        self.check_jar(dest, copier)
+
+        copier.remove('test')
+        copier.add('test', GeneratedFile('replaced-content'))
+        copier.copy(dest)
+        self.check_jar(dest, copier)
+
+        copier.copy(dest)
+        self.check_jar(dest, copier)
+
+        preloaded = ['qux/bar', 'bar']
+        copier.preload(preloaded)
+        copier.copy(dest)
+
+        dest.seek(0)
+        jar = JarReader(fileobj=dest)
+        self.assertEqual([f.filename for f in jar], preloaded +
+                         [p for p in copier.paths() if not p in preloaded])
+        self.assertEqual(jar.last_preloaded, preloaded[-1])
+
+
+    def test_jarrer_compress(self):
+        copier = Jarrer()
+        copier.add('foo/bar', GeneratedFile('ffffff'))
+        copier.add('foo/qux', GeneratedFile('ffffff'), compress=False)
+
+        dest = MockDest()
+        copier.copy(dest)
+        self.check_jar(dest, copier)
+
+        dest.seek(0)
+        jar = JarReader(fileobj=dest)
+        self.assertTrue(jar['foo/bar'].compressed)
+        self.assertFalse(jar['foo/qux'].compressed)
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_errors.py

@@ -0,0 +1,93 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozpack.errors import (
+    errors,
+    ErrorMessage,
+    AccumulatedErrors,
+)
+import unittest
+import mozunit
+import sys
+from cStringIO import StringIO
+
+
+class TestErrors(object):
+    def setUp(self):
+        errors.out = StringIO()
+        errors.ignore_errors(False)
+
+    def tearDown(self):
+        errors.out = sys.stderr
+
+    def get_output(self):
+        return [l.strip() for l in errors.out.getvalue().splitlines()]
+
+
+class TestErrorsImpl(TestErrors, unittest.TestCase):
+    def test_plain_error(self):
+        errors.warn('foo')
+        self.assertRaises(ErrorMessage, errors.error, 'foo')
+        self.assertRaises(ErrorMessage, errors.fatal, 'foo')
+        self.assertEquals(self.get_output(), ['Warning: foo'])
+
+    def test_ignore_errors(self):
+        errors.ignore_errors()
+        errors.warn('foo')
+        errors.error('bar')
+        self.assertRaises(ErrorMessage, errors.fatal, 'foo')
+        self.assertEquals(self.get_output(), ['Warning: foo', 'Warning: bar'])
+
+    def test_no_error(self):
+        with errors.accumulate():
+            errors.warn('1')
+
+    def test_simple_error(self):
+        with self.assertRaises(AccumulatedErrors):
+            with errors.accumulate():
+                errors.error('1')
+        self.assertEquals(self.get_output(), ['Error: 1'])
+
+    def test_error_loop(self):
+        with self.assertRaises(AccumulatedErrors):
+            with errors.accumulate():
+                for i in range(3):
+                    errors.error('%d' % i)
+        self.assertEquals(self.get_output(),
+                          ['Error: 0', 'Error: 1', 'Error: 2'])
+
+    def test_multiple_errors(self):
+        with self.assertRaises(AccumulatedErrors):
+            with errors.accumulate():
+                errors.error('foo')
+                for i in range(3):
+                    if i == 2:
+                        errors.warn('%d' % i)
+                    else:
+                        errors.error('%d' % i)
+                errors.error('bar')
+        self.assertEquals(self.get_output(),
+                          ['Error: foo', 'Error: 0', 'Error: 1',
+                           'Warning: 2', 'Error: bar'])
+
+    def test_errors_context(self):
+        with self.assertRaises(AccumulatedErrors):
+            with errors.accumulate():
+                self.assertEqual(errors.get_context(), None)
+                with errors.context('foo', 1):
+                    self.assertEqual(errors.get_context(), ('foo', 1))
+                    errors.error('a')
+                    with errors.context('bar', 2):
+                        self.assertEqual(errors.get_context(), ('bar', 2))
+                        errors.error('b')
+                    self.assertEqual(errors.get_context(), ('foo', 1))
+                    errors.error('c')
+        self.assertEqual(self.get_output(), [
+            'Error: foo:1: a',
+            'Error: bar:2: b',
+            'Error: foo:1: c',
+        ])
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_manifests.py

@@ -0,0 +1,375 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import os
+
+import mozunit
+
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+)
+from mozpack.manifests import (
+    InstallManifest,
+    UnreadableInstallManifest,
+)
+from mozpack.test.test_files import TestWithTmpDir
+
+
+class TestInstallManifest(TestWithTmpDir):
+    def test_construct(self):
+        m = InstallManifest()
+        self.assertEqual(len(m), 0)
+
+    def test_malformed(self):
+        f = self.tmppath('manifest')
+        open(f, 'wb').write('junk\n')
+        with self.assertRaises(UnreadableInstallManifest):
+            m = InstallManifest(f)
+
+    def test_adds(self):
+        m = InstallManifest()
+        m.add_symlink('s_source', 's_dest')
+        m.add_copy('c_source', 'c_dest')
+        m.add_required_exists('e_dest')
+        m.add_optional_exists('o_dest')
+        m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest')
+        m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
+        m.add_preprocess('p_source', 'p_dest', 'p_source.pp')
+        m.add_content('content', 'content')
+
+        self.assertEqual(len(m), 8)
+        self.assertIn('s_dest', m)
+        self.assertIn('c_dest', m)
+        self.assertIn('p_dest', m)
+        self.assertIn('e_dest', m)
+        self.assertIn('o_dest', m)
+        self.assertIn('content', m)
+
+        with self.assertRaises(ValueError):
+            m.add_symlink('s_other', 's_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_copy('c_other', 'c_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_preprocess('p_other', 'p_dest', 'p_other.pp')
+
+        with self.assertRaises(ValueError):
+            m.add_required_exists('e_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_optional_exists('o_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_pattern_symlink('ps_base', 'ps/*', 'ps_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_pattern_copy('pc_base', 'pc/**', 'pc_dest')
+
+        with self.assertRaises(ValueError):
+            m.add_content('content', 'content')
+
+    def _get_test_manifest(self):
+        m = InstallManifest()
+        m.add_symlink(self.tmppath('s_source'), 's_dest')
+        m.add_copy(self.tmppath('c_source'), 'c_dest')
+        m.add_preprocess(self.tmppath('p_source'), 'p_dest', self.tmppath('p_source.pp'), '#', {'FOO':'BAR', 'BAZ':'QUX'})
+        m.add_required_exists('e_dest')
+        m.add_optional_exists('o_dest')
+        m.add_pattern_symlink('ps_base', '*', 'ps_dest')
+        m.add_pattern_copy('pc_base', '**', 'pc_dest')
+        m.add_content('the content\non\nmultiple lines', 'content')
+
+        return m
+
+    def test_serialization(self):
+        m = self._get_test_manifest()
+
+        p = self.tmppath('m')
+        m.write(path=p)
+        self.assertTrue(os.path.isfile(p))
+
+        with open(p, 'rb') as fh:
+            c = fh.read()
+
+        self.assertEqual(c.count('\n'), 9)
+
+        lines = c.splitlines()
+        self.assertEqual(len(lines), 9)
+
+        self.assertEqual(lines[0], '5')
+
+        m2 = InstallManifest(path=p)
+        self.assertEqual(m, m2)
+        p2 = self.tmppath('m2')
+        m2.write(path=p2)
+
+        with open(p2, 'rb') as fh:
+            c2 = fh.read()
+
+        self.assertEqual(c, c2)
+
+    def test_populate_registry(self):
+        m = self._get_test_manifest()
+        r = FileRegistry()
+        m.populate_registry(r)
+
+        self.assertEqual(len(r), 6)
+        self.assertEqual(r.paths(), ['c_dest', 'content', 'e_dest', 'o_dest',
+                                     'p_dest', 's_dest'])
+
+    def test_pattern_expansion(self):
+        source = self.tmppath('source')
+        os.mkdir(source)
+        os.mkdir('%s/base' % source)
+        os.mkdir('%s/base/foo' % source)
+
+        with open('%s/base/foo/file1' % source, 'a'):
+            pass
+
+        with open('%s/base/foo/file2' % source, 'a'):
+            pass
+
+        m = InstallManifest()
+        m.add_pattern_symlink('%s/base' % source, '**', 'dest')
+
+        c = FileCopier()
+        m.populate_registry(c)
+        self.assertEqual(c.paths(), ['dest/foo/file1', 'dest/foo/file2'])
+
+    def test_or(self):
+        m1 = self._get_test_manifest()
+        orig_length = len(m1)
+        m2 = InstallManifest()
+        m2.add_symlink('s_source2', 's_dest2')
+        m2.add_copy('c_source2', 'c_dest2')
+
+        m1 |= m2
+
+        self.assertEqual(len(m2), 2)
+        self.assertEqual(len(m1), orig_length + 2)
+
+        self.assertIn('s_dest2', m1)
+        self.assertIn('c_dest2', m1)
+
+    def test_copier_application(self):
+        dest = self.tmppath('dest')
+        os.mkdir(dest)
+
+        to_delete = self.tmppath('dest/to_delete')
+        with open(to_delete, 'a'):
+            pass
+
+        with open(self.tmppath('s_source'), 'wt') as fh:
+            fh.write('symlink!')
+
+        with open(self.tmppath('c_source'), 'wt') as fh:
+            fh.write('copy!')
+
+        with open(self.tmppath('p_source'), 'wt') as fh:
+            fh.write('#define FOO 1\npreprocess!')
+
+        with open(self.tmppath('dest/e_dest'), 'a'):
+            pass
+
+        with open(self.tmppath('dest/o_dest'), 'a'):
+            pass
+
+        m = self._get_test_manifest()
+        c = FileCopier()
+        m.populate_registry(c)
+        result = c.copy(dest)
+
+        self.assertTrue(os.path.exists(self.tmppath('dest/s_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/c_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/e_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/o_dest')))
+        self.assertTrue(os.path.exists(self.tmppath('dest/content')))
+        self.assertFalse(os.path.exists(to_delete))
+
+        with open(self.tmppath('dest/s_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'symlink!')
+
+        with open(self.tmppath('dest/c_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'copy!')
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'preprocess!')
+
+        self.assertEqual(result.updated_files, set(self.tmppath(p) for p in (
+            'dest/s_dest', 'dest/c_dest', 'dest/p_dest', 'dest/content')))
+        self.assertEqual(result.existing_files,
+            set([self.tmppath('dest/e_dest'), self.tmppath('dest/o_dest')]))
+        self.assertEqual(result.removed_files, {to_delete})
+        self.assertEqual(result.removed_directories, set())
+
+    def test_preprocessor(self):
+        manifest = self.tmppath('m')
+        deps = self.tmppath('m.pp')
+        dest = self.tmppath('dest')
+        include = self.tmppath('p_incl')
+
+        with open(include, 'wt') as fh:
+            fh.write('#define INCL\n')
+        time = os.path.getmtime(include) - 3
+        os.utime(include, (time, time))
+
+        with open(self.tmppath('p_source'), 'wt') as fh:
+            fh.write('#ifdef FOO\n#if BAZ == QUX\nPASS1\n#endif\n#endif\n')
+            fh.write('#ifdef DEPTEST\nPASS2\n#endif\n')
+            fh.write('#include p_incl\n#ifdef INCLTEST\nPASS3\n#endif\n')
+        time = os.path.getmtime(self.tmppath('p_source')) - 3
+        os.utime(self.tmppath('p_source'), (time, time))
+
+        # Create and write a manifest with the preprocessed file, then apply it.
+        # This should write out our preprocessed file.
+        m = InstallManifest()
+        m.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {'FOO':'BAR', 'BAZ':'QUX'})
+        m.write(path=manifest)
+
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        self.assertTrue(os.path.exists(self.tmppath('dest/p_dest')))
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'PASS1\n')
+
+        # Create a second manifest with the preprocessed file, then apply it.
+        # Since this manifest does not exist on the disk, there should not be a
+        # dependency on it, and the preprocessed file should not be modified.
+        m2 = InstallManifest()
+        m2.add_preprocess(self.tmppath('p_source'), 'p_dest', deps, '#', {'DEPTEST':True})
+        c = FileCopier()
+        m2.populate_registry(c)
+        result = c.copy(dest)
+
+        self.assertFalse(self.tmppath('dest/p_dest') in result.updated_files)
+        self.assertTrue(self.tmppath('dest/p_dest') in result.existing_files)
+
+        # Write out the second manifest, then load it back in from the disk.
+        # This should add the dependency on the manifest file, so our
+        # preprocessed file should be regenerated with the new defines.
+        # We also set the mtime on the destination file back, so it will be
+        # older than the manifest file.
+        m2.write(path=manifest)
+        time = os.path.getmtime(manifest) - 1
+        os.utime(self.tmppath('dest/p_dest'), (time, time))
+        m2 = InstallManifest(path=manifest)
+        c = FileCopier()
+        m2.populate_registry(c)
+        self.assertTrue(c.copy(dest))
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'PASS2\n')
+
+        # Set the time on the manifest back, so it won't be picked up as
+        # modified in the next test
+        time = os.path.getmtime(manifest) - 1
+        os.utime(manifest, (time, time))
+
+        # Update the contents of a file included by the source file. This should
+        # cause the destination to be regenerated.
+        with open(include, 'wt') as fh:
+            fh.write('#define INCLTEST\n')
+
+        time = os.path.getmtime(include) - 1
+        os.utime(self.tmppath('dest/p_dest'), (time, time))
+        c = FileCopier()
+        m2.populate_registry(c)
+        self.assertTrue(c.copy(dest))
+
+        with open(self.tmppath('dest/p_dest'), 'rt') as fh:
+            self.assertEqual(fh.read(), 'PASS2\nPASS3\n')
+
+    def test_preprocessor_dependencies(self):
+        manifest = self.tmppath('m')
+        deps = self.tmppath('m.pp')
+        dest = self.tmppath('dest')
+        source = self.tmppath('p_source')
+        destfile = self.tmppath('dest/p_dest')
+        include = self.tmppath('p_incl')
+        os.mkdir(dest)
+
+        with open(source, 'wt') as fh:
+            fh.write('#define SRC\nSOURCE\n')
+        time = os.path.getmtime(source) - 3
+        os.utime(source, (time, time))
+
+        with open(include, 'wt') as fh:
+            fh.write('INCLUDE\n')
+        time = os.path.getmtime(source) - 3
+        os.utime(include, (time, time))
+
+        # Create and write a manifest with the preprocessed file.
+        m = InstallManifest()
+        m.add_preprocess(source, 'p_dest', deps, '#', {'FOO':'BAR', 'BAZ':'QUX'})
+        m.write(path=manifest)
+
+        time = os.path.getmtime(source) - 5
+        os.utime(manifest, (time, time))
+
+        # Now read the manifest back in, and apply it. This should write out
+        # our preprocessed file.
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        self.assertTrue(c.copy(dest))
+
+        with open(destfile, 'rt') as fh:
+            self.assertEqual(fh.read(), 'SOURCE\n')
+
+        # Next, modify the source to #INCLUDE another file.
+        with open(source, 'wt') as fh:
+            fh.write('SOURCE\n#include p_incl\n')
+        time = os.path.getmtime(source) - 1
+        os.utime(destfile, (time, time))
+
+        # Apply the manifest, and confirm that it also reads the newly included
+        # file.
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        with open(destfile, 'rt') as fh:
+            self.assertEqual(fh.read(), 'SOURCE\nINCLUDE\n')
+
+        # Set the time on the source file back, so it won't be picked up as
+        # modified in the next test.
+        time = os.path.getmtime(source) - 1
+        os.utime(source, (time, time))
+
+        # Now, modify the include file (but not the original source).
+        with open(include, 'wt') as fh:
+            fh.write('INCLUDE MODIFIED\n')
+        time = os.path.getmtime(include) - 1
+        os.utime(destfile, (time, time))
+
+        # Apply the manifest, and confirm that the change to the include file
+        # is detected. That should cause the preprocessor to run again.
+        m = InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        c.copy(dest)
+
+        with open(destfile, 'rt') as fh:
+            self.assertEqual(fh.read(), 'SOURCE\nINCLUDE MODIFIED\n')
+
+        # ORing an InstallManifest should copy file dependencies
+        m = InstallManifest()
+        m |= InstallManifest(path=manifest)
+        c = FileCopier()
+        m.populate_registry(c)
+        e = c._files['p_dest']
+        self.assertEqual(e.extra_depends, [manifest])
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_mozjar.py

@@ -0,0 +1,342 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozpack.files import FileFinder
+from mozpack.mozjar import (
+    JarReaderError,
+    JarWriterError,
+    JarStruct,
+    JarReader,
+    JarWriter,
+    Deflater,
+    JarLog,
+)
+from collections import OrderedDict
+from mozpack.test.test_files import MockDest
+import unittest
+import mozunit
+from cStringIO import StringIO
+from urllib import pathname2url
+import mozpack.path as mozpath
+import os
+
+
+test_data_path = mozpath.abspath(mozpath.dirname(__file__))
+test_data_path = mozpath.join(test_data_path, 'data')
+
+
+class TestJarStruct(unittest.TestCase):
+    class Foo(JarStruct):
+        MAGIC = 0x01020304
+        STRUCT = OrderedDict([
+            ('foo', 'uint32'),
+            ('bar', 'uint16'),
+            ('qux', 'uint16'),
+            ('length', 'uint16'),
+            ('length2', 'uint16'),
+            ('string', 'length'),
+            ('string2', 'length2'),
+        ])
+
+    def test_jar_struct(self):
+        foo = TestJarStruct.Foo()
+        self.assertEqual(foo.signature, TestJarStruct.Foo.MAGIC)
+        self.assertEqual(foo['foo'], 0)
+        self.assertEqual(foo['bar'], 0)
+        self.assertEqual(foo['qux'], 0)
+        self.assertFalse('length' in foo)
+        self.assertFalse('length2' in foo)
+        self.assertEqual(foo['string'], '')
+        self.assertEqual(foo['string2'], '')
+
+        self.assertEqual(foo.size, 16)
+
+        foo['foo'] = 0x42434445
+        foo['bar'] = 0xabcd
+        foo['qux'] = 0xef01
+        foo['string'] = 'abcde'
+        foo['string2'] = 'Arbitrarily long string'
+
+        serialized = b'\x04\x03\x02\x01\x45\x44\x43\x42\xcd\xab\x01\xef' + \
+                     b'\x05\x00\x17\x00abcdeArbitrarily long string'
+        self.assertEqual(foo.size, len(serialized))
+        foo_serialized = foo.serialize()
+        self.assertEqual(foo_serialized, serialized)
+
+    def do_test_read_jar_struct(self, data):
+        self.assertRaises(JarReaderError, TestJarStruct.Foo, data)
+        self.assertRaises(JarReaderError, TestJarStruct.Foo, data[2:])
+
+        foo = TestJarStruct.Foo(data[1:])
+        self.assertEqual(foo['foo'], 0x45444342)
+        self.assertEqual(foo['bar'], 0xcdab)
+        self.assertEqual(foo['qux'], 0x01ef)
+        self.assertFalse('length' in foo)
+        self.assertFalse('length2' in foo)
+        self.assertEqual(foo['string'], '012345')
+        self.assertEqual(foo['string2'], '67')
+
+    def test_read_jar_struct(self):
+        data = b'\x00\x04\x03\x02\x01\x42\x43\x44\x45\xab\xcd\xef' + \
+               b'\x01\x06\x00\x02\x0001234567890'
+        self.do_test_read_jar_struct(data)
+
+    def test_read_jar_struct_memoryview(self):
+        data = b'\x00\x04\x03\x02\x01\x42\x43\x44\x45\xab\xcd\xef' + \
+               b'\x01\x06\x00\x02\x0001234567890'
+        self.do_test_read_jar_struct(memoryview(data))
+
+
+class TestDeflater(unittest.TestCase):
+    def wrap(self, data):
+        return data
+
+    def test_deflater_no_compress(self):
+        deflater = Deflater(False)
+        deflater.write(self.wrap('abc'))
+        self.assertFalse(deflater.compressed)
+        self.assertEqual(deflater.uncompressed_size, 3)
+        self.assertEqual(deflater.compressed_size, deflater.uncompressed_size)
+        self.assertEqual(deflater.compressed_data, 'abc')
+        self.assertEqual(deflater.crc32, 0x352441c2)
+
+    def test_deflater_compress_no_gain(self):
+        deflater = Deflater(True)
+        deflater.write(self.wrap('abc'))
+        self.assertFalse(deflater.compressed)
+        self.assertEqual(deflater.uncompressed_size, 3)
+        self.assertEqual(deflater.compressed_size, deflater.uncompressed_size)
+        self.assertEqual(deflater.compressed_data, 'abc')
+        self.assertEqual(deflater.crc32, 0x352441c2)
+
+    def test_deflater_compress(self):
+        deflater = Deflater(True)
+        deflater.write(self.wrap('aaaaaaaaaaaaanopqrstuvwxyz'))
+        self.assertTrue(deflater.compressed)
+        self.assertEqual(deflater.uncompressed_size, 26)
+        self.assertNotEqual(deflater.compressed_size,
+                            deflater.uncompressed_size)
+        self.assertEqual(deflater.crc32, 0xd46b97ed)
+        # The CRC is the same as when not compressed
+        deflater = Deflater(False)
+        self.assertFalse(deflater.compressed)
+        deflater.write(self.wrap('aaaaaaaaaaaaanopqrstuvwxyz'))
+        self.assertEqual(deflater.crc32, 0xd46b97ed)
+
+
+class TestDeflaterMemoryView(TestDeflater):
+    def wrap(self, data):
+        return memoryview(data)
+
+
+class TestJar(unittest.TestCase):
+    optimize = False
+
+    def test_jar(self):
+        s = MockDest()
+        with JarWriter(fileobj=s, optimize=self.optimize) as jar:
+            jar.add('foo', 'foo')
+            self.assertRaises(JarWriterError, jar.add, 'foo', 'bar')
+            jar.add('bar', 'aaaaaaaaaaaaanopqrstuvwxyz')
+            jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz', False)
+            jar.add('baz\\backslash', 'aaaaaaaaaaaaaaa')
+
+        files = [j for j in JarReader(fileobj=s)]
+
+        self.assertEqual(files[0].filename, 'foo')
+        self.assertFalse(files[0].compressed)
+        self.assertEqual(files[0].read(), 'foo')
+
+        self.assertEqual(files[1].filename, 'bar')
+        self.assertTrue(files[1].compressed)
+        self.assertEqual(files[1].read(), 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+        self.assertEqual(files[2].filename, 'baz/qux')
+        self.assertFalse(files[2].compressed)
+        self.assertEqual(files[2].read(), 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+        if os.sep == '\\':
+            self.assertEqual(files[3].filename, 'baz/backslash',
+                'backslashes in filenames on Windows should get normalized')
+        else:
+            self.assertEqual(files[3].filename, 'baz\\backslash',
+                'backslashes in filenames on POSIX platform are untouched')
+
+        s = MockDest()
+        with JarWriter(fileobj=s, compress=False,
+                       optimize=self.optimize) as jar:
+            jar.add('bar', 'aaaaaaaaaaaaanopqrstuvwxyz')
+            jar.add('foo', 'foo')
+            jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz', True)
+
+        jar = JarReader(fileobj=s)
+        files = [j for j in jar]
+
+        self.assertEqual(files[0].filename, 'bar')
+        self.assertFalse(files[0].compressed)
+        self.assertEqual(files[0].read(), 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+        self.assertEqual(files[1].filename, 'foo')
+        self.assertFalse(files[1].compressed)
+        self.assertEqual(files[1].read(), 'foo')
+
+        self.assertEqual(files[2].filename, 'baz/qux')
+        self.assertTrue(files[2].compressed)
+        self.assertEqual(files[2].read(), 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+        self.assertTrue('bar' in jar)
+        self.assertTrue('foo' in jar)
+        self.assertFalse('baz' in jar)
+        self.assertTrue('baz/qux' in jar)
+        self.assertTrue(jar['bar'], files[1])
+        self.assertTrue(jar['foo'], files[0])
+        self.assertTrue(jar['baz/qux'], files[2])
+
+        s.seek(0)
+        jar = JarReader(fileobj=s)
+        self.assertTrue('bar' in jar)
+        self.assertTrue('foo' in jar)
+        self.assertFalse('baz' in jar)
+        self.assertTrue('baz/qux' in jar)
+
+        files[0].seek(0)
+        self.assertEqual(jar['bar'].filename, files[0].filename)
+        self.assertEqual(jar['bar'].compressed, files[0].compressed)
+        self.assertEqual(jar['bar'].read(), files[0].read())
+
+        files[1].seek(0)
+        self.assertEqual(jar['foo'].filename, files[1].filename)
+        self.assertEqual(jar['foo'].compressed, files[1].compressed)
+        self.assertEqual(jar['foo'].read(), files[1].read())
+
+        files[2].seek(0)
+        self.assertEqual(jar['baz/qux'].filename, files[2].filename)
+        self.assertEqual(jar['baz/qux'].compressed, files[2].compressed)
+        self.assertEqual(jar['baz/qux'].read(), files[2].read())
+
+    def test_rejar(self):
+        s = MockDest()
+        with JarWriter(fileobj=s, optimize=self.optimize) as jar:
+            jar.add('foo', 'foo')
+            jar.add('bar', 'aaaaaaaaaaaaanopqrstuvwxyz')
+            jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz', False)
+
+        new = MockDest()
+        with JarWriter(fileobj=new, optimize=self.optimize) as jar:
+            for j in JarReader(fileobj=s):
+                jar.add(j.filename, j)
+
+        jar = JarReader(fileobj=new)
+        files = [j for j in jar]
+
+        self.assertEqual(files[0].filename, 'foo')
+        self.assertFalse(files[0].compressed)
+        self.assertEqual(files[0].read(), 'foo')
+
+        self.assertEqual(files[1].filename, 'bar')
+        self.assertTrue(files[1].compressed)
+        self.assertEqual(files[1].read(), 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+        self.assertEqual(files[2].filename, 'baz/qux')
+        self.assertTrue(files[2].compressed)
+        self.assertEqual(files[2].read(), 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+    def test_add_from_finder(self):
+        s = MockDest()
+        with JarWriter(fileobj=s, optimize=self.optimize) as jar:
+            finder = FileFinder(test_data_path)
+            for p, f in finder.find('test_data'):
+                jar.add('test_data', f)
+
+        jar = JarReader(fileobj=s)
+        files = [j for j in jar]
+
+        self.assertEqual(files[0].filename, 'test_data')
+        self.assertFalse(files[0].compressed)
+        self.assertEqual(files[0].read(), 'test_data')
+
+
+class TestOptimizeJar(TestJar):
+    optimize = True
+
+
+class TestPreload(unittest.TestCase):
+    def test_preload(self):
+        s = MockDest()
+        with JarWriter(fileobj=s) as jar:
+            jar.add('foo', 'foo')
+            jar.add('bar', 'abcdefghijklmnopqrstuvwxyz')
+            jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz')
+
+        jar = JarReader(fileobj=s)
+        self.assertEqual(jar.last_preloaded, None)
+
+        with JarWriter(fileobj=s) as jar:
+            jar.add('foo', 'foo')
+            jar.add('bar', 'abcdefghijklmnopqrstuvwxyz')
+            jar.add('baz/qux', 'aaaaaaaaaaaaanopqrstuvwxyz')
+            jar.preload(['baz/qux', 'bar'])
+
+        jar = JarReader(fileobj=s)
+        self.assertEqual(jar.last_preloaded, 'bar')
+        files = [j for j in jar]
+
+        self.assertEqual(files[0].filename, 'baz/qux')
+        self.assertEqual(files[1].filename, 'bar')
+        self.assertEqual(files[2].filename, 'foo')
+
+
+class TestJarLog(unittest.TestCase):
+    def test_jarlog(self):
+        base = 'file:' + pathname2url(os.path.abspath(os.curdir))
+        s = StringIO('\n'.join([
+            base + '/bar/baz.jar first',
+            base + '/bar/baz.jar second',
+            base + '/bar/baz.jar third',
+            base + '/bar/baz.jar second',
+            base + '/bar/baz.jar second',
+            'jar:' + base + '/qux.zip!/omni.ja stuff',
+            base + '/bar/baz.jar first',
+            'jar:' + base + '/qux.zip!/omni.ja other/stuff',
+            'jar:' + base + '/qux.zip!/omni.ja stuff',
+            base + '/bar/baz.jar third',
+            'jar:jar:' + base + '/qux.zip!/baz/baz.jar!/omni.ja nested/stuff',
+            'jar:jar:jar:' + base + '/qux.zip!/baz/baz.jar!/foo.zip!/omni.ja' +
+            ' deeply/nested/stuff',
+        ]))
+        log = JarLog(fileobj=s)
+        canonicalize = lambda p: \
+            mozpath.normsep(os.path.normcase(os.path.realpath(p)))
+        baz_jar = canonicalize('bar/baz.jar')
+        qux_zip = canonicalize('qux.zip')
+        self.assertEqual(set(log.keys()), set([
+            baz_jar,
+            (qux_zip, 'omni.ja'),
+            (qux_zip, 'baz/baz.jar', 'omni.ja'),
+            (qux_zip, 'baz/baz.jar', 'foo.zip', 'omni.ja'),
+        ]))
+        self.assertEqual(log[baz_jar], [
+            'first',
+            'second',
+            'third',
+        ])
+        self.assertEqual(log[(qux_zip, 'omni.ja')], [
+            'stuff',
+            'other/stuff',
+        ])
+        self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'omni.ja')],
+                         ['nested/stuff'])
+        self.assertEqual(log[(qux_zip, 'baz/baz.jar', 'foo.zip',
+                              'omni.ja')], ['deeply/nested/stuff'])
+
+        # The above tests also indirectly check the value returned by
+        # JarLog.canonicalize for various jar: and file: urls, but
+        # JarLog.canonicalize also supports plain paths.
+        self.assertEqual(JarLog.canonicalize(os.path.abspath('bar/baz.jar')),
+                         baz_jar)
+        self.assertEqual(JarLog.canonicalize('bar/baz.jar'), baz_jar)
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_packager.py

@@ -0,0 +1,490 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+import os
+from mozpack.packager import (
+    preprocess_manifest,
+    CallDeque,
+    Component,
+    SimplePackager,
+    SimpleManifestSink,
+)
+from mozpack.files import GeneratedFile
+from mozpack.chrome.manifest import (
+    ManifestBinaryComponent,
+    ManifestContent,
+    ManifestResource,
+)
+from mozunit import MockedOpen
+from mozbuild.preprocessor import Preprocessor
+from mozpack.errors import (
+    errors,
+    ErrorMessage,
+)
+import mozpack.path as mozpath
+
+MANIFEST = '''
+bar/*
+[foo]
+foo/*
+-foo/bar
+chrome.manifest
+[zot destdir="destdir"]
+foo/zot
+; comment
+#ifdef baz
+[baz]
+baz@SUFFIX@
+#endif
+'''
+
+
+class TestPreprocessManifest(unittest.TestCase):
+    MANIFEST_PATH = os.path.join(os.path.abspath(os.curdir), 'manifest')
+
+    EXPECTED_LOG = [
+        ((MANIFEST_PATH, 2), 'add', '', 'bar/*'),
+        ((MANIFEST_PATH, 4), 'add', 'foo', 'foo/*'),
+        ((MANIFEST_PATH, 5), 'remove', 'foo', 'foo/bar'),
+        ((MANIFEST_PATH, 6), 'add', 'foo', 'chrome.manifest'),
+        ((MANIFEST_PATH, 8), 'add', 'zot destdir="destdir"', 'foo/zot'),
+    ]
+
+    def setUp(self):
+        class MockSink(object):
+            def __init__(self):
+                self.log = []
+
+            def add(self, component, path):
+                self._log(errors.get_context(), 'add', repr(component), path)
+
+            def remove(self, component, path):
+                self._log(errors.get_context(), 'remove', repr(component), path)
+
+            def _log(self, *args):
+                self.log.append(args)
+
+        self.sink = MockSink()
+
+    def test_preprocess_manifest(self):
+        with MockedOpen({'manifest': MANIFEST}):
+            preprocess_manifest(self.sink, 'manifest')
+        self.assertEqual(self.sink.log, self.EXPECTED_LOG)
+
+    def test_preprocess_manifest_missing_define(self):
+        with MockedOpen({'manifest': MANIFEST}):
+            self.assertRaises(
+                Preprocessor.Error,
+                preprocess_manifest,
+                self.sink,
+                'manifest',
+                {'baz': 1}
+            )
+
+    def test_preprocess_manifest_defines(self):
+        with MockedOpen({'manifest': MANIFEST}):
+            preprocess_manifest(self.sink, 'manifest',
+                                {'baz': 1, 'SUFFIX': '.exe'})
+        self.assertEqual(self.sink.log, self.EXPECTED_LOG +
+                         [((self.MANIFEST_PATH, 12), 'add', 'baz', 'baz.exe')])
+
+
+class MockFinder(object):
+    def __init__(self, files):
+        self.files = files
+        self.log = []
+
+    def find(self, path):
+        self.log.append(path)
+        for f in sorted(self.files):
+            if mozpath.match(f, path):
+                yield f, self.files[f]
+
+    def __iter__(self):
+        return self.find('')
+
+
+class MockFormatter(object):
+    def __init__(self):
+        self.log = []
+
+    def add_base(self, *args):
+        self._log(errors.get_context(), 'add_base', *args)
+
+    def add_manifest(self, *args):
+        self._log(errors.get_context(), 'add_manifest', *args)
+
+    def add_interfaces(self, *args):
+        self._log(errors.get_context(), 'add_interfaces', *args)
+
+    def add(self, *args):
+        self._log(errors.get_context(), 'add', *args)
+
+    def _log(self, *args):
+        self.log.append(args)
+
+
+class TestSimplePackager(unittest.TestCase):
+    def test_simple_packager(self):
+        class GeneratedFileWithPath(GeneratedFile):
+            def __init__(self, path, content):
+                GeneratedFile.__init__(self, content)
+                self.path = path
+
+        formatter = MockFormatter()
+        packager = SimplePackager(formatter)
+        curdir = os.path.abspath(os.curdir)
+        file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
+                                                  'bar.manifest'),
+                                     'resource bar bar/\ncontent bar bar/')
+        with errors.context('manifest', 1):
+            packager.add('foo/bar.manifest', file)
+
+        file = GeneratedFileWithPath(os.path.join(curdir, 'foo',
+                                                  'baz.manifest'),
+                                     'resource baz baz/')
+        with errors.context('manifest', 2):
+            packager.add('bar/baz.manifest', file)
+
+        with errors.context('manifest', 3):
+            packager.add('qux/qux.manifest',
+                         GeneratedFile(''.join([
+                            'resource qux qux/\n',
+                            'binary-component qux.so\n',
+                         ])))
+        bar_xpt = GeneratedFile('bar.xpt')
+        qux_xpt = GeneratedFile('qux.xpt')
+        foo_html = GeneratedFile('foo_html')
+        bar_html = GeneratedFile('bar_html')
+        with errors.context('manifest', 4):
+            packager.add('foo/bar.xpt', bar_xpt)
+        with errors.context('manifest', 5):
+            packager.add('foo/bar/foo.html', foo_html)
+            packager.add('foo/bar/bar.html', bar_html)
+
+        file = GeneratedFileWithPath(os.path.join(curdir, 'foo.manifest'),
+                                     ''.join([
+                                         'manifest foo/bar.manifest\n',
+                                         'manifest bar/baz.manifest\n',
+                                     ]))
+        with errors.context('manifest', 6):
+            packager.add('foo.manifest', file)
+        with errors.context('manifest', 7):
+            packager.add('foo/qux.xpt', qux_xpt)
+
+        file = GeneratedFileWithPath(os.path.join(curdir, 'addon',
+                                                  'chrome.manifest'),
+                                     'resource hoge hoge/')
+        with errors.context('manifest', 8):
+            packager.add('addon/chrome.manifest', file)
+
+        install_rdf = GeneratedFile('<RDF></RDF>')
+        with errors.context('manifest', 9):
+            packager.add('addon/install.rdf', install_rdf)
+
+        with errors.context('manifest', 10):
+            packager.add('addon2/install.rdf', install_rdf)
+            packager.add('addon2/chrome.manifest',
+                         GeneratedFile('binary-component addon2.so'))
+
+        with errors.context('manifest', 11):
+            packager.add('addon3/install.rdf', install_rdf)
+            packager.add('addon3/chrome.manifest', GeneratedFile(
+                'manifest components/components.manifest'))
+            packager.add('addon3/components/components.manifest',
+                         GeneratedFile('binary-component addon3.so'))
+
+        with errors.context('manifest', 12):
+            install_rdf_addon4 = GeneratedFile(
+                '<RDF>\n<...>\n<em:unpack>true</em:unpack>\n<...>\n</RDF>')
+            packager.add('addon4/install.rdf', install_rdf_addon4)
+
+        with errors.context('manifest', 13):
+            install_rdf_addon5 = GeneratedFile(
+                '<RDF>\n<...>\n<em:unpack>false</em:unpack>\n<...>\n</RDF>')
+            packager.add('addon5/install.rdf', install_rdf_addon5)
+
+        with errors.context('manifest', 14):
+            install_rdf_addon6 = GeneratedFile(
+                '<RDF>\n<... em:unpack=true>\n<...>\n</RDF>')
+            packager.add('addon6/install.rdf', install_rdf_addon6)
+
+        with errors.context('manifest', 15):
+            install_rdf_addon7 = GeneratedFile(
+                '<RDF>\n<... em:unpack=false>\n<...>\n</RDF>')
+            packager.add('addon7/install.rdf', install_rdf_addon7)
+
+        with errors.context('manifest', 16):
+            install_rdf_addon8 = GeneratedFile(
+                '<RDF>\n<... em:unpack="true">\n<...>\n</RDF>')
+            packager.add('addon8/install.rdf', install_rdf_addon8)
+
+        with errors.context('manifest', 17):
+            install_rdf_addon9 = GeneratedFile(
+                '<RDF>\n<... em:unpack="false">\n<...>\n</RDF>')
+            packager.add('addon9/install.rdf', install_rdf_addon9)
+
+        with errors.context('manifest', 18):
+            install_rdf_addon10 = GeneratedFile(
+                '<RDF>\n<... em:unpack=\'true\'>\n<...>\n</RDF>')
+            packager.add('addon10/install.rdf', install_rdf_addon10)
+
+        with errors.context('manifest', 19):
+            install_rdf_addon11 = GeneratedFile(
+                '<RDF>\n<... em:unpack=\'false\'>\n<...>\n</RDF>')
+            packager.add('addon11/install.rdf', install_rdf_addon11)
+
+        self.assertEqual(formatter.log, [])
+
+        with errors.context('dummy', 1):
+            packager.close()
+        self.maxDiff = None
+        # The formatter is expected to reorder the manifest entries so that
+        # chrome entries appear before the others.
+        self.assertEqual(formatter.log, [
+            (('dummy', 1), 'add_base', '', False),
+            (('dummy', 1), 'add_base', 'addon', True),
+            (('dummy', 1), 'add_base', 'addon10', 'unpacked'),
+            (('dummy', 1), 'add_base', 'addon11', True),
+            (('dummy', 1), 'add_base', 'addon2', 'unpacked'),
+            (('dummy', 1), 'add_base', 'addon3', 'unpacked'),
+            (('dummy', 1), 'add_base', 'addon4', 'unpacked'),
+            (('dummy', 1), 'add_base', 'addon5', True),
+            (('dummy', 1), 'add_base', 'addon6', 'unpacked'),
+            (('dummy', 1), 'add_base', 'addon7', True),
+            (('dummy', 1), 'add_base', 'addon8', 'unpacked'),
+            (('dummy', 1), 'add_base', 'addon9', True),
+            (('dummy', 1), 'add_base', 'qux', False),
+            ((os.path.join(curdir, 'foo', 'bar.manifest'), 2),
+             'add_manifest', ManifestContent('foo', 'bar', 'bar/')),
+            ((os.path.join(curdir, 'foo', 'bar.manifest'), 1),
+             'add_manifest', ManifestResource('foo', 'bar', 'bar/')),
+            (('bar/baz.manifest', 1),
+             'add_manifest', ManifestResource('bar', 'baz', 'baz/')),
+            (('qux/qux.manifest', 1),
+             'add_manifest', ManifestResource('qux', 'qux', 'qux/')),
+            (('qux/qux.manifest', 2),
+             'add_manifest', ManifestBinaryComponent('qux', 'qux.so')),
+            (('manifest', 4), 'add_interfaces', 'foo/bar.xpt', bar_xpt),
+            (('manifest', 7), 'add_interfaces', 'foo/qux.xpt', qux_xpt),
+            ((os.path.join(curdir, 'addon', 'chrome.manifest'), 1),
+             'add_manifest', ManifestResource('addon', 'hoge', 'hoge/')),
+            (('addon2/chrome.manifest', 1), 'add_manifest',
+             ManifestBinaryComponent('addon2', 'addon2.so')),
+            (('addon3/components/components.manifest', 1), 'add_manifest',
+             ManifestBinaryComponent('addon3/components', 'addon3.so')),
+            (('manifest', 5), 'add', 'foo/bar/foo.html', foo_html),
+            (('manifest', 5), 'add', 'foo/bar/bar.html', bar_html),
+            (('manifest', 9), 'add', 'addon/install.rdf', install_rdf),
+            (('manifest', 10), 'add', 'addon2/install.rdf', install_rdf),
+            (('manifest', 11), 'add', 'addon3/install.rdf', install_rdf),
+            (('manifest', 12), 'add', 'addon4/install.rdf',
+             install_rdf_addon4),
+            (('manifest', 13), 'add', 'addon5/install.rdf',
+             install_rdf_addon5),
+            (('manifest', 14), 'add', 'addon6/install.rdf',
+             install_rdf_addon6),
+            (('manifest', 15), 'add', 'addon7/install.rdf',
+             install_rdf_addon7),
+            (('manifest', 16), 'add', 'addon8/install.rdf',
+             install_rdf_addon8),
+            (('manifest', 17), 'add', 'addon9/install.rdf',
+             install_rdf_addon9),
+            (('manifest', 18), 'add', 'addon10/install.rdf',
+             install_rdf_addon10),
+            (('manifest', 19), 'add', 'addon11/install.rdf',
+             install_rdf_addon11),
+        ])
+
+        self.assertEqual(packager.get_bases(),
+                         set(['', 'addon', 'addon2', 'addon3', 'addon4',
+                              'addon5', 'addon6', 'addon7', 'addon8',
+                              'addon9', 'addon10', 'addon11', 'qux']))
+        self.assertEqual(packager.get_bases(addons=False), set(['', 'qux']))
+
+    def test_simple_packager_manifest_consistency(self):
+        formatter = MockFormatter()
+        # bar/ is detected as an addon because of install.rdf, but top-level
+        # includes a manifest inside bar/.
+        packager = SimplePackager(formatter)
+        packager.add('base.manifest', GeneratedFile(
+            'manifest foo/bar.manifest\n'
+            'manifest bar/baz.manifest\n'
+        ))
+        packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
+        packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
+        packager.add('bar/install.rdf', GeneratedFile(''))
+
+        with self.assertRaises(ErrorMessage) as e:
+            packager.close()
+
+        self.assertEqual(e.exception.message,
+            'Error: "bar/baz.manifest" is included from "base.manifest", '
+            'which is outside "bar"')
+
+        # bar/ is detected as a separate base because of chrome.manifest that
+        # is included nowhere, but top-level includes another manifest inside
+        # bar/.
+        packager = SimplePackager(formatter)
+        packager.add('base.manifest', GeneratedFile(
+            'manifest foo/bar.manifest\n'
+            'manifest bar/baz.manifest\n'
+        ))
+        packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
+        packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
+        packager.add('bar/chrome.manifest', GeneratedFile('resource baz baz'))
+
+        with self.assertRaises(ErrorMessage) as e:
+            packager.close()
+
+        self.assertEqual(e.exception.message,
+            'Error: "bar/baz.manifest" is included from "base.manifest", '
+            'which is outside "bar"')
+
+        # bar/ is detected as a separate base because of chrome.manifest that
+        # is included nowhere, but chrome.manifest includes baz.manifest from
+        # the same directory. This shouldn't error out.
+        packager = SimplePackager(formatter)
+        packager.add('base.manifest', GeneratedFile(
+            'manifest foo/bar.manifest\n'
+        ))
+        packager.add('foo/bar.manifest', GeneratedFile('resource bar bar'))
+        packager.add('bar/baz.manifest', GeneratedFile('resource baz baz'))
+        packager.add('bar/chrome.manifest',
+                     GeneratedFile('manifest baz.manifest'))
+        packager.close()
+
+
+class TestSimpleManifestSink(unittest.TestCase):
+    def test_simple_manifest_parser(self):
+        formatter = MockFormatter()
+        foobar = GeneratedFile('foobar')
+        foobaz = GeneratedFile('foobaz')
+        fooqux = GeneratedFile('fooqux')
+        foozot = GeneratedFile('foozot')
+        finder = MockFinder({
+            'bin/foo/bar': foobar,
+            'bin/foo/baz': foobaz,
+            'bin/foo/qux': fooqux,
+            'bin/foo/zot': foozot,
+            'bin/foo/chrome.manifest': GeneratedFile('resource foo foo/'),
+            'bin/chrome.manifest':
+            GeneratedFile('manifest foo/chrome.manifest'),
+        })
+        parser = SimpleManifestSink(finder, formatter)
+        component0 = Component('component0')
+        component1 = Component('component1')
+        component2 = Component('component2', destdir='destdir')
+        parser.add(component0, 'bin/foo/b*')
+        parser.add(component1, 'bin/foo/qux')
+        parser.add(component1, 'bin/foo/chrome.manifest')
+        parser.add(component2, 'bin/foo/zot')
+        self.assertRaises(ErrorMessage, parser.add, 'component1', 'bin/bar')
+
+        self.assertEqual(formatter.log, [])
+        parser.close()
+        self.assertEqual(formatter.log, [
+            (None, 'add_base', '', False),
+            (('foo/chrome.manifest', 1),
+             'add_manifest', ManifestResource('foo', 'foo', 'foo/')),
+            (None, 'add', 'foo/bar', foobar),
+            (None, 'add', 'foo/baz', foobaz),
+            (None, 'add', 'foo/qux', fooqux),
+            (None, 'add', 'destdir/foo/zot', foozot),
+        ])
+
+        self.assertEqual(finder.log, [
+            'bin/foo/b*',
+            'bin/foo/qux',
+            'bin/foo/chrome.manifest',
+            'bin/foo/zot',
+            'bin/bar',
+            'bin/chrome.manifest'
+        ])
+
+
+class TestCallDeque(unittest.TestCase):
+    def test_call_deque(self):
+        class Logger(object):
+            def __init__(self):
+                self._log = []
+
+            def log(self, str):
+                self._log.append(str)
+
+            @staticmethod
+            def staticlog(logger, str):
+                logger.log(str)
+
+        def do_log(logger, str):
+            logger.log(str)
+
+        logger = Logger()
+        d = CallDeque()
+        d.append(logger.log, 'foo')
+        d.append(logger.log, 'bar')
+        d.append(logger.staticlog, logger, 'baz')
+        d.append(do_log, logger, 'qux')
+        self.assertEqual(logger._log, [])
+        d.execute()
+        self.assertEqual(logger._log, ['foo', 'bar', 'baz', 'qux'])
+
+
+class TestComponent(unittest.TestCase):
+    def do_split(self, string, name, options):
+        n, o = Component._split_component_and_options(string)
+        self.assertEqual(name, n)
+        self.assertEqual(options, o)
+
+    def test_component_split_component_and_options(self):
+        self.do_split('component', 'component', {})
+        self.do_split('trailingspace ', 'trailingspace', {})
+        self.do_split(' leadingspace', 'leadingspace', {})
+        self.do_split(' trim ', 'trim', {})
+        self.do_split(' trim key="value"', 'trim', {'key':'value'})
+        self.do_split(' trim empty=""', 'trim', {'empty':''})
+        self.do_split(' trim space=" "', 'trim', {'space':' '})
+        self.do_split('component key="value"  key2="second" ',
+                      'component', {'key':'value', 'key2':'second'})
+        self.do_split( 'trim key="  value with spaces   "  key2="spaces again"',
+                       'trim', {'key':'  value with spaces   ', 'key2': 'spaces again'})
+
+    def do_split_error(self, string):
+        self.assertRaises(ValueError, Component._split_component_and_options, string)
+
+    def test_component_split_component_and_options_errors(self):
+        self.do_split_error('"component')
+        self.do_split_error('comp"onent')
+        self.do_split_error('component"')
+        self.do_split_error('"component"')
+        self.do_split_error('=component')
+        self.do_split_error('comp=onent')
+        self.do_split_error('component=')
+        self.do_split_error('key="val"')
+        self.do_split_error('component key=')
+        self.do_split_error('component key="val')
+        self.do_split_error('component key=val"')
+        self.do_split_error('component key="val" x')
+        self.do_split_error('component x key="val"')
+        self.do_split_error('component key1="val" x key2="val"')
+
+    def do_from_string(self, string, name, destdir=''):
+        component = Component.from_string(string)
+        self.assertEqual(name, component.name)
+        self.assertEqual(destdir, component.destdir)
+
+    def test_component_from_string(self):
+        self.do_from_string('component', 'component')
+        self.do_from_string('component-with-hyphen', 'component-with-hyphen')
+        self.do_from_string('component destdir="foo/bar"', 'component', 'foo/bar')
+        self.do_from_string('component destdir="bar spc"', 'component', 'bar spc')
+        self.assertRaises(ErrorMessage, Component.from_string, '')
+        self.assertRaises(ErrorMessage, Component.from_string, 'component novalue=')
+        self.assertRaises(ErrorMessage, Component.from_string, 'component badoption=badvalue')
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_packager_formats.py

@@ -0,0 +1,428 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+import unittest
+from mozpack.packager.formats import (
+    FlatFormatter,
+    JarFormatter,
+    OmniJarFormatter,
+)
+from mozpack.copier import FileRegistry
+from mozpack.files import (
+    GeneratedFile,
+    ManifestFile,
+)
+from mozpack.chrome.manifest import (
+    ManifestContent,
+    ManifestComponent,
+    ManifestResource,
+    ManifestBinaryComponent,
+)
+from mozpack.test.test_files import (
+    MockDest,
+    foo_xpt,
+    foo2_xpt,
+    bar_xpt,
+    read_interfaces,
+)
+import mozpack.path as mozpath
+
+
+CONTENTS = {
+    'bases': {
+        # base_path: is_addon?
+        '': False,
+        'app': False,
+        'addon0': 'unpacked',
+        'addon1': True,
+    },
+    'manifests': [
+        ManifestContent('chrome/f', 'oo', 'oo/'),
+        ManifestContent('chrome/f', 'bar', 'oo/bar/'),
+        ManifestResource('chrome/f', 'foo', 'resource://bar/'),
+        ManifestBinaryComponent('components', 'foo.so'),
+        ManifestContent('app/chrome', 'content', 'foo/'),
+        ManifestComponent('app/components', '{foo-id}', 'foo.js'),
+        ManifestContent('addon0/chrome', 'content', 'foo/bar/'),
+        ManifestContent('addon1/chrome', 'content', 'foo/bar/'),
+    ],
+    'files': {
+        'chrome/f/oo/bar/baz': GeneratedFile('foobarbaz'),
+        'chrome/f/oo/baz': GeneratedFile('foobaz'),
+        'chrome/f/oo/qux': GeneratedFile('fooqux'),
+        'components/foo.so': GeneratedFile('foo.so'),
+        'components/foo.xpt': foo_xpt,
+        'components/bar.xpt': bar_xpt,
+        'foo': GeneratedFile('foo'),
+        'app/chrome/foo/foo': GeneratedFile('appfoo'),
+        'app/components/foo.js': GeneratedFile('foo.js'),
+        'addon0/chrome/foo/bar/baz': GeneratedFile('foobarbaz'),
+        'addon0/components/foo.xpt': foo2_xpt,
+        'addon0/components/bar.xpt': bar_xpt,
+        'addon1/chrome/foo/bar/baz': GeneratedFile('foobarbaz'),
+        'addon1/components/foo.xpt': foo2_xpt,
+        'addon1/components/bar.xpt': bar_xpt,
+    },
+}
+
+FILES = CONTENTS['files']
+
+RESULT_FLAT = {
+    'chrome.manifest': [
+        'manifest chrome/chrome.manifest',
+        'manifest components/components.manifest',
+    ],
+    'chrome/chrome.manifest': [
+        'manifest f/f.manifest',
+    ],
+    'chrome/f/f.manifest': [
+        'content oo oo/',
+        'content bar oo/bar/',
+        'resource foo resource://bar/',
+    ],
+    'chrome/f/oo/bar/baz': FILES['chrome/f/oo/bar/baz'],
+    'chrome/f/oo/baz': FILES['chrome/f/oo/baz'],
+    'chrome/f/oo/qux': FILES['chrome/f/oo/qux'],
+    'components/components.manifest': [
+        'binary-component foo.so',
+        'interfaces interfaces.xpt',
+    ],
+    'components/foo.so': FILES['components/foo.so'],
+    'components/interfaces.xpt': {
+        'foo': read_interfaces(foo_xpt.open())['foo'],
+        'bar': read_interfaces(bar_xpt.open())['bar'],
+    },
+    'foo': FILES['foo'],
+    'app/chrome.manifest': [
+        'manifest chrome/chrome.manifest',
+        'manifest components/components.manifest',
+    ],
+    'app/chrome/chrome.manifest': [
+        'content content foo/',
+    ],
+    'app/chrome/foo/foo': FILES['app/chrome/foo/foo'],
+    'app/components/components.manifest': [
+        'component {foo-id} foo.js',
+    ],
+    'app/components/foo.js': FILES['app/components/foo.js'],
+}
+
+for addon in ('addon0', 'addon1'):
+    RESULT_FLAT.update({
+        mozpath.join(addon, p): f
+        for p, f in {
+            'chrome.manifest': [
+                'manifest chrome/chrome.manifest',
+                'manifest components/components.manifest',
+            ],
+            'chrome/chrome.manifest': [
+                'content content foo/bar/',
+            ],
+            'chrome/foo/bar/baz': FILES[mozpath.join(addon, 'chrome/foo/bar/baz')],
+            'components/components.manifest': [
+                'interfaces interfaces.xpt',
+            ],
+            'components/interfaces.xpt': {
+                'foo': read_interfaces(foo2_xpt.open())['foo'],
+                'bar': read_interfaces(bar_xpt.open())['bar'],
+            },
+        }.iteritems()
+    })
+
+RESULT_JAR = {
+    p: RESULT_FLAT[p]
+    for p in (
+        'chrome.manifest',
+        'chrome/chrome.manifest',
+        'components/components.manifest',
+        'components/foo.so',
+        'components/interfaces.xpt',
+        'foo',
+        'app/chrome.manifest',
+        'app/components/components.manifest',
+        'app/components/foo.js',
+        'addon0/chrome.manifest',
+        'addon0/components/components.manifest',
+        'addon0/components/interfaces.xpt',
+    )
+}
+
+RESULT_JAR.update({
+    'chrome/f/f.manifest': [
+        'content oo jar:oo.jar!/',
+        'content bar jar:oo.jar!/bar/',
+        'resource foo resource://bar/',
+    ],
+    'chrome/f/oo.jar': {
+        'bar/baz': FILES['chrome/f/oo/bar/baz'],
+        'baz': FILES['chrome/f/oo/baz'],
+        'qux': FILES['chrome/f/oo/qux'],
+    },
+    'app/chrome/chrome.manifest': [
+        'content content jar:foo.jar!/',
+    ],
+    'app/chrome/foo.jar': {
+        'foo': FILES['app/chrome/foo/foo'],
+    },
+    'addon0/chrome/chrome.manifest': [
+        'content content jar:foo.jar!/bar/',
+    ],
+    'addon0/chrome/foo.jar': {
+        'bar/baz': FILES['addon0/chrome/foo/bar/baz'],
+    },
+    'addon1.xpi': {
+        mozpath.relpath(p, 'addon1'): f
+        for p, f in RESULT_FLAT.iteritems()
+        if p.startswith('addon1/')
+    },
+})
+
+RESULT_OMNIJAR = {
+    p: RESULT_FLAT[p]
+    for p in (
+        'components/foo.so',
+        'foo',
+    )
+}
+
+RESULT_OMNIJAR.update({
+    p: RESULT_JAR[p]
+    for p in RESULT_JAR
+    if p.startswith('addon')
+})
+
+RESULT_OMNIJAR.update({
+    'omni.foo': {
+        'components/components.manifest': [
+            'interfaces interfaces.xpt',
+        ],
+    },
+    'chrome.manifest': [
+        'manifest components/components.manifest',
+    ],
+    'components/components.manifest': [
+        'binary-component foo.so',
+    ],
+    'app/omni.foo': {
+        p: RESULT_FLAT['app/' + p]
+        for p in (
+            'chrome.manifest',
+            'chrome/chrome.manifest',
+            'chrome/foo/foo',
+            'components/components.manifest',
+            'components/foo.js',
+        )
+    },
+    'app/chrome.manifest': [],
+})
+
+RESULT_OMNIJAR['omni.foo'].update({
+    p: RESULT_FLAT[p]
+    for p in (
+        'chrome.manifest',
+        'chrome/chrome.manifest',
+        'chrome/f/f.manifest',
+        'chrome/f/oo/bar/baz',
+        'chrome/f/oo/baz',
+        'chrome/f/oo/qux',
+        'components/interfaces.xpt',
+    )
+})
+
+CONTENTS_WITH_BASE = {
+    'bases': {
+        mozpath.join('base/root', b) if b else 'base/root': a
+        for b, a in CONTENTS['bases'].iteritems()
+    },
+    'manifests': [
+        m.move(mozpath.join('base/root', m.base))
+        for m in CONTENTS['manifests']
+    ],
+    'files': {
+        mozpath.join('base/root', p): f
+        for p, f in CONTENTS['files'].iteritems()
+    },
+}
+
+EXTRA_CONTENTS = {
+    'extra/file': GeneratedFile('extra file'),
+}
+
+CONTENTS_WITH_BASE['files'].update(EXTRA_CONTENTS)
+
+def result_with_base(results):
+    result = {
+        mozpath.join('base/root', p): v
+        for p, v in results.iteritems()
+    }
+    result.update(EXTRA_CONTENTS)
+    return result
+
+RESULT_FLAT_WITH_BASE = result_with_base(RESULT_FLAT)
+RESULT_JAR_WITH_BASE = result_with_base(RESULT_JAR)
+RESULT_OMNIJAR_WITH_BASE = result_with_base(RESULT_OMNIJAR)
+
+
+class MockDest(MockDest):
+    def exists(self):
+        return False
+
+
+def fill_formatter(formatter, contents):
+    for base, is_addon in contents['bases'].items():
+        formatter.add_base(base, is_addon)
+
+    for manifest in contents['manifests']:
+        formatter.add_manifest(manifest)
+
+    for k, v in contents['files'].iteritems():
+        if k.endswith('.xpt'):
+            formatter.add_interfaces(k, v)
+        else:
+            formatter.add(k, v)
+
+
+def get_contents(registry, read_all=False):
+    result = {}
+    for k, v in registry:
+        if k.endswith('.xpt'):
+            tmpfile = MockDest()
+            registry[k].copy(tmpfile)
+            result[k] = read_interfaces(tmpfile)
+        elif isinstance(v, FileRegistry):
+            result[k] = get_contents(v)
+        elif isinstance(v, ManifestFile) or read_all:
+            result[k] = v.open().read().splitlines()
+        else:
+            result[k] = v
+    return result
+
+
+class TestFormatters(unittest.TestCase):
+    maxDiff = None
+
+    def test_bases(self):
+        formatter = FlatFormatter(FileRegistry())
+        formatter.add_base('')
+        formatter.add_base('browser')
+        formatter.add_base('addon0', addon=True)
+        self.assertEqual(formatter._get_base('platform.ini'),
+                         ('', 'platform.ini'))
+        self.assertEqual(formatter._get_base('browser/application.ini'),
+                         ('browser', 'application.ini'))
+        self.assertEqual(formatter._get_base('addon0/install.rdf'),
+                         ('addon0', 'install.rdf'))
+
+    def do_test_contents(self, formatter, contents):
+        for f in contents['files']:
+            # .xpt files are merged, so skip them.
+            if not f.endswith('.xpt'):
+                self.assertTrue(formatter.contains(f))
+
+    def test_flat_formatter(self):
+        registry = FileRegistry()
+        formatter = FlatFormatter(registry)
+
+        fill_formatter(formatter, CONTENTS)
+        self.assertEqual(get_contents(registry), RESULT_FLAT)
+        self.do_test_contents(formatter, CONTENTS)
+
+    def test_jar_formatter(self):
+        registry = FileRegistry()
+        formatter = JarFormatter(registry)
+
+        fill_formatter(formatter, CONTENTS)
+        self.assertEqual(get_contents(registry), RESULT_JAR)
+        self.do_test_contents(formatter, CONTENTS)
+
+    def test_omnijar_formatter(self):
+        registry = FileRegistry()
+        formatter = OmniJarFormatter(registry, 'omni.foo')
+
+        fill_formatter(formatter, CONTENTS)
+        self.assertEqual(get_contents(registry), RESULT_OMNIJAR)
+        self.do_test_contents(formatter, CONTENTS)
+
+    def test_flat_formatter_with_base(self):
+        registry = FileRegistry()
+        formatter = FlatFormatter(registry)
+
+        fill_formatter(formatter, CONTENTS_WITH_BASE)
+        self.assertEqual(get_contents(registry), RESULT_FLAT_WITH_BASE)
+        self.do_test_contents(formatter, CONTENTS_WITH_BASE)
+
+    def test_jar_formatter_with_base(self):
+        registry = FileRegistry()
+        formatter = JarFormatter(registry)
+
+        fill_formatter(formatter, CONTENTS_WITH_BASE)
+        self.assertEqual(get_contents(registry), RESULT_JAR_WITH_BASE)
+        self.do_test_contents(formatter, CONTENTS_WITH_BASE)
+
+    def test_omnijar_formatter_with_base(self):
+        registry = FileRegistry()
+        formatter = OmniJarFormatter(registry, 'omni.foo')
+
+        fill_formatter(formatter, CONTENTS_WITH_BASE)
+        self.assertEqual(get_contents(registry), RESULT_OMNIJAR_WITH_BASE)
+        self.do_test_contents(formatter, CONTENTS_WITH_BASE)
+
+    def test_omnijar_is_resource(self):
+        def is_resource(base, path):
+            registry = FileRegistry()
+            f = OmniJarFormatter(registry, 'omni.foo', non_resources=[
+                'defaults/messenger/mailViews.dat',
+                'defaults/foo/*',
+                '*/dummy',
+            ])
+            f.add_base('')
+            f.add_base('app')
+            f.add(mozpath.join(base, path), GeneratedFile(''))
+            if f.copier.contains(mozpath.join(base, path)):
+                return False
+            self.assertTrue(f.copier.contains(mozpath.join(base, 'omni.foo')))
+            self.assertTrue(f.copier[mozpath.join(base, 'omni.foo')]
+                            .contains(path))
+            return True
+
+        for base in ['', 'app/']:
+            self.assertTrue(is_resource(base, 'chrome'))
+            self.assertTrue(
+                is_resource(base, 'chrome/foo/bar/baz.properties'))
+            self.assertFalse(is_resource(base, 'chrome/icons/foo.png'))
+            self.assertTrue(is_resource(base, 'components/foo.js'))
+            self.assertFalse(is_resource(base, 'components/foo.so'))
+            self.assertTrue(is_resource(base, 'res/foo.css'))
+            self.assertFalse(is_resource(base, 'res/cursors/foo.png'))
+            self.assertFalse(is_resource(base, 'res/MainMenu.nib/foo'))
+            self.assertTrue(is_resource(base, 'defaults/pref/foo.js'))
+            self.assertFalse(
+                is_resource(base, 'defaults/pref/channel-prefs.js'))
+            self.assertTrue(
+                is_resource(base, 'defaults/preferences/foo.js'))
+            self.assertFalse(
+                is_resource(base, 'defaults/preferences/channel-prefs.js'))
+            self.assertTrue(is_resource(base, 'modules/foo.jsm'))
+            self.assertTrue(is_resource(base, 'greprefs.js'))
+            self.assertTrue(is_resource(base, 'hyphenation/foo'))
+            self.assertTrue(is_resource(base, 'update.locale'))
+            self.assertTrue(
+                is_resource(base, 'jsloader/resource/gre/modules/foo.jsm'))
+            self.assertFalse(is_resource(base, 'foo'))
+            self.assertFalse(is_resource(base, 'foo/bar/greprefs.js'))
+            self.assertTrue(is_resource(base, 'defaults/messenger/foo.dat'))
+            self.assertFalse(
+                is_resource(base, 'defaults/messenger/mailViews.dat'))
+            self.assertTrue(is_resource(base, 'defaults/pref/foo.js'))
+            self.assertFalse(is_resource(base, 'defaults/foo/bar.dat'))
+            self.assertFalse(is_resource(base, 'defaults/foo/bar/baz.dat'))
+            self.assertTrue(is_resource(base, 'chrome/foo/bar/baz/dummy_'))
+            self.assertFalse(is_resource(base, 'chrome/foo/bar/baz/dummy'))
+            self.assertTrue(is_resource(base, 'chrome/foo/bar/dummy_'))
+            self.assertFalse(is_resource(base, 'chrome/foo/bar/dummy'))
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_packager_l10n.py

@@ -0,0 +1,126 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import mozunit
+from test_packager import MockFinder
+from mozpack.packager import l10n
+from mozpack.files import (
+    GeneratedFile,
+    ManifestFile,
+)
+from mozpack.chrome.manifest import (
+    Manifest,
+    ManifestLocale,
+    ManifestContent,
+)
+from mozpack.copier import FileRegistry
+from mozpack.packager.formats import FlatFormatter
+
+
+class TestL10NRepack(unittest.TestCase):
+    def test_l10n_repack(self):
+        foo = GeneratedFile('foo')
+        foobar = GeneratedFile('foobar')
+        qux = GeneratedFile('qux')
+        bar = GeneratedFile('bar')
+        baz = GeneratedFile('baz')
+        dict_aa = GeneratedFile('dict_aa')
+        dict_bb = GeneratedFile('dict_bb')
+        dict_cc = GeneratedFile('dict_cc')
+        barbaz = GeneratedFile('barbaz')
+        lst = GeneratedFile('foo\nbar')
+        app_finder = MockFinder({
+            'bar/foo': foo,
+            'chrome/foo/foobar': foobar,
+            'chrome/qux/qux.properties': qux,
+            'chrome/qux/baz/baz.properties': baz,
+            'chrome/chrome.manifest': ManifestFile('chrome', [
+                ManifestContent('chrome', 'foo', 'foo/'),
+                ManifestLocale('chrome', 'qux', 'en-US', 'qux/'),
+            ]),
+            'chrome.manifest':
+            ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
+            'dict/aa': dict_aa,
+            'app/chrome/bar/barbaz.dtd': barbaz,
+            'app/chrome/chrome.manifest': ManifestFile('app/chrome', [
+                ManifestLocale('app/chrome', 'bar', 'en-US', 'bar/')
+            ]),
+            'app/chrome.manifest':
+            ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
+            'app/dict/bb': dict_bb,
+            'app/dict/cc': dict_cc,
+            'app/chrome/bar/search/foo.xml': foo,
+            'app/chrome/bar/search/bar.xml': bar,
+            'app/chrome/bar/search/lst.txt': lst,
+        })
+        app_finder.jarlogs = {}
+        app_finder.base = 'app'
+        foo_l10n = GeneratedFile('foo_l10n')
+        qux_l10n = GeneratedFile('qux_l10n')
+        baz_l10n = GeneratedFile('baz_l10n')
+        barbaz_l10n = GeneratedFile('barbaz_l10n')
+        lst_l10n = GeneratedFile('foo\nqux')
+        l10n_finder = MockFinder({
+            'chrome/qux-l10n/qux.properties': qux_l10n,
+            'chrome/qux-l10n/baz/baz.properties': baz_l10n,
+            'chrome/chrome.manifest': ManifestFile('chrome', [
+                ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
+            ]),
+            'chrome.manifest':
+            ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
+            'dict/bb': dict_bb,
+            'dict/cc': dict_cc,
+            'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n,
+            'app/chrome/chrome.manifest': ManifestFile('app/chrome', [
+                ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')
+            ]),
+            'app/chrome.manifest':
+            ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
+            'app/dict/aa': dict_aa,
+            'app/chrome/bar-l10n/search/foo.xml': foo_l10n,
+            'app/chrome/bar-l10n/search/qux.xml': qux_l10n,
+            'app/chrome/bar-l10n/search/lst.txt': lst_l10n,
+        })
+        l10n_finder.base = 'l10n'
+        copier = FileRegistry()
+        formatter = FlatFormatter(copier)
+
+        l10n._repack(app_finder, l10n_finder, copier, formatter,
+                     ['dict', 'chrome/**/search/*.xml'])
+        self.maxDiff = None
+
+        repacked = {
+            'bar/foo': foo,
+            'chrome/foo/foobar': foobar,
+            'chrome/qux-l10n/qux.properties': qux_l10n,
+            'chrome/qux-l10n/baz/baz.properties': baz_l10n,
+            'chrome/chrome.manifest': ManifestFile('chrome', [
+                ManifestContent('chrome', 'foo', 'foo/'),
+                ManifestLocale('chrome', 'qux', 'x-test', 'qux-l10n/'),
+            ]),
+            'chrome.manifest':
+            ManifestFile('', [Manifest('', 'chrome/chrome.manifest')]),
+            'dict/bb': dict_bb,
+            'dict/cc': dict_cc,
+            'app/chrome/bar-l10n/barbaz.dtd': barbaz_l10n,
+            'app/chrome/chrome.manifest': ManifestFile('app/chrome', [
+                ManifestLocale('app/chrome', 'bar', 'x-test', 'bar-l10n/')
+            ]),
+            'app/chrome.manifest':
+            ManifestFile('app', [Manifest('app', 'chrome/chrome.manifest')]),
+            'app/dict/aa': dict_aa,
+            'app/chrome/bar-l10n/search/foo.xml': foo_l10n,
+            'app/chrome/bar-l10n/search/qux.xml': qux_l10n,
+            'app/chrome/bar-l10n/search/lst.txt': lst_l10n,
+        }
+
+        self.assertEqual(
+            dict((p, f.open().read()) for p, f in copier),
+            dict((p, f.open().read()) for p, f in repacked.iteritems())
+        )
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_packager_unpack.py

@@ -0,0 +1,65 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozunit
+from mozpack.packager.formats import (
+    FlatFormatter,
+    JarFormatter,
+    OmniJarFormatter,
+)
+from mozpack.packager.unpack import unpack_to_registry
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+)
+from mozpack.test.test_packager_formats import (
+    CONTENTS,
+    fill_formatter,
+    get_contents,
+)
+from mozpack.test.test_files import TestWithTmpDir
+
+
+class TestUnpack(TestWithTmpDir):
+    maxDiff = None
+
+    @staticmethod
+    def _get_copier(cls):
+        copier = FileCopier()
+        formatter = cls(copier)
+        fill_formatter(formatter, CONTENTS)
+        return copier
+
+    @classmethod
+    def setUpClass(cls):
+        cls.contents = get_contents(cls._get_copier(FlatFormatter),
+                                    read_all=True)
+
+    def _unpack_test(self, cls):
+        # Format a package with the given formatter class
+        copier = self._get_copier(cls)
+        copier.copy(self.tmpdir)
+
+        # Unpack that package. Its content is expected to match that of a Flat
+        # formatted package.
+        registry = FileRegistry()
+        unpack_to_registry(self.tmpdir, registry)
+        self.assertEqual(get_contents(registry, read_all=True), self.contents)
+
+    def test_flat_unpack(self):
+        self._unpack_test(FlatFormatter)
+
+    def test_jar_unpack(self):
+        self._unpack_test(JarFormatter)
+
+    def test_omnijar_unpack(self):
+        class OmniFooFormatter(OmniJarFormatter):
+            def __init__(self, registry):
+                super(OmniFooFormatter, self).__init__(registry, 'omni.foo')
+
+        self._unpack_test(OmniFooFormatter)
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_path.py

@@ -0,0 +1,143 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozpack.path import (
+    relpath,
+    join,
+    normpath,
+    dirname,
+    commonprefix,
+    basename,
+    split,
+    splitext,
+    basedir,
+    match,
+    rebase,
+)
+import unittest
+import mozunit
+import os
+
+
+class TestPath(unittest.TestCase):
+    SEP = os.sep
+
+    def test_relpath(self):
+        self.assertEqual(relpath('foo', 'foo'), '')
+        self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar'), '')
+        self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo'), 'bar')
+        self.assertEqual(relpath(self.SEP.join(('foo', 'bar', 'baz')), 'foo'),
+                         'bar/baz')
+        self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/bar/baz'),
+                         '..')
+        self.assertEqual(relpath(self.SEP.join(('foo', 'bar')), 'foo/baz'),
+                         '../bar')
+        self.assertEqual(relpath('foo/', 'foo'), '')
+        self.assertEqual(relpath('foo/bar/', 'foo'), 'bar')
+
+    def test_join(self):
+        self.assertEqual(join('foo', 'bar', 'baz'), 'foo/bar/baz')
+        self.assertEqual(join('foo', '', 'bar'), 'foo/bar')
+        self.assertEqual(join('', 'foo', 'bar'), 'foo/bar')
+        self.assertEqual(join('', 'foo', '/bar'), '/bar')
+
+    def test_normpath(self):
+        self.assertEqual(normpath(self.SEP.join(('foo', 'bar', 'baz',
+                                                 '..', 'qux'))), 'foo/bar/qux')
+
+    def test_dirname(self):
+        self.assertEqual(dirname('foo/bar/baz'), 'foo/bar')
+        self.assertEqual(dirname('foo/bar'), 'foo')
+        self.assertEqual(dirname('foo'), '')
+        self.assertEqual(dirname('foo/bar/'), 'foo/bar')
+
+    def test_commonprefix(self):
+        self.assertEqual(commonprefix([self.SEP.join(('foo', 'bar', 'baz')),
+                                       'foo/qux', 'foo/baz/qux']), 'foo/')
+        self.assertEqual(commonprefix([self.SEP.join(('foo', 'bar', 'baz')),
+                                       'foo/qux', 'baz/qux']), '')
+
+    def test_basename(self):
+        self.assertEqual(basename('foo/bar/baz'), 'baz')
+        self.assertEqual(basename('foo/bar'), 'bar')
+        self.assertEqual(basename('foo'), 'foo')
+        self.assertEqual(basename('foo/bar/'), '')
+
+    def test_split(self):
+        self.assertEqual(split(self.SEP.join(('foo', 'bar', 'baz'))),
+                         ['foo', 'bar', 'baz'])
+
+    def test_splitext(self):
+        self.assertEqual(splitext(self.SEP.join(('foo', 'bar', 'baz.qux'))),
+                         ('foo/bar/baz', '.qux'))
+
+    def test_basedir(self):
+        foobarbaz = self.SEP.join(('foo', 'bar', 'baz'))
+        self.assertEqual(basedir(foobarbaz, ['foo', 'bar', 'baz']), 'foo')
+        self.assertEqual(basedir(foobarbaz, ['foo', 'foo/bar', 'baz']),
+                         'foo/bar')
+        self.assertEqual(basedir(foobarbaz, ['foo/bar', 'foo', 'baz']),
+                         'foo/bar')
+        self.assertEqual(basedir(foobarbaz, ['foo', 'bar', '']), 'foo')
+        self.assertEqual(basedir(foobarbaz, ['bar', 'baz', '']), '')
+
+    def test_match(self):
+        self.assertTrue(match('foo', ''))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/bar'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo'))
+        self.assertTrue(match('foo', '*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/bar/*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/*/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '*/bar/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '*/*/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '*/*/*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/*/*'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/*/*.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/b*/*z.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/b*r/ba*z.qux'))
+        self.assertFalse(match('foo/bar/baz.qux', 'foo/b*z/ba*r.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '**'))
+        self.assertTrue(match('foo/bar/baz.qux', '**/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '**/bar/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/**/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/**/*.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '**/foo/bar/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/**/bar/baz.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/**/bar/*.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/**/*.qux'))
+        self.assertTrue(match('foo/bar/baz.qux', '**/*.qux'))
+        self.assertFalse(match('foo/bar/baz.qux', '**.qux'))
+        self.assertFalse(match('foo/bar', 'foo/*/bar'))
+        self.assertTrue(match('foo/bar/baz.qux', 'foo/**/bar/**'))
+        self.assertFalse(match('foo/nobar/baz.qux', 'foo/**/bar/**'))
+        self.assertTrue(match('foo/bar', 'foo/**/bar/**'))
+
+    def test_rebase(self):
+        self.assertEqual(rebase('foo', 'foo/bar', 'bar/baz'), 'baz')
+        self.assertEqual(rebase('foo', 'foo', 'bar/baz'), 'bar/baz')
+        self.assertEqual(rebase('foo/bar', 'foo', 'baz'), 'bar/baz')
+
+
+if os.altsep:
+    class TestAltPath(TestPath):
+        SEP = os.altsep
+
+    class TestReverseAltPath(TestPath):
+        def setUp(self):
+            sep = os.sep
+            os.sep = os.altsep
+            os.altsep = sep
+
+        def tearDown(self):
+            self.setUp()
+
+    class TestAltReverseAltPath(TestReverseAltPath):
+        SEP = os.altsep
+
+
+if __name__ == '__main__':
+    mozunit.main()

python/mozbuild/mozpack/test/test_unify.py

@@ -0,0 +1,199 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozbuild.util import ensureParentDir
+
+from mozpack.unify import (
+    UnifiedFinder,
+    UnifiedBuildFinder,
+)
+import mozunit
+from mozpack.test.test_files import TestWithTmpDir
+from mozpack.files import FileFinder
+from mozpack.mozjar import JarWriter
+from mozpack.test.test_files import MockDest
+from cStringIO import StringIO
+import os
+import sys
+from mozpack.errors import (
+    ErrorMessage,
+    AccumulatedErrors,
+    errors,
+)
+
+
+class TestUnified(TestWithTmpDir):
+    def create_one(self, which, path, content):
+        file = self.tmppath(os.path.join(which, path))
+        ensureParentDir(file)
+        open(file, 'wb').write(content)
+
+    def create_both(self, path, content):
+        for p in ['a', 'b']:
+            self.create_one(p, path, content)
+
+
+class TestUnifiedFinder(TestUnified):
+    def test_unified_finder(self):
+        self.create_both('foo/bar', 'foobar')
+        self.create_both('foo/baz', 'foobaz')
+        self.create_one('a', 'bar', 'bar')
+        self.create_one('b', 'baz', 'baz')
+        self.create_one('a', 'qux', 'foobar')
+        self.create_one('b', 'qux', 'baz')
+        self.create_one('a', 'test/foo', 'a\nb\nc\n')
+        self.create_one('b', 'test/foo', 'b\nc\na\n')
+        self.create_both('test/bar', 'a\nb\nc\n')
+
+        finder = UnifiedFinder(FileFinder(self.tmppath('a')),
+                               FileFinder(self.tmppath('b')),
+                               sorted=['test'])
+        self.assertEqual(sorted([(f, c.open().read())
+                                 for f, c in finder.find('foo')]),
+                         [('foo/bar', 'foobar'), ('foo/baz', 'foobaz')])
+        self.assertRaises(ErrorMessage, any, finder.find('bar'))
+        self.assertRaises(ErrorMessage, any, finder.find('baz'))
+        self.assertRaises(ErrorMessage, any, finder.find('qux'))
+        self.assertEqual(sorted([(f, c.open().read())
+                                 for f, c in finder.find('test')]),
+                         [('test/bar', 'a\nb\nc\n'),
+                          ('test/foo', 'a\nb\nc\n')])
+
+
+class TestUnifiedBuildFinder(TestUnified):
+    def test_unified_build_finder(self):
+        finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')),
+                                    FileFinder(self.tmppath('b')))
+
+        # Test chrome.manifest unification
+        self.create_both('chrome.manifest', 'a\nb\nc\n')
+        self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n')
+        self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n')
+        self.assertEqual(sorted([(f, c.open().read()) for f, c in
+                                 finder.find('**/chrome.manifest')]),
+                         [('chrome.manifest', 'a\nb\nc\n'),
+                          ('chrome/chrome.manifest', 'a\nb\nc\n')])
+
+        # Test buildconfig.html unification
+        self.create_one('a', 'chrome/browser/foo/buildconfig.html',
+                        '\n'.join([
+                            '<html>',
+                            '<body>',
+                            '<h1>about:buildconfig</h1>',
+                            '<div>foo</div>',
+                            '</body>',
+                            '</html>',
+                        ]))
+        self.create_one('b', 'chrome/browser/foo/buildconfig.html',
+                        '\n'.join([
+                            '<html>',
+                            '<body>',
+                            '<h1>about:buildconfig</h1>',
+                            '<div>bar</div>',
+                            '</body>',
+                            '</html>',
+                        ]))
+        self.assertEqual(sorted([(f, c.open().read()) for f, c in
+                                 finder.find('**/buildconfig.html')]),
+                         [('chrome/browser/foo/buildconfig.html', '\n'.join([
+                             '<html>',
+                             '<body>',
+                             '<h1>about:buildconfig</h1>',
+                             '<div>foo</div>',
+                             '<hr> </hr>',
+                             '<div>bar</div>',
+                             '</body>',
+                             '</html>',
+                         ]))])
+
+        # Test xpi file unification
+        xpi = MockDest()
+        with JarWriter(fileobj=xpi, compress=True) as jar:
+            jar.add('foo', 'foo')
+            jar.add('bar', 'bar')
+        foo_xpi = xpi.read()
+        self.create_both('foo.xpi', foo_xpi)
+
+        with JarWriter(fileobj=xpi, compress=True) as jar:
+            jar.add('foo', 'bar')
+        self.create_one('a', 'bar.xpi', foo_xpi)
+        self.create_one('b', 'bar.xpi', xpi.read())
+
+        errors.out = StringIO()
+        with self.assertRaises(AccumulatedErrors), errors.accumulate():
+            self.assertEqual([(f, c.open().read()) for f, c in
+                              finder.find('*.xpi')],
+                             [('foo.xpi', foo_xpi)])
+        errors.out = sys.stderr
+
+        # Test install.rdf unification
+        x86_64 = 'Darwin_x86_64-gcc3'
+        x86 = 'Darwin_x86-gcc3'
+        target_tag = '<{em}targetPlatform>{platform}</{em}targetPlatform>'
+        target_attr = '{em}targetPlatform="{platform}" '
+
+        rdf_tag = ''.join([
+            '<{RDF}Description {em}bar="bar" {em}qux="qux">',
+              '<{em}foo>foo</{em}foo>',
+              '{targets}',
+              '<{em}baz>baz</{em}baz>',
+            '</{RDF}Description>'
+        ])
+        rdf_attr = ''.join([
+            '<{RDF}Description {em}bar="bar" {attr}{em}qux="qux">',
+              '{targets}',
+              '<{em}foo>foo</{em}foo><{em}baz>baz</{em}baz>',
+            '</{RDF}Description>'
+        ])
+
+        for descr_ns, target_ns in (('RDF:', ''), ('', 'em:'), ('RDF:', 'em:')):
+            # First we need to infuse the above strings with our namespaces and
+            # platform values.
+            ns = { 'RDF': descr_ns, 'em': target_ns }
+            target_tag_x86_64 = target_tag.format(platform=x86_64, **ns)
+            target_tag_x86 = target_tag.format(platform=x86, **ns)
+            target_attr_x86_64 = target_attr.format(platform=x86_64, **ns)
+            target_attr_x86 = target_attr.format(platform=x86, **ns)
+
+            tag_x86_64 = rdf_tag.format(targets=target_tag_x86_64, **ns)
+            tag_x86 = rdf_tag.format(targets=target_tag_x86, **ns)
+            tag_merged = rdf_tag.format(targets=target_tag_x86_64 + target_tag_x86, **ns)
+            tag_empty = rdf_tag.format(targets="", **ns)
+
+            attr_x86_64 = rdf_attr.format(attr=target_attr_x86_64, targets="", **ns)
+            attr_x86 = rdf_attr.format(attr=target_attr_x86, targets="", **ns)
+            attr_merged = rdf_attr.format(attr="", targets=target_tag_x86_64 + target_tag_x86, **ns)
+
+            # This table defines the test cases, columns "a" and "b" being the
+            # contents of the install.rdf of the respective platform and
+            # "result" the exepected merged content after unification.
+            testcases = (
+              #_____a_____  _____b_____  ___result___#
+              (tag_x86_64,  tag_x86,     tag_merged  ),
+              (tag_x86_64,  tag_empty,   tag_empty   ),
+              (tag_empty,   tag_x86,     tag_empty   ),
+              (tag_empty,   tag_empty,   tag_empty   ),
+
+              (attr_x86_64, attr_x86,    attr_merged ),
+              (tag_x86_64,  attr_x86,    tag_merged  ),
+              (attr_x86_64, tag_x86,     attr_merged ),
+
+              (attr_x86_64, tag_empty,   tag_empty   ),
+              (tag_empty,   attr_x86,    tag_empty   )
+            )
+
+            # Now create the files from the above table and compare
+            results = []
+            for emid, (rdf_a, rdf_b, result) in enumerate(testcases):
+                filename = 'ext/id{0}/install.rdf'.format(emid)
+                self.create_one('a', filename, rdf_a)
+                self.create_one('b', filename, rdf_b)
+                results.append((filename, result))
+
+            self.assertEqual(sorted([(f, c.open().read()) for f, c in
+                                     finder.find('**/install.rdf')]), results)
+
+
+if __name__ == '__main__':
+    mozunit.main()