import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

python/mozbuild/mozpack/packager/__init__.py

@@ -0,0 +1,408 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from mozbuild.preprocessor import Preprocessor
+import re
+import os
+from mozpack.errors import errors
+from mozpack.chrome.manifest import (
+    Manifest,
+    ManifestBinaryComponent,
+    ManifestChrome,
+    ManifestInterfaces,
+    is_manifest,
+    parse_manifest,
+)
+import mozpack.path as mozpath
+from collections import deque
+
+
+class Component(object):
+    '''
+    Class that represents a component in a package manifest.
+    '''
+    def __init__(self, name, destdir=''):
+        if name.find(' ') > 0:
+            errors.fatal('Malformed manifest: space in component name "%s"'
+                         % component)
+        self._name = name
+        self._destdir = destdir
+
+    def __repr__(self):
+        s = self.name
+        if self.destdir:
+            s += ' destdir="%s"' % self.destdir
+        return s
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def destdir(self):
+        return self._destdir
+
+    @staticmethod
+    def _triples(lst):
+        '''
+        Split [1, 2, 3, 4, 5, 6, 7] into [(1, 2, 3), (4, 5, 6)].
+        '''
+        return zip(*[iter(lst)] * 3)
+
+    KEY_VALUE_RE = re.compile(r'''
+        \s*                 # optional whitespace.
+        ([a-zA-Z0-9_]+)     # key.
+        \s*=\s*             # optional space around =.
+        "([^"]*)"           # value without surrounding quotes.
+        (?:\s+|$)
+        ''', re.VERBOSE)
+
+    @staticmethod
+    def _split_options(string):
+        '''
+        Split 'key1="value1" key2="value2"' into
+        {'key1':'value1', 'key2':'value2'}.
+
+        Returned keys and values are all strings.
+
+        Throws ValueError if the input is malformed.
+        '''
+        options = {}
+        splits = Component.KEY_VALUE_RE.split(string)
+        if len(splits) % 3 != 1:
+            # This should never happen -- we expect to always split
+            # into ['', ('key', 'val', '')*].
+            raise ValueError("Bad input")
+        if splits[0]:
+            raise ValueError('Unrecognized input ' + splits[0])
+        for key, val, no_match in Component._triples(splits[1:]):
+            if no_match:
+                raise ValueError('Unrecognized input ' + no_match)
+            options[key] = val
+        return options
+
+    @staticmethod
+    def _split_component_and_options(string):
+        '''
+        Split 'name key1="value1" key2="value2"' into
+        ('name', {'key1':'value1', 'key2':'value2'}).
+
+        Returned name, keys and values are all strings.
+
+        Raises ValueError if the input is malformed.
+        '''
+        splits = string.strip().split(None, 1)
+        if not splits:
+            raise ValueError('No component found')
+        component = splits[0].strip()
+        if not component:
+            raise ValueError('No component found')
+        if not re.match('[a-zA-Z0-9_\-]+$', component):
+            raise ValueError('Bad component name ' + component)
+        options = Component._split_options(splits[1]) if len(splits) > 1 else {}
+        return component, options
+
+    @staticmethod
+    def from_string(string):
+        '''
+        Create a component from a string.
+        '''
+        try:
+            name, options = Component._split_component_and_options(string)
+        except ValueError as e:
+            errors.fatal('Malformed manifest: %s' % e)
+            return
+        destdir = options.pop('destdir', '')
+        if options:
+            errors.fatal('Malformed manifest: options %s not recognized'
+                         % options.keys())
+        return Component(name, destdir=destdir)
+
+
+class PackageManifestParser(object):
+    '''
+    Class for parsing of a package manifest, after preprocessing.
+
+    A package manifest is a list of file paths, with some syntaxic sugar:
+        [] designates a toplevel component. Example: [xpcom]
+        - in front of a file specifies it to be removed
+        * wildcard support
+        ** expands to all files and zero or more directories
+        ; file comment
+
+    The parser takes input from the preprocessor line by line, and pushes
+    parsed information to a sink object.
+
+    The add and remove methods of the sink object are called with the
+    current Component instance and a path.
+    '''
+    def __init__(self, sink):
+        '''
+        Initialize the package manifest parser with the given sink.
+        '''
+        self._component = Component('')
+        self._sink = sink
+
+    def handle_line(self, str):
+        '''
+        Handle a line of input and push the parsed information to the sink
+        object.
+        '''
+        # Remove comments.
+        str = str.strip()
+        if not str or str.startswith(';'):
+            return
+        if str.startswith('[') and str.endswith(']'):
+            self._component = Component.from_string(str[1:-1])
+        elif str.startswith('-'):
+            str = str[1:]
+            self._sink.remove(self._component, str)
+        elif ',' in str:
+            errors.fatal('Incompatible syntax')
+        else:
+            self._sink.add(self._component, str)
+
+
+class PreprocessorOutputWrapper(object):
+    '''
+    File-like helper to handle the preprocessor output and send it to a parser.
+    The parser's handle_line method is called in the relevant errors.context.
+    '''
+    def __init__(self, preprocessor, parser):
+        self._parser = parser
+        self._pp = preprocessor
+
+    def write(self, str):
+        file = os.path.normpath(os.path.abspath(self._pp.context['FILE']))
+        with errors.context(file, self._pp.context['LINE']):
+            self._parser.handle_line(str)
+
+
+def preprocess(input, parser, defines={}):
+    '''
+    Preprocess the file-like input with the given defines, and send the
+    preprocessed output line by line to the given parser.
+    '''
+    pp = Preprocessor()
+    pp.context.update(defines)
+    pp.do_filter('substitution')
+    pp.out = PreprocessorOutputWrapper(pp, parser)
+    pp.do_include(input)
+
+
+def preprocess_manifest(sink, manifest, defines={}):
+    '''
+    Preprocess the given file-like manifest with the given defines, and push
+    the parsed information to a sink. See PackageManifestParser documentation
+    for more details on the sink.
+    '''
+    preprocess(manifest, PackageManifestParser(sink), defines)
+
+
+class CallDeque(deque):
+    '''
+    Queue of function calls to make.
+    '''
+    def append(self, function, *args):
+        deque.append(self, (errors.get_context(), function, args))
+
+    def execute(self):
+        while True:
+            try:
+                context, function, args = self.popleft()
+            except IndexError:
+                return
+            if context:
+                with errors.context(context[0], context[1]):
+                    function(*args)
+            else:
+                function(*args)
+
+
+class SimplePackager(object):
+    '''
+    Helper used to translate and buffer instructions from the
+    SimpleManifestSink to a formatter. Formatters expect some information to be
+    given first that the simple manifest contents can't guarantee before the
+    end of the input.
+    '''
+    def __init__(self, formatter):
+        self.formatter = formatter
+        # Queue for formatter.add_interfaces()/add_manifest() calls.
+        self._queue = CallDeque()
+        # Queue for formatter.add_manifest() calls for ManifestChrome.
+        self._chrome_queue = CallDeque()
+        # Queue for formatter.add() calls.
+        self._file_queue = CallDeque()
+        # All paths containing addons. (key is path, value is whether it
+        # should be packed or unpacked)
+        self._addons = {}
+        # All manifest paths imported.
+        self._manifests = set()
+        # All manifest paths included from some other manifest.
+        self._included_manifests = {}
+        self._closed = False
+
+    # Parsing RDF is complex, and would require an external library to do
+    # properly. Just go with some hackish but probably sufficient regexp
+    UNPACK_ADDON_RE = re.compile(r'''(?:
+        <em:unpack>true</em:unpack>
+        |em:unpack=(?P<quote>["']?)true(?P=quote)
+    )''', re.VERBOSE)
+
+    def add(self, path, file):
+        '''
+        Add the given BaseFile instance with the given path.
+        '''
+        assert not self._closed
+        if is_manifest(path):
+            self._add_manifest_file(path, file)
+        elif path.endswith('.xpt'):
+            self._queue.append(self.formatter.add_interfaces, path, file)
+        else:
+            self._file_queue.append(self.formatter.add, path, file)
+            if mozpath.basename(path) == 'install.rdf':
+                addon = True
+                install_rdf = file.open().read()
+                if self.UNPACK_ADDON_RE.search(install_rdf):
+                    addon = 'unpacked'
+                self._addons[mozpath.dirname(path)] = addon
+
+    def _add_manifest_file(self, path, file):
+        '''
+        Add the given BaseFile with manifest file contents with the given path.
+        '''
+        self._manifests.add(path)
+        base = ''
+        if hasattr(file, 'path'):
+            # Find the directory the given path is relative to.
+            b = mozpath.normsep(file.path)
+            if b.endswith('/' + path) or b == path:
+                base = os.path.normpath(b[:-len(path)])
+        for e in parse_manifest(base, path, file.open()):
+            # ManifestResources need to be given after ManifestChrome, so just
+            # put all ManifestChrome in a separate queue to make them first.
+            if isinstance(e, ManifestChrome):
+                # e.move(e.base) just returns a clone of the entry.
+                self._chrome_queue.append(self.formatter.add_manifest,
+                                          e.move(e.base))
+            elif not isinstance(e, (Manifest, ManifestInterfaces)):
+                self._queue.append(self.formatter.add_manifest, e.move(e.base))
+            # If a binary component is added to an addon, prevent the addon
+            # from being packed.
+            if isinstance(e, ManifestBinaryComponent):
+                addon = mozpath.basedir(e.base, self._addons)
+                if addon:
+                    self._addons[addon] = 'unpacked'
+            if isinstance(e, Manifest):
+                if e.flags:
+                    errors.fatal('Flags are not supported on ' +
+                                 '"manifest" entries')
+                self._included_manifests[e.path] = path
+
+    def get_bases(self, addons=True):
+        '''
+        Return all paths under which root manifests have been found. Root
+        manifests are manifests that are included in no other manifest.
+        `addons` indicates whether to include addon bases as well.
+        '''
+        all_bases = set(mozpath.dirname(m)
+                        for m in self._manifests
+                                 - set(self._included_manifests))
+        if not addons:
+            all_bases -= set(self._addons)
+        else:
+            # If for some reason some detected addon doesn't have a
+            # non-included manifest.
+            all_bases |= set(self._addons)
+        return all_bases
+
+    def close(self):
+        '''
+        Push all instructions to the formatter.
+        '''
+        self._closed = True
+
+        bases = self.get_bases()
+        broken_bases = sorted(
+            m for m, includer in self._included_manifests.iteritems()
+            if mozpath.basedir(m, bases) != mozpath.basedir(includer, bases))
+        for m in broken_bases:
+            errors.fatal('"%s" is included from "%s", which is outside "%s"' %
+                         (m, self._included_manifests[m],
+                          mozpath.basedir(m, bases)))
+        for base in sorted(bases):
+            self.formatter.add_base(base, self._addons.get(base, False))
+        self._chrome_queue.execute()
+        self._queue.execute()
+        self._file_queue.execute()
+
+
+class SimpleManifestSink(object):
+    '''
+    Parser sink for "simple" package manifests. Simple package manifests use
+    the format described in the PackageManifestParser documentation, but don't
+    support file removals, and require manifests, interfaces and chrome data to
+    be explicitely listed.
+    Entries starting with bin/ are searched under bin/ in the FileFinder, but
+    are packaged without the bin/ prefix.
+    '''
+    def __init__(self, finder, formatter):
+        '''
+        Initialize the SimpleManifestSink. The given FileFinder is used to
+        get files matching the patterns given in the manifest. The given
+        formatter does the packaging job.
+        '''
+        self._finder = finder
+        self.packager = SimplePackager(formatter)
+        self._closed = False
+        self._manifests = set()
+
+    @staticmethod
+    def normalize_path(path):
+        '''
+        Remove any bin/ prefix.
+        '''
+        if mozpath.basedir(path, ['bin']) == 'bin':
+            return mozpath.relpath(path, 'bin')
+        return path
+
+    def add(self, component, pattern):
+        '''
+        Add files with the given pattern in the given component.
+        '''
+        assert not self._closed
+        added = False
+        for p, f in self._finder.find(pattern):
+            added = True
+            if is_manifest(p):
+                self._manifests.add(p)
+            dest = mozpath.join(component.destdir, SimpleManifestSink.normalize_path(p))
+            self.packager.add(dest, f)
+        if not added:
+            errors.error('Missing file(s): %s' % pattern)
+
+    def remove(self, component, pattern):
+        '''
+        Remove files with the given pattern in the given component.
+        '''
+        assert not self._closed
+        errors.fatal('Removal is unsupported')
+
+    def close(self, auto_root_manifest=True):
+        '''
+        Add possibly missing bits and push all instructions to the formatter.
+        '''
+        if auto_root_manifest:
+            # Simple package manifests don't contain the root manifests, so
+            # find and add them.
+            paths = [mozpath.dirname(m) for m in self._manifests]
+            path = mozpath.dirname(mozpath.commonprefix(paths))
+            for p, f in self._finder.find(mozpath.join(path,
+                                          'chrome.manifest')):
+                if not p in self._manifests:
+                    self.packager.add(SimpleManifestSink.normalize_path(p), f)
+        self.packager.close()

python/mozbuild/mozpack/packager/formats.py

@@ -0,0 +1,324 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+from mozpack.chrome.manifest import (
+    Manifest,
+    ManifestInterfaces,
+    ManifestChrome,
+    ManifestBinaryComponent,
+    ManifestResource,
+)
+from urlparse import urlparse
+import mozpack.path as mozpath
+from mozpack.files import (
+    ManifestFile,
+    XPTFile,
+)
+from mozpack.copier import (
+    FileRegistry,
+    FileRegistrySubtree,
+    Jarrer,
+)
+
+STARTUP_CACHE_PATHS = [
+    'jsloader',
+    'jssubloader',
+]
+
+'''
+Formatters are classes receiving packaging instructions and creating the
+appropriate package layout.
+
+There are three distinct formatters, each handling one of the different chrome
+formats:
+    - flat: essentially, copies files from the source with the same file system
+      layout. Manifests entries are grouped in a single manifest per directory,
+      as well as XPT interfaces.
+    - jar: chrome content is packaged in jar files.
+    - omni: chrome content, modules, non-binary components, and many other
+      elements are packaged in an omnijar file for each base directory.
+
+The base interface provides the following methods:
+    - add_base(path [, addon])
+        Register a base directory for an application or GRE, or an addon.
+        Base directories usually contain a root manifest (manifests not
+        included in any other manifest) named chrome.manifest.
+        The optional addon argument tells whether the base directory
+        is that of a packed addon (True), unpacked addon ('unpacked') or
+        otherwise (False).
+    - add(path, content)
+        Add the given content (BaseFile instance) at the given virtual path
+    - add_interfaces(path, content)
+        Add the given content (BaseFile instance) and link it to other
+        interfaces in the parent directory of the given virtual path.
+    - add_manifest(entry)
+        Add a ManifestEntry.
+    - contains(path)
+        Returns whether the given virtual path is known of the formatter.
+
+The virtual paths mentioned above are paths as they would be with a flat
+chrome.
+
+Formatters all take a FileCopier instance they will fill with the packaged
+data.
+'''
+
+
+class PiecemealFormatter(object):
+    '''
+    Generic formatter that dispatches across different sub-formatters
+    according to paths.
+    '''
+    def __init__(self, copier):
+        assert isinstance(copier, (FileRegistry, FileRegistrySubtree))
+        self.copier = copier
+        self._sub_formatter = {}
+        self._frozen_bases = False
+
+    def add_base(self, base, addon=False):
+        # Only allow to add a base directory before calls to _get_base()
+        assert not self._frozen_bases
+        assert base not in self._sub_formatter
+        self._add_base(base, addon)
+
+    def _get_base(self, path):
+        '''
+        Return the deepest base directory containing the given path.
+        '''
+        self._frozen_bases = True
+        base = mozpath.basedir(path, self._sub_formatter.keys())
+        relpath = mozpath.relpath(path, base) if base else path
+        return base, relpath
+
+    def add(self, path, content):
+        base, relpath = self._get_base(path)
+        if base is None:
+            return self.copier.add(relpath, content)
+        return self._sub_formatter[base].add(relpath, content)
+
+    def add_manifest(self, entry):
+        base, relpath = self._get_base(entry.base)
+        assert base is not None
+        return self._sub_formatter[base].add_manifest(entry.move(relpath))
+
+    def add_interfaces(self, path, content):
+        base, relpath = self._get_base(path)
+        assert base is not None
+        return self._sub_formatter[base].add_interfaces(relpath, content)
+
+    def contains(self, path):
+        assert '*' not in path
+        base, relpath = self._get_base(path)
+        if base is None:
+            return self.copier.contains(relpath)
+        return self._sub_formatter[base].contains(relpath)
+
+
+class FlatFormatter(PiecemealFormatter):
+    '''
+    Formatter for the flat package format.
+    '''
+    def _add_base(self, base, addon=False):
+        self._sub_formatter[base] = FlatSubFormatter(
+            FileRegistrySubtree(base, self.copier))
+
+
+class FlatSubFormatter(object):
+    '''
+    Sub-formatter for the flat package format.
+    '''
+    def __init__(self, copier):
+        assert isinstance(copier, (FileRegistry, FileRegistrySubtree))
+        self.copier = copier
+
+    def add(self, path, content):
+        self.copier.add(path, content)
+
+    def add_manifest(self, entry):
+        # Store manifest entries in a single manifest per directory, named
+        # after their parent directory, except for root manifests, all named
+        # chrome.manifest.
+        if entry.base:
+            name = mozpath.basename(entry.base)
+        else:
+            name = 'chrome'
+        path = mozpath.normpath(mozpath.join(entry.base, '%s.manifest' % name))
+        if not self.copier.contains(path):
+            # Add a reference to the manifest file in the parent manifest, if
+            # the manifest file is not a root manifest.
+            if entry.base:
+                parent = mozpath.dirname(entry.base)
+                relbase = mozpath.basename(entry.base)
+                relpath = mozpath.join(relbase,
+                                            mozpath.basename(path))
+                self.add_manifest(Manifest(parent, relpath))
+            self.copier.add(path, ManifestFile(entry.base))
+        self.copier[path].add(entry)
+
+    def add_interfaces(self, path, content):
+        # Interfaces in the same directory are all linked together in an
+        # interfaces.xpt file.
+        interfaces_path = mozpath.join(mozpath.dirname(path),
+                                       'interfaces.xpt')
+        if not self.copier.contains(interfaces_path):
+            self.add_manifest(ManifestInterfaces(mozpath.dirname(path),
+                                                 'interfaces.xpt'))
+            self.copier.add(interfaces_path, XPTFile())
+        self.copier[interfaces_path].add(content)
+
+    def contains(self, path):
+        assert '*' not in path
+        return self.copier.contains(path)
+
+
+class JarFormatter(PiecemealFormatter):
+    '''
+    Formatter for the jar package format. Assumes manifest entries related to
+    chrome are registered before the chrome data files are added. Also assumes
+    manifest entries for resources are registered after chrome manifest
+    entries.
+    '''
+    def __init__(self, copier, compress=True, optimize=True):
+        PiecemealFormatter.__init__(self, copier)
+        self._compress=compress
+        self._optimize=optimize
+
+    def _add_base(self, base, addon=False):
+        if addon is True:
+            jarrer = Jarrer(self._compress, self._optimize)
+            self.copier.add(base + '.xpi', jarrer)
+            self._sub_formatter[base] = FlatSubFormatter(jarrer)
+        else:
+            self._sub_formatter[base] = JarSubFormatter(
+                FileRegistrySubtree(base, self.copier),
+                self._compress, self._optimize)
+
+
+class JarSubFormatter(PiecemealFormatter):
+    '''
+    Sub-formatter for the jar package format. It is a PiecemealFormatter that
+    dispatches between further sub-formatter for each of the jar files it
+    dispatches the chrome data to, and a FlatSubFormatter for the non-chrome
+    files.
+    '''
+    def __init__(self, copier, compress=True, optimize=True):
+        PiecemealFormatter.__init__(self, copier)
+        self._frozen_chrome = False
+        self._compress = compress
+        self._optimize = optimize
+        self._sub_formatter[''] = FlatSubFormatter(copier)
+
+    def _jarize(self, entry, relpath):
+        '''
+        Transform a manifest entry in one pointing to chrome data in a jar.
+        Return the corresponding chrome path and the new entry.
+        '''
+        base = entry.base
+        basepath = mozpath.split(relpath)[0]
+        chromepath = mozpath.join(base, basepath)
+        entry = entry.rebase(chromepath) \
+            .move(mozpath.join(base, 'jar:%s.jar!' % basepath)) \
+            .rebase(base)
+        return chromepath, entry
+
+    def add_manifest(self, entry):
+        if isinstance(entry, ManifestChrome) and \
+                not urlparse(entry.relpath).scheme:
+            chromepath, entry = self._jarize(entry, entry.relpath)
+            assert not self._frozen_chrome
+            if chromepath not in self._sub_formatter:
+                jarrer = Jarrer(self._compress, self._optimize)
+                self.copier.add(chromepath + '.jar', jarrer)
+                self._sub_formatter[chromepath] = FlatSubFormatter(jarrer)
+        elif isinstance(entry, ManifestResource) and \
+                not urlparse(entry.target).scheme:
+            chromepath, new_entry = self._jarize(entry, entry.target)
+            if chromepath in self._sub_formatter:
+                entry = new_entry
+        PiecemealFormatter.add_manifest(self, entry)
+
+
+class OmniJarFormatter(JarFormatter):
+    '''
+    Formatter for the omnijar package format.
+    '''
+    def __init__(self, copier, omnijar_name, compress=True, optimize=True,
+                 non_resources=()):
+        JarFormatter.__init__(self, copier, compress, optimize)
+        self._omnijar_name = omnijar_name
+        self._non_resources = non_resources
+
+    def _add_base(self, base, addon=False):
+        if addon:
+            JarFormatter._add_base(self, base, addon)
+        else:
+            # Initialize a chrome.manifest next to the omnijar file so that
+            # there's always a chrome.manifest file, even an empty one.
+            path = mozpath.normpath(mozpath.join(base, 'chrome.manifest'))
+            if not self.copier.contains(path):
+                self.copier.add(path, ManifestFile(''))
+            self._sub_formatter[base] = OmniJarSubFormatter(
+                FileRegistrySubtree(base, self.copier), self._omnijar_name,
+                self._compress, self._optimize, self._non_resources)
+
+
+class OmniJarSubFormatter(PiecemealFormatter):
+    '''
+    Sub-formatter for the omnijar package format. It is a PiecemealFormatter
+    that dispatches between a FlatSubFormatter for the resources data and
+    another FlatSubFormatter for the other files.
+    '''
+    def __init__(self, copier, omnijar_name, compress=True, optimize=True,
+                 non_resources=()):
+        PiecemealFormatter.__init__(self, copier)
+        self._omnijar_name = omnijar_name
+        self._compress = compress
+        self._optimize = optimize
+        self._non_resources = non_resources
+        self._sub_formatter[''] = FlatSubFormatter(copier)
+        jarrer = Jarrer(self._compress, self._optimize)
+        self._sub_formatter[omnijar_name] = FlatSubFormatter(jarrer)
+
+    def _get_base(self, path):
+        base = self._omnijar_name if self.is_resource(path) else ''
+        # Only add the omnijar file if something ends up in it.
+        if base and not self.copier.contains(base):
+            self.copier.add(base, self._sub_formatter[base].copier)
+        return base, path
+
+    def add_manifest(self, entry):
+        base = ''
+        if not isinstance(entry, ManifestBinaryComponent):
+            base = self._omnijar_name
+        formatter = self._sub_formatter[base]
+        return formatter.add_manifest(entry)
+
+    def is_resource(self, path):
+        '''
+        Return whether the given path corresponds to a resource to be put in an
+        omnijar archive.
+        '''
+        if any(mozpath.match(path, p.replace('*', '**'))
+               for p in self._non_resources):
+            return False
+        path = mozpath.split(path)
+        if path[0] == 'chrome':
+            return len(path) == 1 or path[1] != 'icons'
+        if path[0] == 'components':
+            return path[-1].endswith(('.js', '.xpt'))
+        if path[0] == 'res':
+            return len(path) == 1 or \
+                (path[1] != 'cursors' and path[1] != 'MainMenu.nib')
+        if path[0] == 'defaults':
+            return len(path) != 3 or \
+                not (path[2] == 'channel-prefs.js' and
+                     path[1] in ['pref', 'preferences'])
+        return path[0] in [
+            'modules',
+            'greprefs.js',
+            'hyphenation',
+            'update.locale',
+        ] or path[0] in STARTUP_CACHE_PATHS

python/mozbuild/mozpack/packager/l10n.py

@@ -0,0 +1,259 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+'''
+Replace localized parts of a packaged directory with data from a langpack
+directory.
+'''
+
+import os
+import mozpack.path as mozpath
+from mozpack.packager.formats import (
+    FlatFormatter,
+    JarFormatter,
+    OmniJarFormatter,
+)
+from mozpack.packager import (
+    Component,
+    SimplePackager,
+    SimpleManifestSink,
+)
+from mozpack.files import (
+    ComposedFinder,
+    ManifestFile,
+)
+from mozpack.copier import (
+    FileCopier,
+    Jarrer,
+)
+from mozpack.chrome.manifest import (
+    ManifestLocale,
+    ManifestEntryWithRelPath,
+    is_manifest,
+    ManifestChrome,
+    Manifest,
+)
+from mozpack.errors import errors
+from mozpack.packager.unpack import UnpackFinder
+from createprecomplete import generate_precomplete
+
+
+class LocaleManifestFinder(object):
+    def __init__(self, finder):
+        entries = self.entries = []
+        bases = self.bases = []
+
+        class MockFormatter(object):
+            def add_interfaces(self, path, content):
+                pass
+
+            def add(self, path, content):
+                pass
+
+            def add_manifest(self, entry):
+                if entry.localized:
+                    entries.append(entry)
+
+            def add_base(self, base, addon=False):
+                bases.append(base)
+
+        # SimplePackager rejects "manifest foo.manifest" entries with
+        # additional flags (such as "manifest foo.manifest application=bar").
+        # Those type of entries are used by language packs to work as addons,
+        # but are not necessary for the purpose of l10n repacking. So we wrap
+        # the finder in order to remove those entries.
+        class WrapFinder(object):
+            def __init__(self, finder):
+                self._finder = finder
+
+            def find(self, pattern):
+                for p, f in self._finder.find(pattern):
+                    if isinstance(f, ManifestFile):
+                        unwanted = [
+                            e for e in f._entries
+                            if isinstance(e, Manifest) and e.flags
+                        ]
+                        if unwanted:
+                            f = ManifestFile(
+                                f._base,
+                                [e for e in f._entries if e not in unwanted])
+                    yield p, f
+
+        sink = SimpleManifestSink(WrapFinder(finder), MockFormatter())
+        sink.add(Component(''), '*')
+        sink.close(False)
+
+        # Find unique locales used in these manifest entries.
+        self.locales = list(set(e.id for e in self.entries
+                                if isinstance(e, ManifestLocale)))
+
+
+def _repack(app_finder, l10n_finder, copier, formatter, non_chrome=set()):
+    app = LocaleManifestFinder(app_finder)
+    l10n = LocaleManifestFinder(l10n_finder)
+
+    # The code further below assumes there's only one locale replaced with
+    # another one.
+    if len(app.locales) > 1:
+        errors.fatal("Multiple app locales aren't supported: " +
+                     ",".join(app.locales))
+    if len(l10n.locales) > 1:
+        errors.fatal("Multiple l10n locales aren't supported: " +
+                     ",".join(l10n.locales))
+    locale = app.locales[0]
+    l10n_locale = l10n.locales[0]
+
+    # For each base directory, store what path a locale chrome package name
+    # corresponds to.
+    # e.g., for the following entry under app/chrome:
+    #     locale foo en-US path/to/files
+    # keep track that the locale path for foo in app is
+    # app/chrome/path/to/files.
+    l10n_paths = {}
+    for e in l10n.entries:
+        if isinstance(e, ManifestChrome):
+            base = mozpath.basedir(e.path, app.bases)
+            l10n_paths.setdefault(base, {})
+            l10n_paths[base][e.name] = e.path
+
+    # For chrome and non chrome files or directories, store what langpack path
+    # corresponds to a package path.
+    paths = {}
+    for e in app.entries:
+        if isinstance(e, ManifestEntryWithRelPath):
+            base = mozpath.basedir(e.path, app.bases)
+            if base not in l10n_paths:
+                errors.fatal("Locale doesn't contain %s/" % base)
+                # Allow errors to accumulate
+                continue
+            if e.name not in l10n_paths[base]:
+                errors.fatal("Locale doesn't have a manifest entry for '%s'" %
+                    e.name)
+                # Allow errors to accumulate
+                continue
+            paths[e.path] = l10n_paths[base][e.name]
+
+    for pattern in non_chrome:
+        for base in app.bases:
+            path = mozpath.join(base, pattern)
+            left = set(p for p, f in app_finder.find(path))
+            right = set(p for p, f in l10n_finder.find(path))
+            for p in right:
+                paths[p] = p
+            for p in left - right:
+                paths[p] = None
+
+    # Create a new package, with non localized bits coming from the original
+    # package, and localized bits coming from the langpack.
+    packager = SimplePackager(formatter)
+    for p, f in app_finder:
+        if is_manifest(p):
+            # Remove localized manifest entries.
+            for e in [e for e in f if e.localized]:
+                f.remove(e)
+        # If the path is one that needs a locale replacement, use the
+        # corresponding file from the langpack.
+        path = None
+        if p in paths:
+            path = paths[p]
+            if not path:
+                continue
+        else:
+            base = mozpath.basedir(p, paths.keys())
+            if base:
+                subpath = mozpath.relpath(p, base)
+                path = mozpath.normpath(mozpath.join(paths[base],
+                                                               subpath))
+        if path:
+            files = [f for p, f in l10n_finder.find(path)]
+            if not len(files):
+                if base not in non_chrome:
+                    finderBase = ""
+                    if hasattr(l10n_finder, 'base'):
+                        finderBase = l10n_finder.base
+                    errors.error("Missing file: %s" %
+                                 os.path.join(finderBase, path))
+            else:
+                packager.add(path, files[0])
+        else:
+            packager.add(p, f)
+
+    # Add localized manifest entries from the langpack.
+    l10n_manifests = []
+    for base in set(e.base for e in l10n.entries):
+        m = ManifestFile(base, [e for e in l10n.entries if e.base == base])
+        path = mozpath.join(base, 'chrome.%s.manifest' % l10n_locale)
+        l10n_manifests.append((path, m))
+    bases = packager.get_bases()
+    for path, m in l10n_manifests:
+        base = mozpath.basedir(path, bases)
+        packager.add(path, m)
+        # Add a "manifest $path" entry in the top manifest under that base.
+        m = ManifestFile(base)
+        m.add(Manifest(base, mozpath.relpath(path, base)))
+        packager.add(mozpath.join(base, 'chrome.manifest'), m)
+
+    packager.close()
+
+    # Add any remaining non chrome files.
+    for pattern in non_chrome:
+        for base in bases:
+            for p, f in l10n_finder.find(mozpath.join(base, pattern)):
+                if not formatter.contains(p):
+                    formatter.add(p, f)
+
+    # Transplant jar preloading information.
+    for path, log in app_finder.jarlogs.iteritems():
+        assert isinstance(copier[path], Jarrer)
+        copier[path].preload([l.replace(locale, l10n_locale) for l in log])
+
+
+def repack(source, l10n, extra_l10n={}, non_resources=[], non_chrome=set()):
+    '''
+    Replace localized data from the `source` directory with localized data
+    from `l10n` and `extra_l10n`.
+
+    The `source` argument points to a directory containing a packaged
+    application (in omnijar, jar or flat form).
+    The `l10n` argument points to a directory containing the main localized
+    data (usually in the form of a language pack addon) to use to replace
+    in the packaged application.
+    The `extra_l10n` argument contains a dict associating relative paths in
+    the source to separate directories containing localized data for them.
+    This can be used to point at different language pack addons for different
+    parts of the package application.
+    The `non_resources` argument gives a list of relative paths in the source
+    that should not be added in an omnijar in case the packaged application
+    is in that format.
+    The `non_chrome` argument gives a list of file/directory patterns for
+    localized files that are not listed in a chrome.manifest.
+    '''
+    app_finder = UnpackFinder(source)
+    l10n_finder = UnpackFinder(l10n)
+    if extra_l10n:
+        finders = {
+            '': l10n_finder,
+        }
+        for base, path in extra_l10n.iteritems():
+            finders[base] = UnpackFinder(path)
+        l10n_finder = ComposedFinder(finders)
+    copier = FileCopier()
+    if app_finder.kind == 'flat':
+        formatter = FlatFormatter(copier)
+    elif app_finder.kind == 'jar':
+        formatter = JarFormatter(copier,
+                                 optimize=app_finder.optimizedjars,
+                                 compress=app_finder.compressed)
+    elif app_finder.kind == 'omni':
+        formatter = OmniJarFormatter(copier, app_finder.omnijar,
+                                     optimize=app_finder.optimizedjars,
+                                     compress=app_finder.compressed,
+                                     non_resources=non_resources)
+
+    with errors.accumulate():
+        _repack(app_finder, l10n_finder, copier, formatter, non_chrome)
+    copier.copy(source, skip_if_older=False)
+    generate_precomplete(source)

python/mozbuild/mozpack/packager/unpack.py

@@ -0,0 +1,202 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import mozpack.path as mozpath
+from mozpack.files import (
+    BaseFinder,
+    FileFinder,
+    DeflatedFile,
+    ManifestFile,
+)
+from mozpack.chrome.manifest import (
+    parse_manifest,
+    ManifestEntryWithRelPath,
+    ManifestResource,
+    is_manifest,
+)
+from mozpack.mozjar import JarReader
+from mozpack.copier import (
+    FileRegistry,
+    FileCopier,
+)
+from mozpack.packager import SimplePackager
+from mozpack.packager.formats import (
+    FlatFormatter,
+    STARTUP_CACHE_PATHS,
+)
+from urlparse import urlparse
+
+
+class UnpackFinder(BaseFinder):
+    '''
+    Special Finder object that treats the source package directory as if it
+    were in the flat chrome format, whatever chrome format it actually is in.
+
+    This means that for example, paths like chrome/browser/content/... match
+    files under jar:chrome/browser.jar!/content/... in case of jar chrome
+    format.
+
+    The only argument to the constructor is a Finder instance or a path.
+    The UnpackFinder is populated with files from this Finder instance,
+    or with files from a FileFinder using the given path as its root.
+    '''
+    def __init__(self, source):
+        if isinstance(source, BaseFinder):
+            self._finder = source
+        else:
+            self._finder = FileFinder(source)
+        self.base = self._finder.base
+        self.files = FileRegistry()
+        self.kind = 'flat'
+        self.omnijar = None
+        self.jarlogs = {}
+        self.optimizedjars = False
+        self.compressed = True
+
+        jars = set()
+
+        for p, f in self._finder.find('*'):
+            # Skip the precomplete file, which is generated at packaging time.
+            if p == 'precomplete':
+                continue
+            base = mozpath.dirname(p)
+            # If the file is a zip/jar that is not a .xpi, and contains a
+            # chrome.manifest, it is an omnijar. All the files it contains
+            # go in the directory containing the omnijar. Manifests are merged
+            # if there is a corresponding manifest in the directory.
+            if not p.endswith('.xpi') and self._maybe_zip(f) and \
+                    (mozpath.basename(p) == self.omnijar or
+                     not self.omnijar):
+                jar = self._open_jar(p, f)
+                if 'chrome.manifest' in jar:
+                    self.kind = 'omni'
+                    self.omnijar = mozpath.basename(p)
+                    self._fill_with_jar(base, jar)
+                    continue
+            # If the file is a manifest, scan its entries for some referencing
+            # jar: urls. If there are some, the files contained in the jar they
+            # point to, go under a directory named after the jar.
+            if is_manifest(p):
+                m = self.files[p] if self.files.contains(p) \
+                    else ManifestFile(base)
+                for e in parse_manifest(self.base, p, f.open()):
+                    m.add(self._handle_manifest_entry(e, jars))
+                if self.files.contains(p):
+                    continue
+                f = m
+            # If the file is a packed addon, unpack it under a directory named
+            # after the xpi.
+            if p.endswith('.xpi') and self._maybe_zip(f):
+                self._fill_with_jar(p[:-4], self._open_jar(p, f))
+                continue
+            if not p in jars:
+                self.files.add(p, f)
+
+    def _fill_with_jar(self, base, jar):
+        for j in jar:
+            path = mozpath.join(base, j.filename)
+            if is_manifest(j.filename):
+                m = self.files[path] if self.files.contains(path) \
+                    else ManifestFile(mozpath.dirname(path))
+                for e in parse_manifest(None, path, j):
+                    m.add(e)
+                if not self.files.contains(path):
+                    self.files.add(path, m)
+                continue
+            else:
+                self.files.add(path, DeflatedFile(j))
+
+    def _handle_manifest_entry(self, entry, jars):
+        jarpath = None
+        if isinstance(entry, ManifestEntryWithRelPath) and \
+                urlparse(entry.relpath).scheme == 'jar':
+            jarpath, entry = self._unjarize(entry, entry.relpath)
+        elif isinstance(entry, ManifestResource) and \
+                urlparse(entry.target).scheme == 'jar':
+            jarpath, entry = self._unjarize(entry, entry.target)
+        if jarpath:
+            # Don't defer unpacking the jar file. If we already saw
+            # it, take (and remove) it from the registry. If we
+            # haven't, try to find it now.
+            if self.files.contains(jarpath):
+                jar = self.files[jarpath]
+                self.files.remove(jarpath)
+            else:
+                jar = [f for p, f in self._finder.find(jarpath)]
+                assert len(jar) == 1
+                jar = jar[0]
+            if not jarpath in jars:
+                base = mozpath.splitext(jarpath)[0]
+                for j in self._open_jar(jarpath, jar):
+                    self.files.add(mozpath.join(base,
+                                                     j.filename),
+                                   DeflatedFile(j))
+            jars.add(jarpath)
+            self.kind = 'jar'
+        return entry
+
+    def _open_jar(self, path, file):
+        '''
+        Return a JarReader for the given BaseFile instance, keeping a log of
+        the preloaded entries it has.
+        '''
+        jar = JarReader(fileobj=file.open())
+        if jar.is_optimized:
+            self.optimizedjars = True
+        if not any(f.compressed for f in jar):
+            self.compressed = False
+        if jar.last_preloaded:
+            jarlog = jar.entries.keys()
+            self.jarlogs[path] = jarlog[:jarlog.index(jar.last_preloaded) + 1]
+        return jar
+
+    def find(self, path):
+        for p in self.files.match(path):
+            yield p, self.files[p]
+
+    def _maybe_zip(self, file):
+        '''
+        Return whether the given BaseFile looks like a ZIP/Jar.
+        '''
+        header = file.open().read(8)
+        return len(header) == 8 and (header[0:2] == 'PK' or
+                                     header[4:6] == 'PK')
+
+    def _unjarize(self, entry, relpath):
+        '''
+        Transform a manifest entry pointing to chrome data in a jar in one
+        pointing to the corresponding unpacked path. Return the jar path and
+        the new entry.
+        '''
+        base = entry.base
+        jar, relpath = urlparse(relpath).path.split('!', 1)
+        entry = entry.rebase(mozpath.join(base, 'jar:%s!' % jar)) \
+            .move(mozpath.join(base, mozpath.splitext(jar)[0])) \
+            .rebase(base)
+        return mozpath.join(base, jar), entry
+
+
+def unpack_to_registry(source, registry):
+    '''
+    Transform a jar chrome or omnijar packaged directory into a flat package.
+
+    The given registry is filled with the flat package.
+    '''
+    finder = UnpackFinder(source)
+    packager = SimplePackager(FlatFormatter(registry))
+    for p, f in finder.find('*'):
+        if mozpath.split(p)[0] not in STARTUP_CACHE_PATHS:
+            packager.add(p, f)
+    packager.close()
+
+
+def unpack(source):
+    '''
+    Transform a jar chrome or omnijar packaged directory into a flat package.
+    '''
+    copier = FileCopier()
+    unpack_to_registry(source, copier)
+    copier.copy(source, skip_if_older=False)