import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

python/mozbuild/mozbuild/action/buildlist.py

@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''A generic script to add entries to a file
+if the entry does not already exist.
+
+Usage: buildlist.py <filename> <entry> [<entry> ...]
+'''
+from __future__ import absolute_import, print_function
+
+import sys
+import os
+
+from mozbuild.util import (
+    ensureParentDir,
+    lock_file,
+)
+
+def addEntriesToListFile(listFile, entries):
+  """Given a file |listFile| containing one entry per line,
+  add each entry in |entries| to the file, unless it is already
+  present."""
+  ensureParentDir(listFile)
+  lock = lock_file(listFile + ".lck")
+  try:
+    if os.path.exists(listFile):
+      f = open(listFile)
+      existing = set(x.strip() for x in f.readlines())
+      f.close()
+    else:
+      existing = set()
+    for e in entries:
+      if e not in existing:
+        existing.add(e)
+    with open(listFile, 'wb') as f:
+      f.write("\n".join(sorted(existing))+"\n")
+  finally:
+    lock = None
+
+
+def main(args):
+    if len(args) < 2:
+        print("Usage: buildlist.py <list file> <entry> [<entry> ...]",
+            file=sys.stderr)
+        return 1
+
+    return addEntriesToListFile(args[0], args[1:])
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/cl.py

@@ -0,0 +1,124 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import ctypes
+import os
+import sys
+
+from mozprocess.processhandler import ProcessHandlerMixin
+from mozbuild.makeutil import Makefile
+
+CL_INCLUDES_PREFIX = os.environ.get("CL_INCLUDES_PREFIX", "Note: including file:")
+
+GetShortPathName = ctypes.windll.kernel32.GetShortPathNameW
+GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
+
+
+# cl.exe likes to print inconsistent paths in the showIncludes output
+# (some lowercased, some not, with different directions of slashes),
+# and we need the original file case for make/pymake to be happy.
+# As this is slow and needs to be called a lot of times, use a cache
+# to speed things up.
+_normcase_cache = {}
+
+def normcase(path):
+    # Get*PathName want paths with backslashes
+    path = path.replace('/', os.sep)
+    dir = os.path.dirname(path)
+    # name is fortunately always going to have the right case,
+    # so we can use a cache for the directory part only.
+    name = os.path.basename(path)
+    if dir in _normcase_cache:
+        result = _normcase_cache[dir]
+    else:
+        path = ctypes.create_unicode_buffer(dir)
+        length = GetShortPathName(path, None, 0)
+        shortpath = ctypes.create_unicode_buffer(length)
+        GetShortPathName(path, shortpath, length)
+        length = GetLongPathName(shortpath, None, 0)
+        if length > len(path):
+            path = ctypes.create_unicode_buffer(length)
+        GetLongPathName(shortpath, path, length)
+        result = _normcase_cache[dir] = path.value
+    return os.path.join(result, name)
+
+
+def InvokeClWithDependencyGeneration(cmdline):
+    target = ""
+    # Figure out what the target is
+    for arg in cmdline:
+        if arg.startswith("-Fo"):
+            target = arg[3:]
+            break
+
+    if target is None:
+        print >>sys.stderr, "No target set"
+        return 1
+
+    # Assume the source file is the last argument
+    source = cmdline[-1]
+    assert not source.startswith('-')
+
+    # The deps target lives here
+    depstarget = os.path.basename(target) + ".pp"
+
+    cmdline += ['-showIncludes']
+
+    mk = Makefile()
+    rule = mk.create_rule([target])
+    rule.add_dependencies([normcase(source)])
+
+    def on_line(line):
+        # cl -showIncludes prefixes every header with "Note: including file:"
+        # and an indentation corresponding to the depth (which we don't need)
+        if line.startswith(CL_INCLUDES_PREFIX):
+            dep = line[len(CL_INCLUDES_PREFIX):].strip()
+            # We can't handle pathes with spaces properly in mddepend.pl, but
+            # we can assume that anything in a path with spaces is a system
+            # header and throw it away.
+            dep = normcase(dep)
+            if ' ' not in dep:
+                rule.add_dependencies([dep])
+        else:
+            # Make sure we preserve the relevant output from cl. mozprocess
+            # swallows the newline delimiter, so we need to re-add it.
+            sys.stdout.write(line)
+            sys.stdout.write('\n')
+
+    # We need to ignore children because MSVC can fire up a background process
+    # during compilation. This process is cleaned up on its own. If we kill it,
+    # we can run into weird compilation issues.
+    p = ProcessHandlerMixin(cmdline, processOutputLine=[on_line],
+        ignore_children=True)
+    p.run()
+    p.processOutput()
+    ret = p.wait()
+
+    if ret != 0 or target == "":
+        # p.wait() returns a long. Somehow sys.exit(long(0)) is like
+        # sys.exit(1). Don't ask why.
+        return int(ret)
+
+    depsdir = os.path.normpath(os.path.join(os.curdir, ".deps"))
+    depstarget = os.path.join(depsdir, depstarget)
+    if not os.path.isdir(depsdir):
+        try:
+            os.makedirs(depsdir)
+        except OSError:
+            pass # This suppresses the error we get when the dir exists, at the
+                 # cost of masking failure to create the directory.  We'll just
+                 # die on the next line though, so it's not that much of a loss.
+
+    with open(depstarget, "w") as f:
+        mk.dump(f)
+
+    return 0
+
+def main(args):
+    return InvokeClWithDependencyGeneration(args)
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/dump_env.py

@@ -0,0 +1,10 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# We invoke a Python program to dump our environment in order to get
+# native paths printed on Windows so that these paths can be incorporated
+# into Python configure's environment.
+import os
+for key, value in os.environ.items():
+    print('%s=%s' % (key, value))

python/mozbuild/mozbuild/action/explode_aar.py

@@ -0,0 +1,72 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import errno
+import os
+import shutil
+import sys
+import zipfile
+
+from mozpack.files import FileFinder
+import mozpack.path as mozpath
+from mozbuild.util import ensureParentDir
+
+def explode(aar, destdir):
+    # Take just the support-v4-22.2.1 part.
+    name, _ = os.path.splitext(os.path.basename(aar))
+
+    destdir = mozpath.join(destdir, name)
+    if os.path.exists(destdir):
+        # We always want to start fresh.
+        shutil.rmtree(destdir)
+    ensureParentDir(destdir)
+    with zipfile.ZipFile(aar) as zf:
+        zf.extractall(destdir)
+
+    # classes.jar is always present.  However, multiple JAR files with the same
+    # name confuses our staged Proguard process in
+    # mobile/android/base/Makefile.in, so we make the names unique here.
+    classes_jar = mozpath.join(destdir, name + '-classes.jar')
+    os.rename(mozpath.join(destdir, 'classes.jar'), classes_jar)
+
+    # Embedded JAR libraries are optional.
+    finder = FileFinder(mozpath.join(destdir, 'libs'), find_executables=False)
+    for p, _ in finder.find('*.jar'):
+        jar = mozpath.join(finder.base, name + '-' + p)
+        os.rename(mozpath.join(finder.base, p), jar)
+
+    # Frequently assets/ is present but empty.  Protect against meaningless
+    # changes to the AAR files by deleting empty assets/ directories.
+    assets = mozpath.join(destdir, 'assets')
+    try:
+        os.rmdir(assets)
+    except OSError, e:
+        if e.errno in (errno.ENOTEMPTY, errno.ENOENT):
+            pass
+        else:
+            raise
+
+    return True
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Explode Android AAR file.')
+
+    parser.add_argument('--destdir', required=True, help='Destination directory.')
+    parser.add_argument('aars', nargs='+', help='Path to AAR file(s).')
+
+    args = parser.parse_args(argv)
+
+    for aar in args.aars:
+        if not explode(aar, args.destdir):
+            return 1
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/file_generate.py

@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Given a Python script and arguments describing the output file, and
+# the arguments that can be used to generate the output file, call the
+# script's |main| method with appropriate arguments.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import imp
+import os
+import sys
+import traceback
+
+from mozbuild.pythonutil import iter_modules_in_path
+from mozbuild.makeutil import Makefile
+from mozbuild.util import FileAvoidWrite
+import buildconfig
+
+
+def main(argv):
+    parser = argparse.ArgumentParser('Generate a file from a Python script',
+                                     add_help=False)
+    parser.add_argument('python_script', metavar='python-script', type=str,
+                        help='The Python script to run')
+    parser.add_argument('method_name', metavar='method-name', type=str,
+                        help='The method of the script to invoke')
+    parser.add_argument('output_file', metavar='output-file', type=str,
+                        help='The file to generate')
+    parser.add_argument('dep_file', metavar='dep-file', type=str,
+                        help='File to write any additional make dependencies to')
+    parser.add_argument('additional_arguments', metavar='arg',
+                        nargs=argparse.REMAINDER,
+                        help="Additional arguments to the script's main() method")
+
+    args = parser.parse_args(argv)
+
+    script = args.python_script
+    # Permit the script to import modules from the same directory in which it
+    # resides.  The justification for doing this is that if we were invoking
+    # the script as:
+    #
+    #    python script arg1...
+    #
+    # then importing modules from the script's directory would come for free.
+    # Since we're invoking the script in a roundabout way, we provide this
+    # bit of convenience.
+    sys.path.append(os.path.dirname(script))
+    with open(script, 'r') as fh:
+        module = imp.load_module('script', fh, script,
+                                 ('.py', 'r', imp.PY_SOURCE))
+    method = args.method_name
+    if not hasattr(module, method):
+        print('Error: script "{0}" is missing a {1} method'.format(script, method),
+              file=sys.stderr)
+        return 1
+
+    ret = 1
+    try:
+        with FileAvoidWrite(args.output_file) as output:
+            ret = module.__dict__[method](output, *args.additional_arguments)
+            # The following values indicate a statement of success:
+            #  - a set() (see below)
+            #  - 0
+            #  - False
+            #  - None
+            #
+            # Everything else is an error (so scripts can conveniently |return
+            # 1| or similar). If a set is returned, the elements of the set
+            # indicate additional dependencies that will be listed in the deps
+            # file. Python module imports are automatically included as
+            # dependencies.
+            if isinstance(ret, set):
+                deps = ret
+                # The script succeeded, so reset |ret| to indicate that.
+                ret = None
+            else:
+                deps = set()
+
+            # Only write out the dependencies if the script was successful
+            if not ret:
+                # Add dependencies on any python modules that were imported by
+                # the script.
+                deps |= set(iter_modules_in_path(buildconfig.topsrcdir,
+                                                 buildconfig.topobjdir))
+                mk = Makefile()
+                mk.create_rule([args.output_file]).add_dependencies(deps)
+                with FileAvoidWrite(args.dep_file) as dep_file:
+                    mk.dump(dep_file)
+        # Even when our file's contents haven't changed, we want to update
+        # the file's mtime so make knows this target isn't still older than
+        # whatever prerequisite caused it to be built this time around.
+        try:
+            os.utime(args.output_file, None)
+        except:
+            print('Error processing file "{0}"'.format(args.output_file),
+                  file=sys.stderr)
+            traceback.print_exc()
+    except IOError as e:
+        print('Error opening file "{0}"'.format(e.filename), file=sys.stderr)
+        traceback.print_exc()
+        return 1
+    return ret
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/generate_browsersearch.py

@@ -0,0 +1,131 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Script to generate the browsersearch.json file for Fennec.
+
+This script follows these steps:
+
+1. Read the region.properties file in all the given source directories (see
+srcdir option). Merge all properties into a single dict accounting for the
+priority of source directories.
+
+2. Read the default search plugin from 'browser.search.defaultenginename'.
+
+3. Read the list of search plugins from the 'browser.search.order.INDEX'
+properties with values identifying particular search plugins by name.
+
+4. Read each region-specific default search plugin from each property named like
+'browser.search.defaultenginename.REGION'.
+
+5. Read the list of region-specific search plugins from the
+'browser.search.order.REGION.INDEX' properties with values identifying
+particular search plugins by name. Here, REGION is derived from a REGION for
+which we have seen a region-specific default plugin.
+
+6. Generate a JSON representation of the above information, and write the result
+to browsersearch.json in the locale-specific raw resource directory
+e.g. raw/browsersearch.json, raw-pt-rBR/browsersearch.json.
+'''
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+)
+
+import argparse
+import codecs
+import json
+import sys
+import os
+
+from mozbuild.dotproperties import (
+    DotProperties,
+)
+from mozbuild.util import (
+    FileAvoidWrite,
+)
+import mozpack.path as mozpath
+
+
+def merge_properties(filename, srcdirs):
+    """Merges properties from the given file in the given source directories."""
+    properties = DotProperties()
+    for srcdir in srcdirs:
+        path = mozpath.join(srcdir, filename)
+        try:
+            properties.update(path)
+        except IOError:
+            # Ignore non-existing files
+            continue
+    return properties
+
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--verbose', '-v', default=False, action='store_true',
+                        help='be verbose')
+    parser.add_argument('--silent', '-s', default=False, action='store_true',
+                        help='be silent')
+    parser.add_argument('--srcdir', metavar='SRCDIR',
+                        action='append', required=True,
+                        help='directories to read inputs from, in order of priority')
+    parser.add_argument('output', metavar='OUTPUT',
+                        help='output')
+    opts = parser.parse_args(args)
+
+    # Use reversed order so that the first srcdir has higher priority to override keys.
+    properties = merge_properties('region.properties', reversed(opts.srcdir))
+
+    # Default, not region-specific.
+    default = properties.get('browser.search.defaultenginename')
+    engines = properties.get_list('browser.search.order')
+
+    writer = codecs.getwriter('utf-8')(sys.stdout)
+    if opts.verbose:
+        print('Read {len} engines: {engines}'.format(len=len(engines), engines=engines), file=writer)
+        print("Default engine is '{default}'.".format(default=default), file=writer)
+
+    browsersearch = {}
+    browsersearch['default'] = default
+    browsersearch['engines'] = engines
+
+    # This gets defaults, yes; but it also gets the list of regions known.
+    regions = properties.get_dict('browser.search.defaultenginename')
+
+    browsersearch['regions'] = {}
+    for region in regions.keys():
+        region_default = regions[region]
+        region_engines = properties.get_list('browser.search.order.{region}'.format(region=region))
+
+        if opts.verbose:
+            print("Region '{region}': Read {len} engines: {region_engines}".format(
+                len=len(region_engines), region=region, region_engines=region_engines), file=writer)
+            print("Region '{region}': Default engine is '{region_default}'.".format(
+                region=region, region_default=region_default), file=writer)
+
+        browsersearch['regions'][region] = {
+            'default': region_default,
+            'engines': region_engines,
+        }
+
+    # FileAvoidWrite creates its parent directories.
+    output = os.path.abspath(opts.output)
+    fh = FileAvoidWrite(output)
+    json.dump(browsersearch, fh)
+    existed, updated = fh.close()
+
+    if not opts.silent:
+        if updated:
+            print('{output} updated'.format(output=output))
+        else:
+            print('{output} already up-to-date'.format(output=output))
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/generate_searchjson.py

@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import json
+
+engines = []
+
+locale = sys.argv[2]
+output_file = sys.argv[3]
+
+output = open(output_file, 'w')
+
+with open(sys.argv[1]) as f:
+  searchinfo = json.load(f)
+
+if locale in searchinfo["locales"]:
+  output.write(json.dumps(searchinfo["locales"][locale]))
+else:
+  output.write(json.dumps(searchinfo["default"]))
+
+output.close();

python/mozbuild/mozbuild/action/generate_suggestedsites.py

@@ -0,0 +1,147 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+''' Script to generate the suggestedsites.json file for Fennec.
+
+This script follows these steps:
+
+1. Read the region.properties file in all the given source directories
+(see srcdir option). Merge all properties into a single dict accounting for
+the priority of source directories.
+
+2. Read the list of sites from the list 'browser.suggestedsites.list.INDEX' and
+'browser.suggestedsites.restricted.list.INDEX' properties with value of these keys
+being an identifier for each suggested site e.g. browser.suggestedsites.list.0=mozilla,
+browser.suggestedsites.list.1=fxmarketplace.
+
+3. For each site identifier defined by the list keys, look for matching branches
+containing the respective properties i.e. url, title, etc. For example,
+for a 'mozilla' identifier, we'll look for keys like:
+browser.suggestedsites.mozilla.url, browser.suggestedsites.mozilla.title, etc.
+
+4. Generate a JSON representation of each site, join them in a JSON array, and
+write the result to suggestedsites.json on the locale-specific raw resource
+directory e.g. raw/suggestedsites.json, raw-pt-rBR/suggestedsites.json.
+'''
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import copy
+import json
+import sys
+import os
+
+from mozbuild.dotproperties import (
+    DotProperties,
+)
+from mozbuild.util import (
+    FileAvoidWrite,
+)
+from mozpack.files import (
+    FileFinder,
+)
+import mozpack.path as mozpath
+
+
+def merge_properties(filename, srcdirs):
+    """Merges properties from the given file in the given source directories."""
+    properties = DotProperties()
+    for srcdir in srcdirs:
+        path = mozpath.join(srcdir, filename)
+        try:
+            properties.update(path)
+        except IOError:
+            # Ignore non-existing files
+            continue
+    return properties
+
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--verbose', '-v', default=False, action='store_true',
+                        help='be verbose')
+    parser.add_argument('--silent', '-s', default=False, action='store_true',
+                        help='be silent')
+    parser.add_argument('--android-package-name', metavar='NAME',
+                        required=True,
+                        help='Android package name')
+    parser.add_argument('--resources', metavar='RESOURCES',
+                        default=None,
+                        help='optional Android resource directory to find drawables in')
+    parser.add_argument('--srcdir', metavar='SRCDIR',
+                        action='append', required=True,
+                        help='directories to read inputs from, in order of priority')
+    parser.add_argument('output', metavar='OUTPUT',
+                        help='output')
+    opts = parser.parse_args(args)
+
+    # Use reversed order so that the first srcdir has higher priority to override keys.
+    properties = merge_properties('region.properties', reversed(opts.srcdir))
+
+    # Keep these two in sync.
+    image_url_template = 'android.resource://%s/drawable/suggestedsites_{name}' % opts.android_package_name
+    drawables_template = 'drawable*/suggestedsites_{name}.*'
+
+    # Load properties corresponding to each site name and define their
+    # respective image URL.
+    sites = []
+
+    def add_names(names, defaults={}):
+        for name in names:
+            site = copy.deepcopy(defaults)
+            site.update(properties.get_dict('browser.suggestedsites.{name}'.format(name=name), required_keys=('title', 'url', 'bgcolor')))
+            site['imageurl'] = image_url_template.format(name=name)
+            sites.append(site)
+
+            # Now check for existence of an appropriately named drawable.  If none
+            # exists, throw.  This stops a locale discovering, at runtime, that the
+            # corresponding drawable was not added to en-US.
+            if not opts.resources:
+                continue
+            resources = os.path.abspath(opts.resources)
+            finder = FileFinder(resources)
+            matches = [p for p, _ in finder.find(drawables_template.format(name=name))]
+            if not matches:
+                raise Exception("Could not find drawable in '{resources}' for '{name}'"
+                    .format(resources=resources, name=name))
+            else:
+                if opts.verbose:
+                    print("Found {len} drawables in '{resources}' for '{name}': {matches}"
+                          .format(len=len(matches), resources=resources, name=name, matches=matches))
+
+    # We want the lists to be ordered for reproducibility.  Each list has a
+    # "default" JSON list item which will be extended by the properties read.
+    lists = [
+        ('browser.suggestedsites.list', {}),
+        ('browser.suggestedsites.restricted.list', {'restricted': True}),
+    ]
+    if opts.verbose:
+        print('Reading {len} suggested site lists: {lists}'.format(len=len(lists), lists=[list_name for list_name, _ in lists]))
+
+    for (list_name, list_item_defaults) in lists:
+        names = properties.get_list(list_name)
+        if opts.verbose:
+            print('Reading {len} suggested sites from {list}: {names}'.format(len=len(names), list=list_name, names=names))
+        add_names(names, list_item_defaults)
+
+
+    # FileAvoidWrite creates its parent directories.
+    output = os.path.abspath(opts.output)
+    fh = FileAvoidWrite(output)
+    json.dump(sites, fh)
+    existed, updated = fh.close()
+
+    if not opts.silent:
+        if updated:
+            print('{output} updated'.format(output=output))
+        else:
+            print('{output} already up-to-date'.format(output=output))
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/generate_symbols_file.py

@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import buildconfig
+import os
+from StringIO import StringIO
+from mozbuild.preprocessor import Preprocessor
+from mozbuild.util import DefinesAction
+
+
+def generate_symbols_file(output, *args):
+    ''' '''
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input')
+    parser.add_argument('-D', action=DefinesAction)
+    parser.add_argument('-U', action='append', default=[])
+    args = parser.parse_args(args)
+    input = os.path.abspath(args.input)
+
+    pp = Preprocessor()
+    pp.context.update(buildconfig.defines)
+    if args.D:
+        pp.context.update(args.D)
+    for undefine in args.U:
+        if undefine in pp.context:
+            del pp.context[undefine]
+    # Hack until MOZ_DEBUG_FLAGS are simply part of buildconfig.defines
+    if buildconfig.substs['MOZ_DEBUG']:
+        pp.context['DEBUG'] = '1'
+    # Ensure @DATA@ works as expected (see the Windows section further below)
+    if buildconfig.substs['OS_TARGET'] == 'WINNT':
+        pp.context['DATA'] = 'DATA'
+    else:
+        pp.context['DATA'] = ''
+    pp.out = StringIO()
+    pp.do_filter('substitution')
+    pp.do_include(input)
+
+    symbols = [s.strip() for s in pp.out.getvalue().splitlines() if s.strip()]
+
+    if buildconfig.substs['OS_TARGET'] == 'WINNT':
+        # A def file is generated for MSVC link.exe that looks like the
+        # following:
+        # LIBRARY library.dll
+        # EXPORTS
+        #   symbol1
+        #   symbol2
+        #   ...
+        #
+        # link.exe however requires special markers for data symbols, so in
+        # that case the symbols look like:
+        #   data_symbol1 DATA
+        #   data_symbol2 DATA
+        #   ...
+        #
+        # In the input file, this is just annotated with the following syntax:
+        #   data_symbol1 @DATA@
+        #   data_symbol2 @DATA@
+        #   ...
+        # The DATA variable is "simply" expanded by the preprocessor, to
+        # nothing on non-Windows, such that we only get the symbol name on
+        # those platforms, and to DATA on Windows, so that the "DATA" part
+        # is, in fact, part of the symbol name as far as the symbols variable
+        # is concerned.
+        libname, ext = os.path.splitext(os.path.basename(output.name))
+        assert ext == '.def'
+        output.write('LIBRARY %s\nEXPORTS\n  %s\n'
+                     % (libname, '\n  '.join(symbols)))
+    elif buildconfig.substs['GCC_USE_GNU_LD']:
+        # A linker version script is generated for GNU LD that looks like the
+        # following:
+        # {
+        # global:
+        #   symbol1;
+        #   symbol2;
+        #   ...
+        # local:
+        #   *;
+        # };
+        output.write('{\nglobal:\n  %s;\nlocal:\n  *;\n};'
+                     % ';\n  '.join(symbols))
+    elif buildconfig.substs['OS_TARGET'] == 'Darwin':
+        # A list of symbols is generated for Apple ld that simply lists all
+        # symbols, with an underscore prefix.
+        output.write(''.join('_%s\n' % s for s in symbols))
+
+    return set(pp.includes)

python/mozbuild/mozbuild/action/jar_maker.py

@@ -0,0 +1,17 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import sys
+
+import mozbuild.jar
+
+
+def main(args):
+    return mozbuild.jar.main(args)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/make_dmg.py

@@ -0,0 +1,37 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function
+
+from mozbuild.base import MozbuildObject
+from mozpack import dmg
+
+import os
+import sys
+
+
+def make_dmg(source_directory, output_dmg):
+    build = MozbuildObject.from_environment()
+    extra_files = [
+        (os.path.join(build.distdir, 'branding', 'dsstore'), '.DS_Store'),
+        (os.path.join(build.distdir, 'branding', 'background.png'),
+         '.background/background.png'),
+        (os.path.join(build.distdir, 'branding', 'disk.icns'),
+         '.VolumeIcon.icns'),
+    ]
+    volume_name = build.substs['MOZ_APP_DISPLAYNAME']
+    dmg.create_dmg(source_directory, output_dmg, volume_name, extra_files)
+
+
+def main(args):
+    if len(args) != 2:
+        print('Usage: make_dmg.py <source directory> <output dmg>',
+              file=sys.stderr)
+        return 1
+    make_dmg(args[0], args[1])
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/output_searchplugins_list.py

@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import json
+
+engines = []
+
+locale = sys.argv[2]
+
+with open(sys.argv[1]) as f:
+  searchinfo = json.load(f)
+
+if locale in searchinfo["locales"]:
+  for region in searchinfo["locales"][locale]:
+    engines = list(set(engines)|set(searchinfo["locales"][locale][region]["visibleDefaultEngines"]))
+else:
+  engines = searchinfo["default"]["visibleDefaultEngines"]
+
+print '\n'.join(engines)

python/mozbuild/mozbuild/action/package_fennec_apk.py

@@ -0,0 +1,150 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''
+Script to produce an Android package (.apk) for Fennec.
+'''
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import buildconfig
+import os
+import subprocess
+import sys
+
+from mozpack.copier import Jarrer
+from mozpack.files import (
+    DeflatedFile,
+    File,
+    FileFinder,
+)
+from mozpack.mozjar import JarReader
+import mozpack.path as mozpath
+
+
+def package_fennec_apk(inputs=[], omni_ja=None, classes_dex=None,
+                       lib_dirs=[],
+                       assets_dirs=[],
+                       features_dirs=[],
+                       root_files=[],
+                       verbose=False):
+    jarrer = Jarrer(optimize=False)
+
+    # First, take input files.  The contents of the later files overwrites the
+    # content of earlier files.
+    for input in inputs:
+        jar = JarReader(input)
+        for file in jar:
+            path = file.filename
+            if jarrer.contains(path):
+                jarrer.remove(path)
+            jarrer.add(path, DeflatedFile(file), compress=file.compressed)
+
+    def add(path, file, compress=None):
+        abspath = os.path.abspath(file.path)
+        if verbose:
+            print('Packaging %s from %s' % (path, file.path))
+        if not os.path.exists(abspath):
+            raise ValueError('File %s not found (looked for %s)' % \
+                             (file.path, abspath))
+        if jarrer.contains(path):
+            jarrer.remove(path)
+        jarrer.add(path, file, compress=compress)
+
+    for features_dir in features_dirs:
+        finder = FileFinder(features_dir, find_executables=False)
+        for p, f in finder.find('**'):
+            add(mozpath.join('assets', 'features', p), f, False)
+
+    for assets_dir in assets_dirs:
+        finder = FileFinder(assets_dir, find_executables=False)
+        for p, f in finder.find('**'):
+            compress = None  # Take default from Jarrer.
+            if p.endswith('.so'):
+                # Asset libraries are special.
+                if f.open().read(5)[1:] == '7zXZ':
+                    print('%s is already compressed' % p)
+                    # We need to store (rather than deflate) compressed libraries
+                    # (even if we don't compress them ourselves).
+                    compress = False
+                elif buildconfig.substs.get('XZ'):
+                    cmd = [buildconfig.substs.get('XZ'), '-zkf',
+                           mozpath.join(finder.base, p)]
+
+                    bcj = None
+                    if buildconfig.substs.get('MOZ_THUMB2'):
+                        bcj = '--armthumb'
+                    elif buildconfig.substs.get('CPU_ARCH') == 'arm':
+                        bcj = '--arm'
+                    elif buildconfig.substs.get('CPU_ARCH') == 'x86':
+                        bcj = '--x86'
+
+                    if bcj:
+                        cmd.extend([bcj, '--lzma2'])
+                    print('xz-compressing %s with %s' % (p, ' '.join(cmd)))
+                    subprocess.check_output(cmd)
+                    os.rename(f.path + '.xz', f.path)
+                    compress = False
+
+            add(mozpath.join('assets', p), f, compress=compress)
+
+    for lib_dir in lib_dirs:
+        finder = FileFinder(lib_dir, find_executables=False)
+        for p, f in finder.find('**'):
+            add(mozpath.join('lib', p), f)
+
+    for root_file in root_files:
+        add(os.path.basename(root_file), File(root_file))
+
+    if omni_ja:
+        add(mozpath.join('assets', 'omni.ja'), File(omni_ja), compress=False)
+
+    if classes_dex:
+        add('classes.dex', File(classes_dex))
+
+    return jarrer
+
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--verbose', '-v', default=False, action='store_true',
+                        help='be verbose')
+    parser.add_argument('--inputs', nargs='+',
+                        help='Input skeleton AP_ or APK file(s).')
+    parser.add_argument('-o', '--output',
+                        help='Output APK file.')
+    parser.add_argument('--omnijar', default=None,
+                        help='Optional omni.ja to pack into APK file.')
+    parser.add_argument('--classes-dex', default=None,
+                        help='Optional classes.dex to pack into APK file.')
+    parser.add_argument('--lib-dirs', nargs='*', default=[],
+                        help='Optional lib/ dirs to pack into APK file.')
+    parser.add_argument('--assets-dirs', nargs='*', default=[],
+                        help='Optional assets/ dirs to pack into APK file.')
+    parser.add_argument('--features-dirs', nargs='*', default=[],
+                        help='Optional features/ dirs to pack into APK file.')
+    parser.add_argument('--root-files', nargs='*', default=[],
+                        help='Optional files to pack into APK file root.')
+    args = parser.parse_args(args)
+
+    if buildconfig.substs.get('OMNIJAR_NAME') != 'assets/omni.ja':
+        raise ValueError("Don't know how package Fennec APKs when "
+                         " OMNIJAR_NAME is not 'assets/omni.jar'.")
+
+    jarrer = package_fennec_apk(inputs=args.inputs,
+                                omni_ja=args.omnijar,
+                                classes_dex=args.classes_dex,
+                                lib_dirs=args.lib_dirs,
+                                assets_dirs=args.assets_dirs,
+                                features_dirs=args.features_dirs,
+                                root_files=args.root_files,
+                                verbose=args.verbose)
+    jarrer.copy(args.output)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/preprocessor.py

@@ -0,0 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import sys
+
+from mozbuild.preprocessor import Preprocessor
+
+
+def main(args):
+  pp = Preprocessor()
+  pp.handleCommandLine(args, True)
+
+
+if __name__ == "__main__":
+  main(sys.argv[1:])

python/mozbuild/mozbuild/action/process_define_files.py

@@ -0,0 +1,94 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import os
+import re
+import sys
+from buildconfig import topobjdir
+from mozbuild.backend.configenvironment import ConfigEnvironment
+from mozbuild.util import FileAvoidWrite
+import mozpack.path as mozpath
+
+
+def process_define_file(output, input):
+    '''Creates the given config header. A config header is generated by
+    taking the corresponding source file and replacing some #define/#undef
+    occurences:
+        "#undef NAME" is turned into "#define NAME VALUE"
+        "#define NAME" is unchanged
+        "#define NAME ORIGINAL_VALUE" is turned into "#define NAME VALUE"
+        "#undef UNKNOWN_NAME" is turned into "/* #undef UNKNOWN_NAME */"
+        Whitespaces are preserved.
+
+    As a special rule, "#undef ALLDEFINES" is turned into "#define NAME
+    VALUE" for all the defined variables.
+    '''
+
+    path = os.path.abspath(input)
+
+    config = ConfigEnvironment.from_config_status(
+        mozpath.join(topobjdir, 'config.status'))
+
+    if mozpath.basedir(path,
+                       [mozpath.join(config.topsrcdir, 'js/src')]) and \
+            not config.substs.get('JS_STANDALONE'):
+        config = ConfigEnvironment.from_config_status(
+            mozpath.join(topobjdir, 'js', 'src', 'config.status'))
+
+    with open(path, 'rU') as input:
+        r = re.compile('^\s*#\s*(?P<cmd>[a-z]+)(?:\s+(?P<name>\S+)(?:\s+(?P<value>\S+))?)?', re.U)
+        for l in input:
+            m = r.match(l)
+            if m:
+                cmd = m.group('cmd')
+                name = m.group('name')
+                value = m.group('value')
+                if name:
+                    if name == 'ALLDEFINES':
+                        if cmd == 'define':
+                            raise Exception(
+                                '`#define ALLDEFINES` is not allowed in a '
+                                'CONFIGURE_DEFINE_FILE')
+                        defines = '\n'.join(sorted(
+                            '#define %s %s' % (name, val)
+                            for name, val in config.defines.iteritems()
+                            if name not in config.non_global_defines))
+                        l = l[:m.start('cmd') - 1] \
+                            + defines + l[m.end('name'):]
+                    elif name in config.defines:
+                        if cmd == 'define' and value:
+                            l = l[:m.start('value')] \
+                                + str(config.defines[name]) \
+                                + l[m.end('value'):]
+                        elif cmd == 'undef':
+                            l = l[:m.start('cmd')] \
+                                + 'define' \
+                                + l[m.end('cmd'):m.end('name')] \
+                                + ' ' \
+                                + str(config.defines[name]) \
+                                + l[m.end('name'):]
+                    elif cmd == 'undef':
+                       l = '/* ' + l[:m.end('name')] + ' */' + l[m.end('name'):]
+
+            output.write(l)
+
+    return {path, config.source}
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Process define files.')
+
+    parser.add_argument('input', help='Input define file.')
+
+    args = parser.parse_args(argv)
+
+    return process_define_file(sys.stdout, args.input)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -0,0 +1,120 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import os
+import sys
+import time
+
+from mozpack.copier import (
+    FileCopier,
+    FileRegistry,
+)
+from mozpack.files import (
+    BaseFile,
+    FileFinder,
+)
+from mozpack.manifests import (
+    InstallManifest,
+    InstallManifestNoSymlinks,
+)
+from mozbuild.util import DefinesAction
+
+
+COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
+    'Added/updated {updated}; ' \
+    'Removed {rm_files} files and {rm_dirs} directories.'
+
+
+def process_manifest(destdir, paths, track=None,
+        remove_unaccounted=True,
+        remove_all_directory_symlinks=True,
+        remove_empty_directories=True,
+        no_symlinks=False,
+        defines={}):
+
+    if track:
+        if os.path.exists(track):
+            # We use the same format as install manifests for the tracking
+            # data.
+            manifest = InstallManifest(path=track)
+            remove_unaccounted = FileRegistry()
+            dummy_file = BaseFile()
+
+            finder = FileFinder(destdir, find_executables=False,
+                                find_dotfiles=True)
+            for dest in manifest._dests:
+                for p, f in finder.find(dest):
+                    remove_unaccounted.add(p, dummy_file)
+
+        else:
+            # If tracking is enabled and there is no file, we don't want to
+            # be removing anything.
+            remove_unaccounted=False
+            remove_empty_directories=False
+            remove_all_directory_symlinks=False
+
+    manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest
+    manifest = manifest_cls()
+    for path in paths:
+        manifest |= manifest_cls(path=path)
+
+    copier = FileCopier()
+    manifest.populate_registry(copier, defines_override=defines)
+    result = copier.copy(destdir,
+        remove_unaccounted=remove_unaccounted,
+        remove_all_directory_symlinks=remove_all_directory_symlinks,
+        remove_empty_directories=remove_empty_directories)
+
+    if track:
+        manifest.write(path=track)
+
+    return result
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Process install manifest files.')
+
+    parser.add_argument('destdir', help='Destination directory.')
+    parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
+    parser.add_argument('--no-remove', action='store_true',
+        help='Do not remove unaccounted files from destination.')
+    parser.add_argument('--no-remove-all-directory-symlinks', action='store_true',
+        help='Do not remove all directory symlinks from destination.')
+    parser.add_argument('--no-remove-empty-directories', action='store_true',
+        help='Do not remove empty directories from destination.')
+    parser.add_argument('--no-symlinks', action='store_true',
+        help='Do not install symbolic links. Always copy files')
+    parser.add_argument('--track', metavar="PATH",
+        help='Use installed files tracking information from the given path.')
+    parser.add_argument('-D', action=DefinesAction,
+        dest='defines', metavar="VAR[=VAL]",
+        help='Define a variable to override what is specified in the manifest')
+
+    args = parser.parse_args(argv)
+
+    start = time.time()
+
+    result = process_manifest(args.destdir, args.manifests,
+        track=args.track, remove_unaccounted=not args.no_remove,
+        remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
+        remove_empty_directories=not args.no_remove_empty_directories,
+        no_symlinks=args.no_symlinks,
+        defines=args.defines)
+
+    elapsed = time.time() - start
+
+    print(COMPLETE.format(
+        elapsed=elapsed,
+        dest=args.destdir,
+        existing=result.existing_files_count,
+        updated=result.updated_files_count,
+        rm_files=result.removed_files_count,
+        rm_dirs=result.removed_directories_count))
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

python/mozbuild/mozbuild/action/test_archive.py

@@ -0,0 +1,565 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This action is used to produce test archives.
+#
+# Ideally, the data in this file should be defined in moz.build files.
+# It is defined inline because this was easiest to make test archive
+# generation faster.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import itertools
+import os
+import sys
+import time
+
+from manifestparser import TestManifest
+from reftest import ReftestManifest
+
+from mozbuild.util import ensureParentDir
+from mozpack.files import FileFinder
+from mozpack.mozjar import JarWriter
+import mozpack.path as mozpath
+
+import buildconfig
+
+STAGE = mozpath.join(buildconfig.topobjdir, 'dist', 'test-stage')
+
+TEST_HARNESS_BINS = [
+    'BadCertServer',
+    'GenerateOCSPResponse',
+    'OCSPStaplingServer',
+    'SmokeDMD',
+    'certutil',
+    'crashinject',
+    'fileid',
+    'minidumpwriter',
+    'pk12util',
+    'screenshot',
+    'screentopng',
+    'ssltunnel',
+    'xpcshell',
+]
+
+# The fileid utility depends on mozglue. See bug 1069556.
+TEST_HARNESS_DLLS = [
+    'crashinjectdll',
+    'mozglue'
+]
+
+TEST_PLUGIN_DLLS = [
+    'npctrltest',
+    'npsecondtest',
+    'npswftest',
+    'nptest',
+    'nptestjava',
+    'npthirdtest',
+]
+
+TEST_PLUGIN_DIRS = [
+    'JavaTest.plugin/**',
+    'SecondTest.plugin/**',
+    'Test.plugin/**',
+    'ThirdTest.plugin/**',
+    'npctrltest.plugin/**',
+    'npswftest.plugin/**',
+]
+
+GMP_TEST_PLUGIN_DIRS = [
+    'gmp-clearkey/**',
+    'gmp-fake/**',
+    'gmp-fakeopenh264/**',
+]
+
+
+ARCHIVE_FILES = {
+    'common': [
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': '**',
+            'ignore': [
+                'cppunittest/**',
+                'gtest/**',
+                'mochitest/**',
+                'reftest/**',
+                'talos/**',
+                'web-platform/**',
+                'xpcshell/**',
+            ],
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests',
+            'pattern': 'modules/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing/marionette',
+            'patterns': [
+                'client/**',
+                'harness/**',
+                'puppeteer/**',
+                'mach_test_package_commands.py',
+            ],
+            'dest': 'marionette',
+            'ignore': [
+                'client/docs',
+                'harness/marionette_harness/tests',
+                'puppeteer/firefox/docs',
+            ],
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': '',
+            'manifests': [
+                'testing/marionette/harness/marionette_harness/tests/unit-tests.ini',
+                'testing/marionette/harness/marionette_harness/tests/webapi-tests.ini',
+            ],
+            # We also need the manifests and harness_unit tests
+            'pattern': 'testing/marionette/harness/marionette_harness/tests/**',
+            'dest': 'marionette/tests',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests',
+            'pattern': 'mozbase/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'firefox-ui/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'dom/media/test/external',
+            'pattern': '**',
+            'dest': 'external-media-tests',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'js/src',
+            'pattern': 'jit-test/**',
+            'dest': 'jit-test',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'js/src/tests',
+            'pattern': 'ecma_6/**',
+            'dest': 'jit-test/tests',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'js/src/tests',
+            'pattern': 'js1_8_5/**',
+            'dest': 'jit-test/tests',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'js/src/tests',
+            'pattern': 'lib/**',
+            'dest': 'jit-test/tests',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'js/src',
+            'pattern': 'jsapi.h',
+            'dest': 'jit-test',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'tps/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'services/sync/',
+            'pattern': 'tps/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'services/sync/tests/tps',
+            'pattern': '**',
+            'dest': 'tps/tests',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing/web-platform/tests/tools/wptserve',
+            'pattern': '**',
+            'dest': 'tools/wptserve',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '',
+            'pattern': 'mozinfo.json',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'dist/bin',
+            'patterns': [
+                '%s%s' % (f, buildconfig.substs['BIN_SUFFIX'])
+                for f in TEST_HARNESS_BINS
+            ] + [
+                '%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX'])
+                for f in TEST_HARNESS_DLLS
+            ],
+            'dest': 'bin',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'dist/plugins',
+            'patterns': [
+                '%s%s%s' % (buildconfig.substs['DLL_PREFIX'], f, buildconfig.substs['DLL_SUFFIX'])
+                for f in TEST_PLUGIN_DLLS
+            ],
+            'dest': 'bin/plugins',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'dist/plugins',
+            'patterns': TEST_PLUGIN_DIRS,
+            'dest': 'bin/plugins',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'dist/bin',
+            'patterns': GMP_TEST_PLUGIN_DIRS,
+            'dest': 'bin/plugins',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'dist/bin',
+            'patterns': [
+                'dmd.py',
+                'fix_linux_stack.py',
+                'fix_macosx_stack.py',
+                'fix_stack_using_bpsyms.py',
+            ],
+            'dest': 'bin',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'dist/bin/components',
+            'patterns': [
+                'httpd.js',
+                'httpd.manifest',
+                'test_necko.xpt',
+            ],
+            'dest': 'bin/components',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'build/pgo/certs',
+            'pattern': '**',
+            'dest': 'certs',
+        }
+    ],
+    'cppunittest': [
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': 'cppunittest/**',
+        },
+        # We don't ship these files if startup cache is disabled, which is
+        # rare. But it shouldn't matter for test archives.
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'startupcache/test',
+            'pattern': 'TestStartupCacheTelemetry.*',
+            'dest': 'cppunittest',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'runcppunittests.py',
+            'dest': 'cppunittest',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'remotecppunittests.py',
+            'dest': 'cppunittest',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'cppunittest.ini',
+            'dest': 'cppunittest',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '',
+            'pattern': 'mozinfo.json',
+            'dest': 'cppunittest',
+        },
+    ],
+    'gtest': [
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': 'gtest/**',
+        },
+    ],
+    'mochitest': [
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests/testing',
+            'pattern': 'mochitest/**',
+        },
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': 'mochitest/**',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '',
+            'pattern': 'mozinfo.json',
+            'dest': 'mochitest'
+        }
+    ],
+    'mozharness': [
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'mozharness/**',
+        },
+    ],
+    'reftest': [
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests',
+            'pattern': 'reftest/**',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '',
+            'pattern': 'mozinfo.json',
+            'dest': 'reftest',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': '',
+            'manifests': [
+                'layout/reftests/reftest.list',
+                'testing/crashtest/crashtests.list',
+            ],
+            'dest': 'reftest/tests',
+        }
+    ],
+    'talos': [
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'talos/**',
+        },
+    ],
+    'web-platform': [
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'web-platform/meta/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'web-platform/mozilla/**',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing',
+            'pattern': 'web-platform/tests/**',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests',
+            'pattern': 'web-platform/**',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '',
+            'pattern': 'mozinfo.json',
+            'dest': 'web-platform',
+        },
+    ],
+    'xpcshell': [
+        {
+            'source': buildconfig.topobjdir,
+            'base': '_tests/xpcshell',
+            'pattern': '**',
+            'dest': 'xpcshell/tests',
+        },
+        {
+            'source': buildconfig.topsrcdir,
+            'base': 'testing/xpcshell',
+            'patterns': [
+                'head.js',
+                'mach_test_package_commands.py',
+                'moz-http2/**',
+                'moz-spdy/**',
+                'node-http2/**',
+                'node-spdy/**',
+                'remotexpcshelltests.py',
+                'runtestsb2g.py',
+                'runxpcshelltests.py',
+                'xpcshellcommandline.py',
+            ],
+            'dest': 'xpcshell',
+        },
+        {
+            'source': STAGE,
+            'base': '',
+            'pattern': 'xpcshell/**',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': '',
+            'pattern': 'mozinfo.json',
+            'dest': 'xpcshell',
+        },
+        {
+            'source': buildconfig.topobjdir,
+            'base': 'build',
+            'pattern': 'automation.py',
+            'dest': 'xpcshell',
+        },
+    ],
+}
+
+
+# "common" is our catch all archive and it ignores things from other archives.
+# Verify nothing sneaks into ARCHIVE_FILES without a corresponding exclusion
+# rule in the "common" archive.
+for k, v in ARCHIVE_FILES.items():
+    # Skip mozharness because it isn't staged.
+    if k in ('common', 'mozharness'):
+        continue
+
+    ignores = set(itertools.chain(*(e.get('ignore', [])
+                                  for e in ARCHIVE_FILES['common'])))
+
+    if not any(p.startswith('%s/' % k) for p in ignores):
+        raise Exception('"common" ignore list probably should contain %s' % k)
+
+
+def find_files(archive):
+    for entry in ARCHIVE_FILES[archive]:
+        source = entry['source']
+        dest = entry.get('dest')
+        base = entry.get('base', '')
+
+        pattern = entry.get('pattern')
+        patterns = entry.get('patterns', [])
+        if pattern:
+            patterns.append(pattern)
+
+        manifest = entry.get('manifest')
+        manifests = entry.get('manifests', [])
+        if manifest:
+            manifests.append(manifest)
+        if manifests:
+            dirs = find_manifest_dirs(buildconfig.topsrcdir, manifests)
+            patterns.extend({'{}/**'.format(d) for d in dirs})
+
+        ignore = list(entry.get('ignore', []))
+        ignore.extend([
+            '**/.flake8',
+            '**/.mkdir.done',
+            '**/*.pyc',
+        ])
+
+        common_kwargs = {
+            'find_executables': False,
+            'find_dotfiles': True,
+            'ignore': ignore,
+        }
+
+        finder = FileFinder(os.path.join(source, base), **common_kwargs)
+
+        for pattern in patterns:
+            for p, f in finder.find(pattern):
+                if dest:
+                    p = mozpath.join(dest, p)
+                yield p, f
+
+
+def find_manifest_dirs(topsrcdir, manifests):
+    """Routine to retrieve directories specified in a manifest, relative to topsrcdir.
+
+    It does not recurse into manifests, as we currently have no need for that.
+    """
+    dirs = set()
+
+    for p in manifests:
+        p = os.path.join(topsrcdir, p)
+
+        if p.endswith('.ini'):
+            test_manifest = TestManifest()
+            test_manifest.read(p)
+            dirs |= set([os.path.dirname(m) for m in test_manifest.manifests()])
+
+        elif p.endswith('.list'):
+            m = ReftestManifest()
+            m.load(p)
+            dirs |= m.dirs
+
+        else:
+            raise Exception('"{}" is not a supported manifest format.'.format(
+                os.path.splitext(p)[1]))
+
+    dirs = {mozpath.normpath(d[len(topsrcdir):]).lstrip('/') for d in dirs}
+
+    # Filter out children captured by parent directories because duplicates
+    # will confuse things later on.
+    def parents(p):
+        while True:
+            p = mozpath.dirname(p)
+            if not p:
+                break
+            yield p
+
+    seen = set()
+    for d in sorted(dirs, key=len):
+        if not any(p in seen for p in parents(d)):
+            seen.add(d)
+
+    return sorted(seen)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        description='Produce test archives')
+    parser.add_argument('archive', help='Which archive to generate')
+    parser.add_argument('outputfile', help='File to write output to')
+
+    args = parser.parse_args(argv)
+
+    if not args.outputfile.endswith('.zip'):
+        raise Exception('expected zip output file')
+
+    file_count = 0
+    t_start = time.time()
+    ensureParentDir(args.outputfile)
+    with open(args.outputfile, 'wb') as fh:
+        # Experimentation revealed that level 5 is significantly faster and has
+        # marginally larger sizes than higher values and is the sweet spot
+        # for optimal compression. Read the detailed commit message that
+        # introduced this for raw numbers.
+        with JarWriter(fileobj=fh, optimize=False, compress_level=5) as writer:
+            res = find_files(args.archive)
+            for p, f in res:
+                writer.add(p.encode('utf-8'), f.read(), mode=f.mode, skip_duplicates=True)
+                file_count += 1
+
+    duration = time.time() - t_start
+    zip_size = os.path.getsize(args.outputfile)
+    basename = os.path.basename(args.outputfile)
+    print('Wrote %d files in %d bytes to %s in %.2fs' % (
+          file_count, zip_size, basename, duration))
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/webidl.py

@@ -0,0 +1,19 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import sys
+
+from mozwebidlcodegen import BuildSystemWebIDL
+
+
+def main(argv):
+    """Perform WebIDL code generation required by the build system."""
+    manager = BuildSystemWebIDL.from_environment().manager
+    manager.generate_build_files()
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/action/xpccheck.py

@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''A generic script to verify all test files are in the
+corresponding .ini file.
+
+Usage: xpccheck.py <directory> [<directory> ...]
+'''
+
+from __future__ import absolute_import
+
+import sys
+import os
+from glob import glob
+import manifestparser
+
+def getIniTests(testdir):
+  mp = manifestparser.ManifestParser(strict=False)
+  mp.read(os.path.join(testdir, 'xpcshell.ini'))
+  return mp.tests
+
+def verifyDirectory(initests, directory):
+  files = glob(os.path.join(os.path.abspath(directory), "test_*"))
+  for f in files:
+    if (not os.path.isfile(f)):
+      continue
+
+    name = os.path.basename(f)
+    if name.endswith('.in'):
+      name = name[:-3]
+
+    if not name.endswith('.js'):
+      continue
+
+    found = False
+    for test in initests:
+      if os.path.join(os.path.abspath(directory), name) == test['path']:
+        found = True
+        break
+   
+    if not found:
+      print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | test %s is missing from test manifest %s!" % (name, os.path.join(directory, 'xpcshell.ini'))
+      sys.exit(1)
+
+def verifyIniFile(initests, directory):
+  files = glob(os.path.join(os.path.abspath(directory), "test_*"))
+  for test in initests:
+    name = test['path'].split('/')[-1]
+
+    found = False
+    for f in files:
+
+      fname = f.split('/')[-1]
+      if fname.endswith('.in'):
+        fname = '.in'.join(fname.split('.in')[:-1])
+
+      if os.path.join(os.path.abspath(directory), fname) == test['path']:
+        found = True
+        break
+
+    if not found:
+      print >>sys.stderr, "TEST-UNEXPECTED-FAIL | xpccheck | found %s in xpcshell.ini and not in directory '%s'" % (name, directory)
+      sys.exit(1)
+
+def main(argv):
+  if len(argv) < 2:
+    print >>sys.stderr, "Usage: xpccheck.py <topsrcdir> <directory> [<directory> ...]"
+    sys.exit(1)
+
+  topsrcdir = argv[0]
+  for d in argv[1:]:
+    # xpcshell-unpack is a copy of xpcshell sibling directory and in the Makefile
+    # we copy all files (including xpcshell.ini from the sibling directory.
+    if d.endswith('toolkit/mozapps/extensions/test/xpcshell-unpack'):
+      continue
+
+    initests = getIniTests(d)
+    verifyDirectory(initests, d)
+    verifyIniFile(initests, d)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

python/mozbuild/mozbuild/action/xpidl-process.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script is used to generate an output header and xpt file for
+# input IDL file(s). It's purpose is to directly support the build
+# system. The API will change to meet the needs of the build system.
+
+from __future__ import absolute_import
+
+import argparse
+import os
+import sys
+
+from io import BytesIO
+
+from buildconfig import topsrcdir
+from xpidl.header import print_header
+from xpidl.typelib import write_typelib
+from xpidl.xpidl import IDLParser
+from xpt import xpt_link
+
+from mozbuild.makeutil import Makefile
+from mozbuild.pythonutil import iter_modules_in_path
+from mozbuild.util import FileAvoidWrite
+
+
+def process(input_dir, inc_paths, cache_dir, header_dir, xpt_dir, deps_dir, module, stems):
+    p = IDLParser(outputdir=cache_dir)
+
+    xpts = {}
+    mk = Makefile()
+    rule = mk.create_rule()
+
+    # Write out dependencies for Python modules we import. If this list isn't
+    # up to date, we will not re-process XPIDL files if the processor changes.
+    rule.add_dependencies(iter_modules_in_path(topsrcdir))
+
+    for stem in stems:
+        path = os.path.join(input_dir, '%s.idl' % stem)
+        idl_data = open(path).read()
+
+        idl = p.parse(idl_data, filename=path)
+        idl.resolve([input_dir] + inc_paths, p)
+
+        header_path = os.path.join(header_dir, '%s.h' % stem)
+
+        xpt = BytesIO()
+        write_typelib(idl, xpt, path)
+        xpt.seek(0)
+        xpts[stem] = xpt
+
+        rule.add_dependencies(idl.deps)
+
+        with FileAvoidWrite(header_path) as fh:
+            print_header(idl, fh, path)
+
+    # TODO use FileAvoidWrite once it supports binary mode.
+    xpt_path = os.path.join(xpt_dir, '%s.xpt' % module)
+    xpt_link(xpts.values()).write(xpt_path)
+
+    rule.add_targets([xpt_path])
+    if deps_dir:
+        deps_path = os.path.join(deps_dir, '%s.pp' % module)
+        with FileAvoidWrite(deps_path) as fh:
+            mk.dump(fh)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--cache-dir',
+        help='Directory in which to find or write cached lexer data.')
+    parser.add_argument('--depsdir',
+        help='Directory in which to write dependency files.')
+    parser.add_argument('inputdir',
+        help='Directory in which to find source .idl files.')
+    parser.add_argument('headerdir',
+        help='Directory in which to write header files.')
+    parser.add_argument('xptdir',
+        help='Directory in which to write xpt file.')
+    parser.add_argument('module',
+        help='Final module name to use for linked output xpt file.')
+    parser.add_argument('idls', nargs='+',
+        help='Source .idl file(s). Specified as stems only.')
+    parser.add_argument('-I', dest='incpath', action='append', default=[],
+        help='Extra directories where to look for included .idl files.')
+
+    args = parser.parse_args(argv)
+    process(args.inputdir, args.incpath, args.cache_dir, args.headerdir,
+        args.xptdir, args.depsdir, args.module, args.idls)
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

python/mozbuild/mozbuild/action/zip.py

@@ -0,0 +1,39 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script creates a zip file, but will also strip any binaries
+# it finds before adding them to the zip.
+
+from __future__ import absolute_import
+
+from mozpack.files import FileFinder
+from mozpack.copier import Jarrer
+from mozpack.errors import errors
+
+import argparse
+import mozpack.path as mozpath
+import sys
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-C", metavar='DIR', default=".",
+                        help="Change to given directory before considering "
+                        "other paths")
+    parser.add_argument("zip", help="Path to zip file to write")
+    parser.add_argument("input", nargs="+",
+                        help="Path to files to add to zip")
+    args = parser.parse_args(args)
+
+    jarrer = Jarrer(optimize=False)
+
+    with errors.accumulate():
+        finder = FileFinder(args.C)
+        for path in args.input:
+            for p, f in finder.find(path):
+                jarrer.add(p, f)
+        jarrer.copy(mozpath.join(args.C, args.zip))
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

python/mozbuild/mozbuild/android_version_code.py

@@ -0,0 +1,167 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import math
+import sys
+import time
+
+# Builds before this build ID use the v0 version scheme.  Builds after this
+# build ID use the v1 version scheme.
+V1_CUTOFF = 20150801000000 # YYYYmmddHHMMSS
+
+def android_version_code_v0(buildid, cpu_arch=None, min_sdk=0, max_sdk=0):
+    base = int(str(buildid)[:10])
+    # None is interpreted as arm.
+    if not cpu_arch or cpu_arch in ['armeabi', 'armeabi-v7a']:
+        # Increment by MIN_SDK_VERSION -- this adds 9 to every build ID as a
+        # minimum.  Our split APK starts at 15.
+        return base + min_sdk + 0
+    elif cpu_arch in ['x86']:
+        # Increment the version code by 3 for x86 builds so they are offered to
+        # x86 phones that have ARM emulators, beating the 2-point advantage that
+        # the v15+ ARMv7 APK has.  If we change our splits in the future, we'll
+        # need to do this further still.
+        return base + min_sdk + 3
+    else:
+        raise ValueError("Don't know how to compute android:versionCode "
+                         "for CPU arch %s" % cpu_arch)
+
+def android_version_code_v1(buildid, cpu_arch=None, min_sdk=0, max_sdk=0):
+    '''Generate a v1 android:versionCode.
+
+    The important consideration is that version codes be monotonically
+    increasing (per Android package name) for all published builds.  The input
+    build IDs are based on timestamps and hence are always monotonically
+    increasing.
+
+    The generated v1 version codes look like (in binary):
+
+    0111 1000 0010 tttt tttt tttt tttt txpg
+
+    The 17 bits labelled 't' represent the number of hours since midnight on
+    September 1, 2015.  (2015090100 in YYYYMMMDDHH format.)  This yields a
+    little under 15 years worth of hourly build identifiers, since 2**17 / (366
+    * 24) =~ 14.92.
+
+    The bits labelled 'x', 'p', and 'g' are feature flags.
+
+    The bit labelled 'x' is 1 if the build is for an x86 architecture and 0
+    otherwise, which means the build is for an ARM architecture.  (Fennec no
+    longer supports ARMv6, so ARM is equivalent to ARMv7 and above.)
+
+    The bit labelled 'p' is a placeholder that is always 0 (for now).
+
+    Firefox no longer supports API 14 or earlier.
+
+    This version code computation allows for a split on API levels that allowed
+    us to ship builds specifically for Gingerbread (API 9-10); we preserve
+    that functionality for sanity's sake, and to allow us to reintroduce a
+    split in the future.
+
+    At present, the bit labelled 'g' is 1 if the build is an ARM build
+    targeting API 15+, which will always be the case.
+
+    We throw an explanatory exception when we are within one calendar year of
+    running out of build events.  This gives lots of time to update the version
+    scheme.  The responsible individual should then bump the range (to allow
+    builds to continue) and use the time remaining to update the version scheme
+    via the reserved high order bits.
+
+    N.B.: the reserved 0 bit to the left of the highest order 't' bit can,
+    sometimes, be used to bump the version scheme.  In addition, by reducing the
+    granularity of the build identifiers (for example, moving to identifying
+    builds every 2 or 4 hours), the version scheme may be adjusted further still
+    without losing a (valuable) high order bit.
+    '''
+    def hours_since_cutoff(buildid):
+        # The ID is formatted like YYYYMMDDHHMMSS (using
+        # datetime.now().strftime('%Y%m%d%H%M%S'); see build/variables.py).
+        # The inverse function is time.strptime.
+        # N.B.: the time module expresses time as decimal seconds since the
+        # epoch.
+        fmt = '%Y%m%d%H%M%S'
+        build = time.strptime(str(buildid), fmt)
+        cutoff = time.strptime(str(V1_CUTOFF), fmt)
+        return int(math.floor((time.mktime(build) - time.mktime(cutoff)) / (60.0 * 60.0)))
+
+    # Of the 21 low order bits, we take 17 bits for builds.
+    base = hours_since_cutoff(buildid)
+    if base < 0:
+        raise ValueError("Something has gone horribly wrong: cannot calculate "
+                         "android:versionCode from build ID %s: hours underflow "
+                         "bits allotted!" % buildid)
+    if base > 2**17:
+        raise ValueError("Something has gone horribly wrong: cannot calculate "
+                         "android:versionCode from build ID %s: hours overflow "
+                         "bits allotted!" % buildid)
+    if base > 2**17 - 366 * 24:
+        raise ValueError("Running out of low order bits calculating "
+                         "android:versionCode from build ID %s: "
+                         "; YOU HAVE ONE YEAR TO UPDATE THE VERSION SCHEME." % buildid)
+
+    version = 0b1111000001000000000000000000000
+    # We reserve 1 "middle" high order bit for the future, and 3 low order bits
+    # for architecture and APK splits.
+    version |= base << 3
+
+    # None is interpreted as arm.
+    if not cpu_arch or cpu_arch in ['armeabi', 'armeabi-v7a']:
+        # 0 is interpreted as SDK 9.
+        if not min_sdk or min_sdk == 9:
+            pass
+        # This used to compare to 11. The 15+ APK directly supersedes 11+, so
+        # we reuse this check.
+        elif min_sdk == 15:
+            version |= 1 << 0
+        else:
+            raise ValueError("Don't know how to compute android:versionCode "
+                             "for CPU arch %s and min SDK %s" % (cpu_arch, min_sdk))
+    elif cpu_arch in ['x86']:
+        version |= 1 << 2
+    else:
+        raise ValueError("Don't know how to compute android:versionCode "
+                         "for CPU arch %s" % cpu_arch)
+
+    return version
+
+def android_version_code(buildid, *args, **kwargs):
+    base = int(str(buildid))
+    if base < V1_CUTOFF:
+        return android_version_code_v0(buildid, *args, **kwargs)
+    else:
+        return android_version_code_v1(buildid, *args, **kwargs)
+
+
+def main(argv):
+    parser = argparse.ArgumentParser('Generate an android:versionCode',
+                                     add_help=False)
+    parser.add_argument('--verbose', action='store_true',
+                        default=False,
+                        help='Be verbose')
+    parser.add_argument('--with-android-cpu-arch', dest='cpu_arch',
+                        choices=['armeabi', 'armeabi-v7a', 'mips', 'x86'],
+                        help='The target CPU architecture')
+    parser.add_argument('--with-android-min-sdk-version', dest='min_sdk',
+                        type=int, default=0,
+                        help='The minimum target SDK')
+    parser.add_argument('--with-android-max-sdk-version', dest='max_sdk',
+                        type=int, default=0,
+                        help='The maximum target SDK')
+    parser.add_argument('buildid', type=int,
+                        help='The input build ID')
+
+    args = parser.parse_args(argv)
+    code = android_version_code(args.buildid,
+        cpu_arch=args.cpu_arch,
+        min_sdk=args.min_sdk,
+        max_sdk=args.max_sdk)
+    print(code)
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

python/mozbuild/mozbuild/backend/__init__.py

@@ -0,0 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+backends = {
+    'AndroidEclipse': 'mozbuild.backend.android_eclipse',
+    'ChromeMap': 'mozbuild.codecoverage.chrome_map',
+    'CompileDB': 'mozbuild.compilation.database',
+    'CppEclipse': 'mozbuild.backend.cpp_eclipse',
+    'FasterMake': 'mozbuild.backend.fastermake',
+    'FasterMake+RecursiveMake': None,
+    'RecursiveMake': 'mozbuild.backend.recursivemake',
+    'Tup': 'mozbuild.backend.tup',
+    'VisualStudio': 'mozbuild.backend.visualstudio',
+}
+
+
+def get_backend_class(name):
+    if '+' in name:
+        from mozbuild.backend.base import HybridBackend
+        return HybridBackend(*(get_backend_class(name)
+                               for name in name.split('+')))
+
+    class_name = '%sBackend' % name
+    module = __import__(backends[name], globals(), locals(), [class_name])
+    return getattr(module, class_name)

python/mozbuild/mozbuild/backend/android_eclipse.py

@@ -0,0 +1,267 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import itertools
+import os
+import time
+import types
+import xml.dom.minidom as minidom
+import xml.etree.ElementTree as ET
+
+from mozpack.copier import FileCopier
+from mozpack.files import (FileFinder, PreprocessedFile)
+from mozpack.manifests import InstallManifest
+import mozpack.path as mozpath
+
+from .common import CommonBackend
+from ..frontend.data import (
+    AndroidEclipseProjectData,
+    ContextDerived,
+    ContextWrapped,
+)
+from ..makeutil import Makefile
+from ..util import ensureParentDir
+from mozbuild.base import (
+    ExecutionSummary,
+    MachCommandConditions,
+)
+
+
+def pretty_print(element):
+    """Return a pretty-printed XML string for an Element.
+    """
+    s = ET.tostring(element, 'utf-8')
+    # minidom wraps element in a Document node; firstChild strips it.
+    return minidom.parseString(s).firstChild.toprettyxml(indent='  ')
+
+
+class AndroidEclipseBackend(CommonBackend):
+    """Backend that generates Android Eclipse project files.
+    """
+    def __init__(self, environment):
+        if not MachCommandConditions.is_android(environment):
+            raise Exception(
+                'The Android Eclipse backend is not available with this '
+                'configuration.')
+
+        super(AndroidEclipseBackend, self).__init__(environment)
+
+    def summary(self):
+        return ExecutionSummary(
+            'AndroidEclipse backend executed in {execution_time:.2f}s\n'
+            'Wrote {projects:d} Android Eclipse projects to {path:s}; '
+            '{created:d} created; {updated:d} updated',
+            execution_time=self._execution_time,
+            projects=self._created_count + self._updated_count,
+            path=mozpath.join(self.environment.topobjdir, 'android_eclipse'),
+            created=self._created_count,
+            updated=self._updated_count,
+        )
+
+    def consume_object(self, obj):
+        """Write out Android Eclipse project files."""
+
+        if not isinstance(obj, ContextDerived):
+            return False
+
+        if CommonBackend.consume_object(self, obj):
+            # If CommonBackend acknowledged the object, we're done with it.
+            return True
+
+        # Handle the one case we care about specially.
+        if isinstance(obj, ContextWrapped) and isinstance(obj.wrapped, AndroidEclipseProjectData):
+            self._process_android_eclipse_project_data(obj.wrapped, obj.srcdir, obj.objdir)
+
+        # We don't want to handle most things, so we just acknowledge all objects
+        return True
+
+    def consume_finished(self):
+        """The common backend handles WebIDL and test files. We don't handle
+        these, so we don't call our superclass.
+        """
+
+    def _Element_for_classpathentry(self, cpe):
+        """Turn a ClassPathEntry into an XML Element, like one of:
+        <classpathentry including="**/*.java" kind="src" path="preprocessed"/>
+        <classpathentry including="**/*.java" excluding="org/mozilla/gecko/Excluded.java|org/mozilla/gecko/SecondExcluded.java" kind="src" path="src"/>
+        <classpathentry including="**/*.java" kind="src" path="thirdparty">
+            <attributes>
+                <attribute name="ignore_optional_problems" value="true"/>
+            </attributes>
+        </classpathentry>
+        """
+        e = ET.Element('classpathentry')
+        e.set('kind', 'src')
+        e.set('including', '**/*.java')
+        e.set('path', cpe.path)
+        if cpe.exclude_patterns:
+            e.set('excluding', '|'.join(sorted(cpe.exclude_patterns)))
+        if cpe.ignore_warnings:
+            attrs = ET.SubElement(e, 'attributes')
+            attr = ET.SubElement(attrs, 'attribute')
+            attr.set('name', 'ignore_optional_problems')
+            attr.set('value', 'true')
+        return e
+
+    def _Element_for_referenced_project(self, name):
+        """Turn a referenced project name into an XML Element, like:
+        <classpathentry combineaccessrules="false" kind="src" path="/Fennec"/>
+        """
+        e = ET.Element('classpathentry')
+        e.set('kind', 'src')
+        e.set('combineaccessrules', 'false')
+         # All project directories are in the same root; this
+         # reference is absolute in the Eclipse namespace.
+        e.set('path', '/' + name)
+        return e
+
+    def _Element_for_extra_jar(self, name):
+        """Turn a referenced JAR name into an XML Element, like:
+        <classpathentry exported="true" kind="lib" path="/Users/nalexander/Mozilla/gecko-dev/build/mobile/robocop/robotium-solo-4.3.1.jar"/>
+        """
+        e = ET.Element('classpathentry')
+        e.set('kind', 'lib')
+        e.set('exported', 'true')
+        e.set('path', name)
+        return e
+
+    def _Element_for_filtered_resources(self, filtered_resources):
+        """Turn a list of filtered resource arguments like
+        ['1.0-projectRelativePath-matches-false-false-*org/mozilla/gecko/resources/**']
+        into an XML Element, like:
+        <filteredResources>
+          <filter>
+            <id>1393009101322</id>
+            <name></name>
+            <type>30</type>
+            <matcher>
+              <id>org.eclipse.ui.ide.multiFilter</id>
+              <arguments>1.0-projectRelativePath-matches-false-false-*org/mozilla/gecko/resources/**</arguments>
+            </matcher>
+          </filter>
+        </filteredResources>
+
+        The id is random; the values are magic."""
+
+        id = int(1000 * time.time())
+        filteredResources = ET.Element('filteredResources')
+        for arg in sorted(filtered_resources):
+            e = ET.SubElement(filteredResources, 'filter')
+            ET.SubElement(e, 'id').text = str(id)
+            id += 1
+            ET.SubElement(e, 'name')
+            ET.SubElement(e, 'type').text = '30' # It's magic!
+            matcher = ET.SubElement(e, 'matcher')
+            ET.SubElement(matcher, 'id').text = 'org.eclipse.ui.ide.multiFilter'
+            ET.SubElement(matcher, 'arguments').text = str(arg)
+        return filteredResources
+
+    def _manifest_for_project(self, srcdir, project):
+        manifest = InstallManifest()
+
+        if project.manifest:
+            manifest.add_copy(mozpath.join(srcdir, project.manifest), 'AndroidManifest.xml')
+
+        if project.res:
+            manifest.add_symlink(mozpath.join(srcdir, project.res), 'res')
+        else:
+            # Eclipse expects a res directory no matter what, so we
+            # make an empty directory if the project doesn't specify.
+            res = os.path.abspath(mozpath.join(os.path.dirname(__file__),
+                'templates', 'android_eclipse_empty_resource_directory'))
+            manifest.add_pattern_copy(res, '.**', 'res')
+
+        if project.assets:
+            manifest.add_symlink(mozpath.join(srcdir, project.assets), 'assets')
+
+        for cpe in project._classpathentries:
+            manifest.add_symlink(mozpath.join(srcdir, cpe.srcdir), cpe.dstdir)
+
+        # JARs and native libraries go in the same place. For now, we're adding
+        # class path entries with the full path to required JAR files (which
+        # makes sense for JARs in the source directory, but probably doesn't for
+        # JARs in the object directory). This could be a problem because we only
+        # know the contents of (a subdirectory of) libs/ after a successful
+        # build and package, which is after build-backend time. At the cost of
+        # some flexibility, we explicitly copy certain libraries here; if the
+        # libraries aren't present -- namely, when the tree hasn't been packaged
+        # -- this fails. That's by design, to avoid crashes on device caused by
+        # missing native libraries.
+        for src, dst in project.libs:
+            manifest.add_copy(mozpath.join(srcdir, src), dst)
+
+        return manifest
+
+    def _process_android_eclipse_project_data(self, data, srcdir, objdir):
+        # This can't be relative to the environment's topsrcdir,
+        # because during testing topsrcdir is faked.
+        template_directory = os.path.abspath(mozpath.join(os.path.dirname(__file__),
+            'templates', 'android_eclipse'))
+
+        project_directory = mozpath.join(self.environment.topobjdir, 'android_eclipse', data.name)
+        manifest_path = mozpath.join(self.environment.topobjdir, 'android_eclipse', '%s.manifest' % data.name)
+
+        manifest = self._manifest_for_project(srcdir, data)
+        ensureParentDir(manifest_path)
+        manifest.write(path=manifest_path)
+
+        classpathentries = []
+        for cpe in sorted(data._classpathentries, key=lambda x: x.path):
+            e = self._Element_for_classpathentry(cpe)
+            classpathentries.append(ET.tostring(e))
+
+        for name in sorted(data.referenced_projects):
+            e = self._Element_for_referenced_project(name)
+            classpathentries.append(ET.tostring(e))
+
+        for name in sorted(data.extra_jars):
+            e = self._Element_for_extra_jar(mozpath.join(srcdir, name))
+            classpathentries.append(ET.tostring(e))
+
+        defines = {}
+        defines['IDE_OBJDIR'] = objdir
+        defines['IDE_TOPOBJDIR'] = self.environment.topobjdir
+        defines['IDE_SRCDIR'] = srcdir
+        defines['IDE_TOPSRCDIR'] = self.environment.topsrcdir
+        defines['IDE_PROJECT_NAME'] = data.name
+        defines['IDE_PACKAGE_NAME'] = data.package_name
+        defines['IDE_PROJECT_DIRECTORY'] = project_directory
+        defines['IDE_RELSRCDIR'] = mozpath.relpath(srcdir, self.environment.topsrcdir)
+        defines['IDE_CLASSPATH_ENTRIES'] = '\n'.join('\t' + cpe for cpe in classpathentries)
+        defines['IDE_RECURSIVE_MAKE_TARGETS'] = ' '.join(sorted(data.recursive_make_targets))
+        # Like android.library=true
+        defines['IDE_PROJECT_LIBRARY_SETTING'] = 'android.library=true' if data.is_library else ''
+        # Like android.library.reference.1=FennecBrandingResources
+        defines['IDE_PROJECT_LIBRARY_REFERENCES'] = '\n'.join(
+            'android.library.reference.%s=%s' % (i + 1, ref)
+            for i, ref in enumerate(sorted(data.included_projects)))
+        if data.filtered_resources:
+            filteredResources = self._Element_for_filtered_resources(data.filtered_resources)
+            defines['IDE_PROJECT_FILTERED_RESOURCES'] = pretty_print(filteredResources).strip()
+        else:
+            defines['IDE_PROJECT_FILTERED_RESOURCES'] = ''
+        defines['ANDROID_TARGET_SDK'] = self.environment.substs['ANDROID_TARGET_SDK']
+        defines['MOZ_ANDROID_MIN_SDK_VERSION'] = self.environment.defines['MOZ_ANDROID_MIN_SDK_VERSION']
+
+        copier = FileCopier()
+        finder = FileFinder(template_directory)
+        for input_filename, f in itertools.chain(finder.find('**'), finder.find('.**')):
+            if input_filename == 'AndroidManifest.xml' and not data.is_library:
+                # Main projects supply their own manifests.
+                continue
+            copier.add(input_filename, PreprocessedFile(
+                mozpath.join(finder.base, input_filename),
+                depfile_path=None,
+                marker='#',
+                defines=defines,
+                extra_depends={mozpath.join(finder.base, input_filename)}))
+
+        # When we re-create the build backend, we kill everything that was there.
+        if os.path.isdir(project_directory):
+            self._updated_count += 1
+        else:
+            self._created_count += 1
+        copier.copy(project_directory, skip_if_older=False, remove_unaccounted=True)

python/mozbuild/mozbuild/backend/base.py

@@ -0,0 +1,317 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+from abc import (
+    ABCMeta,
+    abstractmethod,
+)
+
+import errno
+import itertools
+import os
+import time
+
+from contextlib import contextmanager
+
+from mach.mixin.logging import LoggingMixin
+
+import mozpack.path as mozpath
+from ..preprocessor import Preprocessor
+from ..pythonutil import iter_modules_in_path
+from ..util import (
+    FileAvoidWrite,
+    simple_diff,
+)
+from ..frontend.data import ContextDerived
+from .configenvironment import ConfigEnvironment
+from mozbuild.base import ExecutionSummary
+
+
+class BuildBackend(LoggingMixin):
+    """Abstract base class for build backends.
+
+    A build backend is merely a consumer of the build configuration (the output
+    of the frontend processing). It does something with said data. What exactly
+    is the discretion of the specific implementation.
+    """
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self, environment):
+        assert isinstance(environment, ConfigEnvironment)
+
+        self.populate_logger()
+
+        self.environment = environment
+
+        # Files whose modification should cause a new read and backend
+        # generation.
+        self.backend_input_files = set()
+
+        # Files generated by the backend.
+        self._backend_output_files = set()
+
+        self._environments = {}
+        self._environments[environment.topobjdir] = environment
+
+        # The number of backend files created.
+        self._created_count = 0
+
+        # The number of backend files updated.
+        self._updated_count = 0
+
+        # The number of unchanged backend files.
+        self._unchanged_count = 0
+
+        # The number of deleted backend files.
+        self._deleted_count = 0
+
+        # The total wall time spent in the backend. This counts the time the
+        # backend writes out files, etc.
+        self._execution_time = 0.0
+
+        # Mapping of changed file paths to diffs of the changes.
+        self.file_diffs = {}
+
+        self.dry_run = False
+
+        self._init()
+
+    def summary(self):
+        return ExecutionSummary(
+            self.__class__.__name__.replace('Backend', '') +
+            ' backend executed in {execution_time:.2f}s\n  '
+            '{total:d} total backend files; '
+            '{created:d} created; '
+            '{updated:d} updated; '
+            '{unchanged:d} unchanged; '
+            '{deleted:d} deleted',
+            execution_time=self._execution_time,
+            total=self._created_count + self._updated_count +
+            self._unchanged_count,
+            created=self._created_count,
+            updated=self._updated_count,
+            unchanged=self._unchanged_count,
+            deleted=self._deleted_count)
+
+    def _init(self):
+        """Hook point for child classes to perform actions during __init__.
+
+        This exists so child classes don't need to implement __init__.
+        """
+
+    def consume(self, objs):
+        """Consume a stream of TreeMetadata instances.
+
+        This is the main method of the interface. This is what takes the
+        frontend output and does something with it.
+
+        Child classes are not expected to implement this method. Instead, the
+        base class consumes objects and calls methods (possibly) implemented by
+        child classes.
+        """
+
+        # Previously generated files.
+        list_file = mozpath.join(self.environment.topobjdir, 'backend.%s'
+                                 % self.__class__.__name__)
+        backend_output_list = set()
+        if os.path.exists(list_file):
+            with open(list_file) as fh:
+                backend_output_list.update(mozpath.normsep(p)
+                                           for p in fh.read().splitlines())
+
+        for obj in objs:
+            obj_start = time.time()
+            if (not self.consume_object(obj) and
+                    not isinstance(self, PartialBackend)):
+                raise Exception('Unhandled object of type %s' % type(obj))
+            self._execution_time += time.time() - obj_start
+
+            if (isinstance(obj, ContextDerived) and
+                    not isinstance(self, PartialBackend)):
+                self.backend_input_files |= obj.context_all_paths
+
+        # Pull in all loaded Python as dependencies so any Python changes that
+        # could influence our output result in a rescan.
+        self.backend_input_files |= set(iter_modules_in_path(
+            self.environment.topsrcdir, self.environment.topobjdir))
+
+        finished_start = time.time()
+        self.consume_finished()
+        self._execution_time += time.time() - finished_start
+
+        # Purge backend files created in previous run, but not created anymore
+        delete_files = backend_output_list - self._backend_output_files
+        for path in delete_files:
+            full_path = mozpath.join(self.environment.topobjdir, path)
+            try:
+                with open(full_path, 'r') as existing:
+                    old_content = existing.read()
+                    if old_content:
+                        self.file_diffs[full_path] = simple_diff(
+                            full_path, old_content.splitlines(), None)
+            except IOError:
+                pass
+            try:
+                if not self.dry_run:
+                    os.unlink(full_path)
+                self._deleted_count += 1
+            except OSError:
+                pass
+        # Remove now empty directories
+        for dir in set(mozpath.dirname(d) for d in delete_files):
+            try:
+                os.removedirs(dir)
+            except OSError:
+                pass
+
+        # Write out the list of backend files generated, if it changed.
+        if self._deleted_count or self._created_count or \
+                not os.path.exists(list_file):
+            with self._write_file(list_file) as fh:
+                fh.write('\n'.join(sorted(self._backend_output_files)))
+        else:
+            # Always update its mtime.
+            with open(list_file, 'a'):
+                os.utime(list_file, None)
+
+        # Write out the list of input files for the backend
+        with self._write_file('%s.in' % list_file) as fh:
+            fh.write('\n'.join(sorted(
+                mozpath.normsep(f) for f in self.backend_input_files)))
+
+    @abstractmethod
+    def consume_object(self, obj):
+        """Consumes an individual TreeMetadata instance.
+
+        This is the main method used by child classes to react to build
+        metadata.
+        """
+
+    def consume_finished(self):
+        """Called when consume() has completed handling all objects."""
+
+    def build(self, config, output, jobs, verbose):
+        """Called when 'mach build' is executed.
+
+        This should return the status value of a subprocess, where 0 denotes
+        success and any other value is an error code. A return value of None
+        indicates that the default 'make -f client.mk' should run.
+        """
+        return None
+
+    @contextmanager
+    def _write_file(self, path=None, fh=None, mode='rU'):
+        """Context manager to write a file.
+
+        This is a glorified wrapper around FileAvoidWrite with integration to
+        update the summary data on this instance.
+
+        Example usage:
+
+            with self._write_file('foo.txt') as fh:
+                fh.write('hello world')
+        """
+
+        if path is not None:
+            assert fh is None
+            fh = FileAvoidWrite(path, capture_diff=True, dry_run=self.dry_run,
+                                mode=mode)
+        else:
+            assert fh is not None
+
+        dirname = mozpath.dirname(fh.name)
+        try:
+            os.makedirs(dirname)
+        except OSError as error:
+            if error.errno != errno.EEXIST:
+                raise
+
+        yield fh
+
+        self._backend_output_files.add(mozpath.relpath(fh.name, self.environment.topobjdir))
+        existed, updated = fh.close()
+        if fh.diff:
+            self.file_diffs[fh.name] = fh.diff
+        if not existed:
+            self._created_count += 1
+        elif updated:
+            self._updated_count += 1
+        else:
+            self._unchanged_count += 1
+
+    @contextmanager
+    def _get_preprocessor(self, obj):
+        '''Returns a preprocessor with a few predefined values depending on
+        the given BaseConfigSubstitution(-like) object, and all the substs
+        in the current environment.'''
+        pp = Preprocessor()
+        srcdir = mozpath.dirname(obj.input_path)
+        pp.context.update({
+            k: ' '.join(v) if isinstance(v, list) else v
+            for k, v in obj.config.substs.iteritems()
+        })
+        pp.context.update(
+            top_srcdir=obj.topsrcdir,
+            topobjdir=obj.topobjdir,
+            srcdir=srcdir,
+            relativesrcdir=mozpath.relpath(srcdir, obj.topsrcdir) or '.',
+            DEPTH=mozpath.relpath(obj.topobjdir, mozpath.dirname(obj.output_path)) or '.',
+        )
+        pp.do_filter('attemptSubstitution')
+        pp.setMarker(None)
+        with self._write_file(obj.output_path) as fh:
+            pp.out = fh
+            yield pp
+
+
+class PartialBackend(BuildBackend):
+    """A PartialBackend is a BuildBackend declaring that its consume_object
+    method may not handle all build configuration objects it's passed, and
+    that it's fine."""
+
+
+def HybridBackend(*backends):
+    """A HybridBackend is the combination of one or more PartialBackends
+    with a non-partial BuildBackend.
+
+    Build configuration objects are passed to each backend, stopping at the
+    first of them that declares having handled them.
+    """
+    assert len(backends) >= 2
+    assert all(issubclass(b, PartialBackend) for b in backends[:-1])
+    assert not(issubclass(backends[-1], PartialBackend))
+    assert all(issubclass(b, BuildBackend) for b in backends)
+
+    class TheHybridBackend(BuildBackend):
+        def __init__(self, environment):
+            self._backends = [b(environment) for b in backends]
+            super(TheHybridBackend, self).__init__(environment)
+
+        def consume_object(self, obj):
+            return any(b.consume_object(obj) for b in self._backends)
+
+        def consume_finished(self):
+            for backend in self._backends:
+                backend.consume_finished()
+
+            for attr in ('_execution_time', '_created_count', '_updated_count',
+                         '_unchanged_count', '_deleted_count'):
+                setattr(self, attr,
+                        sum(getattr(b, attr) for b in self._backends))
+
+            for b in self._backends:
+                self.file_diffs.update(b.file_diffs)
+                for attr in ('backend_input_files', '_backend_output_files'):
+                    files = getattr(self, attr)
+                    files |= getattr(b, attr)
+
+    name = '+'.join(itertools.chain(
+        (b.__name__.replace('Backend', '') for b in backends[:1]),
+        (b.__name__ for b in backends[-1:])
+    ))
+
+    return type(str(name), (TheHybridBackend,), {})

python/mozbuild/mozbuild/backend/common.py

@@ -0,0 +1,567 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import cPickle as pickle
+import itertools
+import json
+import os
+
+import mozpack.path as mozpath
+
+from mozbuild.backend.base import BuildBackend
+
+from mozbuild.frontend.context import (
+    Context,
+    Path,
+    RenamedSourcePath,
+    VARIABLES,
+)
+from mozbuild.frontend.data import (
+    BaseProgram,
+    ChromeManifestEntry,
+    ConfigFileSubstitution,
+    ExampleWebIDLInterface,
+    IPDLFile,
+    FinalTargetPreprocessedFiles,
+    FinalTargetFiles,
+    GeneratedEventWebIDLFile,
+    GeneratedWebIDLFile,
+    PreprocessedTestWebIDLFile,
+    PreprocessedWebIDLFile,
+    SharedLibrary,
+    TestManifest,
+    TestWebIDLFile,
+    UnifiedSources,
+    XPIDLFile,
+    WebIDLFile,
+)
+from mozbuild.jar import (
+    DeprecatedJarManifest,
+    JarManifestParser,
+)
+from mozbuild.preprocessor import Preprocessor
+from mozpack.chrome.manifest import parse_manifest_line
+
+from collections import defaultdict
+
+from mozbuild.util import group_unified_files
+
+class XPIDLManager(object):
+    """Helps manage XPCOM IDLs in the context of the build system."""
+    def __init__(self, config):
+        self.config = config
+        self.topsrcdir = config.topsrcdir
+        self.topobjdir = config.topobjdir
+
+        self.idls = {}
+        self.modules = {}
+        self.interface_manifests = {}
+        self.chrome_manifests = set()
+
+    def register_idl(self, idl, allow_existing=False):
+        """Registers an IDL file with this instance.
+
+        The IDL file will be built, installed, etc.
+        """
+        basename = mozpath.basename(idl.source_path)
+        root = mozpath.splitext(basename)[0]
+        xpt = '%s.xpt' % idl.module
+        manifest = mozpath.join(idl.install_target, 'components', 'interfaces.manifest')
+        chrome_manifest = mozpath.join(idl.install_target, 'chrome.manifest')
+
+        entry = {
+            'source': idl.source_path,
+            'module': idl.module,
+            'basename': basename,
+            'root': root,
+            'manifest': manifest,
+        }
+
+        if not allow_existing and entry['basename'] in self.idls:
+            raise Exception('IDL already registered: %s' % entry['basename'])
+
+        self.idls[entry['basename']] = entry
+        t = self.modules.setdefault(entry['module'], (idl.install_target, set()))
+        t[1].add(entry['root'])
+
+        if idl.add_to_manifest:
+            self.interface_manifests.setdefault(manifest, set()).add(xpt)
+            self.chrome_manifests.add(chrome_manifest)
+
+
+class WebIDLCollection(object):
+    """Collects WebIDL info referenced during the build."""
+
+    def __init__(self):
+        self.sources = set()
+        self.generated_sources = set()
+        self.generated_events_sources = set()
+        self.preprocessed_sources = set()
+        self.test_sources = set()
+        self.preprocessed_test_sources = set()
+        self.example_interfaces = set()
+
+    def all_regular_sources(self):
+        return self.sources | self.generated_sources | \
+            self.generated_events_sources | self.preprocessed_sources
+
+    def all_regular_basenames(self):
+        return [os.path.basename(source) for source in self.all_regular_sources()]
+
+    def all_regular_stems(self):
+        return [os.path.splitext(b)[0] for b in self.all_regular_basenames()]
+
+    def all_regular_bindinggen_stems(self):
+        for stem in self.all_regular_stems():
+            yield '%sBinding' % stem
+
+        for source in self.generated_events_sources:
+            yield os.path.splitext(os.path.basename(source))[0]
+
+    def all_regular_cpp_basenames(self):
+        for stem in self.all_regular_bindinggen_stems():
+            yield '%s.cpp' % stem
+
+    def all_test_sources(self):
+        return self.test_sources | self.preprocessed_test_sources
+
+    def all_test_basenames(self):
+        return [os.path.basename(source) for source in self.all_test_sources()]
+
+    def all_test_stems(self):
+        return [os.path.splitext(b)[0] for b in self.all_test_basenames()]
+
+    def all_test_cpp_basenames(self):
+        return ['%sBinding.cpp' % s for s in self.all_test_stems()]
+
+    def all_static_sources(self):
+        return self.sources | self.generated_events_sources | \
+            self.test_sources
+
+    def all_non_static_sources(self):
+        return self.generated_sources | self.all_preprocessed_sources()
+
+    def all_non_static_basenames(self):
+        return [os.path.basename(s) for s in self.all_non_static_sources()]
+
+    def all_preprocessed_sources(self):
+        return self.preprocessed_sources | self.preprocessed_test_sources
+
+    def all_sources(self):
+        return set(self.all_regular_sources()) | set(self.all_test_sources())
+
+    def all_basenames(self):
+        return [os.path.basename(source) for source in self.all_sources()]
+
+    def all_stems(self):
+        return [os.path.splitext(b)[0] for b in self.all_basenames()]
+
+    def generated_events_basenames(self):
+        return [os.path.basename(s) for s in self.generated_events_sources]
+
+    def generated_events_stems(self):
+        return [os.path.splitext(b)[0] for b in self.generated_events_basenames()]
+
+
+class TestManager(object):
+    """Helps hold state related to tests."""
+
+    def __init__(self, config):
+        self.config = config
+        self.topsrcdir = mozpath.normpath(config.topsrcdir)
+
+        self.tests_by_path = defaultdict(list)
+        self.installs_by_path = defaultdict(list)
+        self.deferred_installs = set()
+        self.manifest_defaults = {}
+
+    def add(self, t, flavor, topsrcdir):
+        t = dict(t)
+        t['flavor'] = flavor
+
+        path = mozpath.normpath(t['path'])
+        assert mozpath.basedir(path, [topsrcdir])
+
+        key = path[len(topsrcdir)+1:]
+        t['file_relpath'] = key
+        t['dir_relpath'] = mozpath.dirname(key)
+
+        self.tests_by_path[key].append(t)
+
+    def add_defaults(self, manifest):
+        if not hasattr(manifest, 'manifest_defaults'):
+            return
+        for sub_manifest, defaults in manifest.manifest_defaults.items():
+            self.manifest_defaults[sub_manifest] = defaults
+
+    def add_installs(self, obj, topsrcdir):
+        for src, (dest, _) in obj.installs.iteritems():
+            key = src[len(topsrcdir)+1:]
+            self.installs_by_path[key].append((src, dest))
+        for src, pat, dest in obj.pattern_installs:
+            key = mozpath.join(src[len(topsrcdir)+1:], pat)
+            self.installs_by_path[key].append((src, pat, dest))
+        for path in obj.deferred_installs:
+            self.deferred_installs.add(path[2:])
+
+
+class BinariesCollection(object):
+    """Tracks state of binaries produced by the build."""
+
+    def __init__(self):
+        self.shared_libraries = []
+        self.programs = []
+
+
+class CommonBackend(BuildBackend):
+    """Holds logic common to all build backends."""
+
+    def _init(self):
+        self._idl_manager = XPIDLManager(self.environment)
+        self._test_manager = TestManager(self.environment)
+        self._webidls = WebIDLCollection()
+        self._binaries = BinariesCollection()
+        self._configs = set()
+        self._ipdl_sources = set()
+
+    def consume_object(self, obj):
+        self._configs.add(obj.config)
+
+        if isinstance(obj, TestManifest):
+            for test in obj.tests:
+                self._test_manager.add(test, obj.flavor, obj.topsrcdir)
+            self._test_manager.add_defaults(obj.manifest)
+            self._test_manager.add_installs(obj, obj.topsrcdir)
+
+        elif isinstance(obj, XPIDLFile):
+            # TODO bug 1240134 tracks not processing XPIDL files during
+            # artifact builds.
+            self._idl_manager.register_idl(obj)
+
+        elif isinstance(obj, ConfigFileSubstitution):
+            # Do not handle ConfigFileSubstitution for Makefiles. Leave that
+            # to other
+            if mozpath.basename(obj.output_path) == 'Makefile':
+                return False
+            with self._get_preprocessor(obj) as pp:
+                pp.do_include(obj.input_path)
+            self.backend_input_files.add(obj.input_path)
+
+        # We should consider aggregating WebIDL types in emitter.py.
+        elif isinstance(obj, WebIDLFile):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.sources.add(mozpath.join(obj.srcdir, obj.basename))
+
+        elif isinstance(obj, GeneratedEventWebIDLFile):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.generated_events_sources.add(mozpath.join(
+                obj.srcdir, obj.basename))
+
+        elif isinstance(obj, TestWebIDLFile):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.test_sources.add(mozpath.join(obj.srcdir,
+                obj.basename))
+
+        elif isinstance(obj, PreprocessedTestWebIDLFile):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.preprocessed_test_sources.add(mozpath.join(
+                obj.srcdir, obj.basename))
+
+        elif isinstance(obj, GeneratedWebIDLFile):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.generated_sources.add(mozpath.join(obj.srcdir,
+                obj.basename))
+
+        elif isinstance(obj, PreprocessedWebIDLFile):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.preprocessed_sources.add(mozpath.join(
+                obj.srcdir, obj.basename))
+
+        elif isinstance(obj, ExampleWebIDLInterface):
+            # WebIDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._webidls.example_interfaces.add(obj.name)
+
+        elif isinstance(obj, IPDLFile):
+            # IPDL isn't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            self._ipdl_sources.add(mozpath.join(obj.srcdir, obj.basename))
+
+        elif isinstance(obj, UnifiedSources):
+            # Unified sources aren't relevant to artifact builds.
+            if self.environment.is_artifact_build:
+                return True
+
+            if obj.have_unified_mapping:
+                self._write_unified_files(obj.unified_source_mapping, obj.objdir)
+            if hasattr(self, '_process_unified_sources'):
+                self._process_unified_sources(obj)
+
+        elif isinstance(obj, BaseProgram):
+            self._binaries.programs.append(obj)
+            return False
+
+        elif isinstance(obj, SharedLibrary):
+            self._binaries.shared_libraries.append(obj)
+            return False
+
+        else:
+            return False
+
+        return True
+
+    def consume_finished(self):
+        if len(self._idl_manager.idls):
+            self._handle_idl_manager(self._idl_manager)
+
+        self._handle_webidl_collection(self._webidls)
+
+        sorted_ipdl_sources = list(sorted(self._ipdl_sources))
+
+        def files_from(ipdl):
+            base = mozpath.basename(ipdl)
+            root, ext = mozpath.splitext(base)
+
+            # Both .ipdl and .ipdlh become .cpp files
+            files = ['%s.cpp' % root]
+            if ext == '.ipdl':
+                # .ipdl also becomes Child/Parent.cpp files
+                files.extend(['%sChild.cpp' % root,
+                              '%sParent.cpp' % root])
+            return files
+
+        ipdl_dir = mozpath.join(self.environment.topobjdir, 'ipc', 'ipdl')
+
+        ipdl_cppsrcs = list(itertools.chain(*[files_from(p) for p in sorted_ipdl_sources]))
+        unified_source_mapping = list(group_unified_files(ipdl_cppsrcs,
+                                                          unified_prefix='UnifiedProtocols',
+                                                          unified_suffix='cpp',
+                                                          files_per_unified_file=16))
+
+        self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
+        self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
+
+        for config in self._configs:
+            self.backend_input_files.add(config.source)
+
+        # Write out a machine-readable file describing every test.
+        topobjdir = self.environment.topobjdir
+        with self._write_file(mozpath.join(topobjdir, 'all-tests.pkl'), mode='rb') as fh:
+            pickle.dump(dict(self._test_manager.tests_by_path), fh, protocol=2)
+
+        with self._write_file(mozpath.join(topobjdir, 'test-defaults.pkl'), mode='rb') as fh:
+            pickle.dump(self._test_manager.manifest_defaults, fh, protocol=2)
+
+        path = mozpath.join(self.environment.topobjdir, 'test-installs.pkl')
+        with self._write_file(path, mode='rb') as fh:
+            pickle.dump({k: v for k, v in self._test_manager.installs_by_path.items()
+                         if k in self._test_manager.deferred_installs},
+                        fh,
+                        protocol=2)
+
+        # Write out a machine-readable file describing binaries.
+        with self._write_file(mozpath.join(topobjdir, 'binaries.json')) as fh:
+            d = {
+                'shared_libraries': [s.to_dict() for s in self._binaries.shared_libraries],
+                'programs': [p.to_dict() for p in self._binaries.programs],
+            }
+            json.dump(d, fh, sort_keys=True, indent=4)
+
+    def _handle_webidl_collection(self, webidls):
+        if not webidls.all_stems():
+            return
+
+        bindings_dir = mozpath.join(self.environment.topobjdir, 'dom', 'bindings')
+
+        all_inputs = set(webidls.all_static_sources())
+        for s in webidls.all_non_static_basenames():
+            all_inputs.add(mozpath.join(bindings_dir, s))
+
+        generated_events_stems = webidls.generated_events_stems()
+        exported_stems = webidls.all_regular_stems()
+
+        # The WebIDL manager reads configuration from a JSON file. So, we
+        # need to write this file early.
+        o = dict(
+            webidls=sorted(all_inputs),
+            generated_events_stems=sorted(generated_events_stems),
+            exported_stems=sorted(exported_stems),
+            example_interfaces=sorted(webidls.example_interfaces),
+        )
+
+        file_lists = mozpath.join(bindings_dir, 'file-lists.json')
+        with self._write_file(file_lists) as fh:
+            json.dump(o, fh, sort_keys=True, indent=2)
+
+        import mozwebidlcodegen
+
+        manager = mozwebidlcodegen.create_build_system_manager(
+            self.environment.topsrcdir,
+            self.environment.topobjdir,
+            mozpath.join(self.environment.topobjdir, 'dist')
+        )
+
+        # Bindings are compiled in unified mode to speed up compilation and
+        # to reduce linker memory size. Note that test bindings are separated
+        # from regular ones so tests bindings aren't shipped.
+        unified_source_mapping = list(group_unified_files(webidls.all_regular_cpp_basenames(),
+                                                          unified_prefix='UnifiedBindings',
+                                                          unified_suffix='cpp',
+                                                          files_per_unified_file=32))
+        self._write_unified_files(unified_source_mapping, bindings_dir,
+                                  poison_windows_h=True)
+        self._handle_webidl_build(bindings_dir, unified_source_mapping,
+                                  webidls,
+                                  manager.expected_build_output_files(),
+                                  manager.GLOBAL_DEFINE_FILES)
+
+    def _write_unified_file(self, unified_file, source_filenames,
+                            output_directory, poison_windows_h=False):
+        with self._write_file(mozpath.join(output_directory, unified_file)) as f:
+            f.write('#define MOZ_UNIFIED_BUILD\n')
+            includeTemplate = '#include "%(cppfile)s"'
+            if poison_windows_h:
+                includeTemplate += (
+                    '\n'
+                    '#ifdef _WINDOWS_\n'
+                    '#error "%(cppfile)s included windows.h"\n'
+                    "#endif")
+            includeTemplate += (
+                '\n'
+                '#ifdef PL_ARENA_CONST_ALIGN_MASK\n'
+                '#error "%(cppfile)s uses PL_ARENA_CONST_ALIGN_MASK, '
+                'so it cannot be built in unified mode."\n'
+                '#undef PL_ARENA_CONST_ALIGN_MASK\n'
+                '#endif\n'
+                '#ifdef INITGUID\n'
+                '#error "%(cppfile)s defines INITGUID, '
+                'so it cannot be built in unified mode."\n'
+                '#undef INITGUID\n'
+                '#endif')
+            f.write('\n'.join(includeTemplate % { "cppfile": s } for
+                              s in source_filenames))
+
+    def _write_unified_files(self, unified_source_mapping, output_directory,
+                             poison_windows_h=False):
+        for unified_file, source_filenames in unified_source_mapping:
+            self._write_unified_file(unified_file, source_filenames,
+                                     output_directory, poison_windows_h)
+
+    def _consume_jar_manifest(self, obj):
+        # Ideally, this would all be handled somehow in the emitter, but
+        # this would require all the magic surrounding l10n and addons in
+        # the recursive make backend to die, which is not going to happen
+        # any time soon enough.
+        # Notably missing:
+        # - DEFINES from config/config.mk
+        # - L10n support
+        # - The equivalent of -e when USE_EXTENSION_MANIFEST is set in
+        #   moz.build, but it doesn't matter in dist/bin.
+        pp = Preprocessor()
+        if obj.defines:
+            pp.context.update(obj.defines.defines)
+        pp.context.update(self.environment.defines)
+        pp.context.update(
+            AB_CD='en-US',
+            BUILD_FASTER=1,
+        )
+        pp.out = JarManifestParser()
+        try:
+            pp.do_include(obj.path.full_path)
+        except DeprecatedJarManifest as e:
+            raise DeprecatedJarManifest('Parsing error while processing %s: %s'
+                                        % (obj.path.full_path, e.message))
+        self.backend_input_files |= pp.includes
+
+        for jarinfo in pp.out:
+            jar_context = Context(
+                allowed_variables=VARIABLES, config=obj._context.config)
+            jar_context.push_source(obj._context.main_path)
+            jar_context.push_source(obj.path.full_path)
+
+            install_target = obj.install_target
+            if jarinfo.base:
+                install_target = mozpath.normpath(
+                    mozpath.join(install_target, jarinfo.base))
+            jar_context['FINAL_TARGET'] = install_target
+            if obj.defines:
+                jar_context['DEFINES'] = obj.defines.defines
+            files = jar_context['FINAL_TARGET_FILES']
+            files_pp = jar_context['FINAL_TARGET_PP_FILES']
+
+            for e in jarinfo.entries:
+                if e.is_locale:
+                    if jarinfo.relativesrcdir:
+                        src = '/%s' % jarinfo.relativesrcdir
+                    else:
+                        src = ''
+                    src = mozpath.join(src, 'en-US', e.source)
+                else:
+                    src = e.source
+
+                src = Path(jar_context, src)
+
+                if '*' not in e.source and not os.path.exists(src.full_path):
+                    if e.is_locale:
+                        raise Exception(
+                            '%s: Cannot find %s' % (obj.path, e.source))
+                    if e.source.startswith('/'):
+                        src = Path(jar_context, '!' + e.source)
+                    else:
+                        # This actually gets awkward if the jar.mn is not
+                        # in the same directory as the moz.build declaring
+                        # it, but it's how it works in the recursive make,
+                        # not that anything relies on that, but it's simpler.
+                        src = Path(obj._context, '!' + e.source)
+
+                output_basename = mozpath.basename(e.output)
+                if output_basename != src.target_basename:
+                    src = RenamedSourcePath(jar_context,
+                                            (src, output_basename))
+                path = mozpath.dirname(mozpath.join(jarinfo.name, e.output))
+
+                if e.preprocess:
+                    if '*' in e.source:
+                        raise Exception('%s: Wildcards are not supported with '
+                                        'preprocessing' % obj.path)
+                    files_pp[path] += [src]
+                else:
+                    files[path] += [src]
+
+            if files:
+                self.consume_object(FinalTargetFiles(jar_context, files))
+            if files_pp:
+                self.consume_object(
+                    FinalTargetPreprocessedFiles(jar_context, files_pp))
+
+            for m in jarinfo.chrome_manifests:
+                entry = parse_manifest_line(
+                    mozpath.dirname(jarinfo.name),
+                    m.replace('%', mozpath.basename(jarinfo.name) + '/'))
+                self.consume_object(ChromeManifestEntry(
+                    jar_context, '%s.manifest' % jarinfo.name, entry))

python/mozbuild/mozbuild/backend/configenvironment.py

@@ -0,0 +1,199 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import os
+import sys
+
+from collections import Iterable
+from types import StringTypes, ModuleType
+
+import mozpack.path as mozpath
+
+from mozbuild.util import ReadOnlyDict
+from mozbuild.shellutil import quote as shell_quote
+
+
+if sys.version_info.major == 2:
+    text_type = unicode
+else:
+    text_type = str
+
+
+class BuildConfig(object):
+    """Represents the output of configure."""
+
+    _CODE_CACHE = {}
+
+    def __init__(self):
+        self.topsrcdir = None
+        self.topobjdir = None
+        self.defines = {}
+        self.non_global_defines = []
+        self.substs = {}
+        self.files = []
+        self.mozconfig = None
+
+    @classmethod
+    def from_config_status(cls, path):
+        """Create an instance from a config.status file."""
+        code_cache = cls._CODE_CACHE
+        mtime = os.path.getmtime(path)
+
+        # cache the compiled code as it can be reused
+        # we cache it the first time, or if the file changed
+        if not path in code_cache or code_cache[path][0] != mtime:
+            # Add config.status manually to sys.modules so it gets picked up by
+            # iter_modules_in_path() for automatic dependencies.
+            mod = ModuleType('config.status')
+            mod.__file__ = path
+            sys.modules['config.status'] = mod
+
+            with open(path, 'rt') as fh:
+                source = fh.read()
+                code_cache[path] = (
+                    mtime,
+                    compile(source, path, 'exec', dont_inherit=1)
+                )
+
+        g = {
+            '__builtins__': __builtins__,
+            '__file__': path,
+        }
+        l = {}
+        exec(code_cache[path][1], g, l)
+
+        config = BuildConfig()
+
+        for name in l['__all__']:
+            setattr(config, name, l[name])
+
+        return config
+
+
+class ConfigEnvironment(object):
+    """Perform actions associated with a configured but bare objdir.
+
+    The purpose of this class is to preprocess files from the source directory
+    and output results in the object directory.
+
+    There are two types of files: config files and config headers,
+    each treated through a different member function.
+
+    Creating a ConfigEnvironment requires a few arguments:
+      - topsrcdir and topobjdir are, respectively, the top source and
+        the top object directory.
+      - defines is a dict filled from AC_DEFINE and AC_DEFINE_UNQUOTED in
+        autoconf.
+      - non_global_defines are a list of names appearing in defines above
+        that are not meant to be exported in ACDEFINES (see below)
+      - substs is a dict filled from AC_SUBST in autoconf.
+
+    ConfigEnvironment automatically defines one additional substs variable
+    from all the defines not appearing in non_global_defines:
+      - ACDEFINES contains the defines in the form -DNAME=VALUE, for use on
+        preprocessor command lines. The order in which defines were given
+        when creating the ConfigEnvironment is preserved.
+    and two other additional subst variables from all the other substs:
+      - ALLSUBSTS contains the substs in the form NAME = VALUE, in sorted
+        order, for use in autoconf.mk. It includes ACDEFINES
+        Only substs with a VALUE are included, such that the resulting file
+        doesn't change when new empty substs are added.
+        This results in less invalidation of build dependencies in the case
+        of autoconf.mk..
+      - ALLEMPTYSUBSTS contains the substs with an empty value, in the form
+        NAME =.
+
+    ConfigEnvironment expects a "top_srcdir" subst to be set with the top
+    source directory, in msys format on windows. It is used to derive a
+    "srcdir" subst when treating config files. It can either be an absolute
+    path or a path relative to the topobjdir.
+    """
+
+    def __init__(self, topsrcdir, topobjdir, defines=None,
+        non_global_defines=None, substs=None, source=None, mozconfig=None):
+
+        if not source:
+            source = mozpath.join(topobjdir, 'config.status')
+        self.source = source
+        self.defines = ReadOnlyDict(defines or {})
+        self.non_global_defines = non_global_defines or []
+        self.substs = dict(substs or {})
+        self.topsrcdir = mozpath.abspath(topsrcdir)
+        self.topobjdir = mozpath.abspath(topobjdir)
+        self.mozconfig = mozpath.abspath(mozconfig) if mozconfig else None
+        self.lib_prefix = self.substs.get('LIB_PREFIX', '')
+        if 'LIB_SUFFIX' in self.substs:
+            self.lib_suffix = '.%s' % self.substs['LIB_SUFFIX']
+        self.dll_prefix = self.substs.get('DLL_PREFIX', '')
+        self.dll_suffix = self.substs.get('DLL_SUFFIX', '')
+        if self.substs.get('IMPORT_LIB_SUFFIX'):
+            self.import_prefix = self.lib_prefix
+            self.import_suffix = '.%s' % self.substs['IMPORT_LIB_SUFFIX']
+        else:
+            self.import_prefix = self.dll_prefix
+            self.import_suffix = self.dll_suffix
+
+        global_defines = [name for name in self.defines
+            if not name in self.non_global_defines]
+        self.substs['ACDEFINES'] = ' '.join(['-D%s=%s' % (name,
+            shell_quote(self.defines[name]).replace('$', '$$'))
+            for name in sorted(global_defines)])
+        def serialize(obj):
+            if isinstance(obj, StringTypes):
+                return obj
+            if isinstance(obj, Iterable):
+                return ' '.join(obj)
+            raise Exception('Unhandled type %s', type(obj))
+        self.substs['ALLSUBSTS'] = '\n'.join(sorted(['%s = %s' % (name,
+            serialize(self.substs[name])) for name in self.substs if self.substs[name]]))
+        self.substs['ALLEMPTYSUBSTS'] = '\n'.join(sorted(['%s =' % name
+            for name in self.substs if not self.substs[name]]))
+
+        self.substs = ReadOnlyDict(self.substs)
+
+        self.external_source_dir = None
+        external = self.substs.get('EXTERNAL_SOURCE_DIR', '')
+        if external:
+            external = mozpath.normpath(external)
+            if not os.path.isabs(external):
+                external = mozpath.join(self.topsrcdir, external)
+            self.external_source_dir = mozpath.normpath(external)
+
+        # Populate a Unicode version of substs. This is an optimization to make
+        # moz.build reading faster, since each sandbox needs a Unicode version
+        # of these variables and doing it over a thousand times is a hotspot
+        # during sandbox execution!
+        # Bug 844509 tracks moving everything to Unicode.
+        self.substs_unicode = {}
+
+        def decode(v):
+            if not isinstance(v, text_type):
+                try:
+                    return v.decode('utf-8')
+                except UnicodeDecodeError:
+                    return v.decode('utf-8', 'replace')
+
+        for k, v in self.substs.items():
+            if not isinstance(v, StringTypes):
+                if isinstance(v, Iterable):
+                    type(v)(decode(i) for i in v)
+            elif not isinstance(v, text_type):
+                v = decode(v)
+
+            self.substs_unicode[k] = v
+
+        self.substs_unicode = ReadOnlyDict(self.substs_unicode)
+
+    @property
+    def is_artifact_build(self):
+        return self.substs.get('MOZ_ARTIFACT_BUILDS', False)
+
+    @staticmethod
+    def from_config_status(path):
+        config = BuildConfig.from_config_status(path)
+
+        return ConfigEnvironment(config.topsrcdir, config.topobjdir,
+            config.defines, config.non_global_defines, config.substs, path)

python/mozbuild/mozbuild/backend/cpp_eclipse.py

@@ -0,0 +1,698 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import errno
+import random
+import os
+import subprocess
+import types
+import xml.etree.ElementTree as ET
+from .common import CommonBackend
+
+from ..frontend.data import (
+    Defines,
+)
+from mozbuild.base import ExecutionSummary
+
+# TODO Have ./mach eclipse generate the workspace and index it:
+# /Users/bgirard/mozilla/eclipse/eclipse/eclipse/eclipse -application org.eclipse.cdt.managedbuilder.core.headlessbuild -data $PWD/workspace -importAll $PWD/eclipse
+# Open eclipse:
+# /Users/bgirard/mozilla/eclipse/eclipse/eclipse/eclipse -data $PWD/workspace
+
+class CppEclipseBackend(CommonBackend):
+    """Backend that generates Cpp Eclipse project files.
+    """
+
+    def __init__(self, environment):
+        if os.name == 'nt':
+            raise Exception('Eclipse is not supported on Windows. '
+                            'Consider using Visual Studio instead.')
+        super(CppEclipseBackend, self).__init__(environment)
+
+    def _init(self):
+        CommonBackend._init(self)
+
+        self._paths_to_defines = {}
+        self._project_name = 'Gecko'
+        self._workspace_dir = self._get_workspace_path()
+        self._project_dir = os.path.join(self._workspace_dir, self._project_name)
+        self._overwriting_workspace = os.path.isdir(self._workspace_dir)
+
+        self._macbundle = self.environment.substs['MOZ_MACBUNDLE_NAME']
+        self._appname = self.environment.substs['MOZ_APP_NAME']
+        self._bin_suffix = self.environment.substs['BIN_SUFFIX']
+        self._cxx = self.environment.substs['CXX']
+        # Note: We need the C Pre Processor (CPP) flags, not the CXX flags
+        self._cppflags = self.environment.substs.get('CPPFLAGS', '')
+
+    def summary(self):
+        return ExecutionSummary(
+            'CppEclipse backend executed in {execution_time:.2f}s\n'
+            'Generated Cpp Eclipse workspace in "{workspace:s}".\n'
+            'If missing, import the project using File > Import > General > Existing Project into workspace\n'
+            '\n'
+            'Run with: eclipse -data {workspace:s}\n',
+            execution_time=self._execution_time,
+            workspace=self._workspace_dir)
+
+    def _get_workspace_path(self):
+        return CppEclipseBackend.get_workspace_path(self.environment.topsrcdir, self.environment.topobjdir)
+
+    @staticmethod
+    def get_workspace_path(topsrcdir, topobjdir):
+        # Eclipse doesn't support having the workspace inside the srcdir.
+        # Since most people have their objdir inside their srcdir it's easier
+        # and more consistent to just put the workspace along side the srcdir
+        srcdir_parent = os.path.dirname(topsrcdir)
+        workspace_dirname = "eclipse_" + os.path.basename(topobjdir)
+        return os.path.join(srcdir_parent, workspace_dirname)
+
+    def consume_object(self, obj):
+        reldir = getattr(obj, 'relativedir', None)
+
+        # Note that unlike VS, Eclipse' indexer seem to crawl the headers and
+        # isn't picky about the local includes.
+        if isinstance(obj, Defines):
+            self._paths_to_defines.setdefault(reldir, {}).update(obj.defines)
+
+        return True
+
+    def consume_finished(self):
+        settings_dir = os.path.join(self._project_dir, '.settings')
+        launch_dir = os.path.join(self._project_dir, 'RunConfigurations')
+        workspace_settings_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.core.runtime/.settings')
+        workspace_language_dir = os.path.join(self._workspace_dir, '.metadata/.plugins/org.eclipse.cdt.core')
+
+        for dir_name in [self._project_dir, settings_dir, launch_dir, workspace_settings_dir, workspace_language_dir]:
+            try:
+                os.makedirs(dir_name)
+            except OSError as e:
+                if e.errno != errno.EEXIST:
+                    raise
+
+        project_path = os.path.join(self._project_dir, '.project')
+        with open(project_path, 'wb') as fh:
+            self._write_project(fh)
+
+        cproject_path = os.path.join(self._project_dir, '.cproject')
+        with open(cproject_path, 'wb') as fh:
+            self._write_cproject(fh)
+
+        language_path = os.path.join(settings_dir, 'language.settings.xml')
+        with open(language_path, 'wb') as fh:
+            self._write_language_settings(fh)
+
+        workspace_language_path = os.path.join(workspace_language_dir, 'language.settings.xml')
+        with open(workspace_language_path, 'wb') as fh:
+            workspace_lang_settings = WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE
+            workspace_lang_settings = workspace_lang_settings.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags);
+            fh.write(workspace_lang_settings)
+
+        self._write_launch_files(launch_dir)
+
+        # This will show up as an 'unmanged' formatter. This can be named by generating
+        # another file.
+        formatter_prefs_path = os.path.join(settings_dir, 'org.eclipse.cdt.core.prefs')
+        with open(formatter_prefs_path, 'wb') as fh:
+            fh.write(FORMATTER_SETTINGS);
+
+        editor_prefs_path = os.path.join(workspace_settings_dir, "org.eclipse.ui.editors.prefs");
+        with open(editor_prefs_path, 'wb') as fh:
+            fh.write(EDITOR_SETTINGS);
+
+        # Now import the project into the workspace
+        self._import_project()
+
+    def _import_project(self):
+        # If the workspace already exists then don't import the project again because
+        # eclipse doesn't handle this properly
+        if self._overwriting_workspace:
+            return
+
+        # We disable the indexer otherwise we're forced to index
+        # the whole codebase when importing the project. Indexing the project can take 20 minutes.
+        self._write_noindex()
+
+        try:
+            process = subprocess.check_call(
+                             ["eclipse", "-application", "-nosplash",
+                              "org.eclipse.cdt.managedbuilder.core.headlessbuild",
+                              "-data", self._workspace_dir, "-importAll", self._project_dir])
+        finally:
+            self._remove_noindex()
+
+    def _write_noindex(self):
+        noindex_path = os.path.join(self._project_dir, '.settings/org.eclipse.cdt.core.prefs')
+        with open(noindex_path, 'wb') as fh:
+            fh.write(NOINDEX_TEMPLATE);
+
+    def _remove_noindex(self):
+        noindex_path = os.path.join(self._project_dir, '.settings/org.eclipse.cdt.core.prefs')
+        os.remove(noindex_path)
+
+    def _define_entry(self, name, value):
+        define = ET.Element('entry')
+        define.set('kind', 'macro')
+        define.set('name', name)
+        define.set('value', value)
+        return ET.tostring(define)
+
+    def _write_language_settings(self, fh):
+        settings = LANGUAGE_SETTINGS_TEMPLATE
+
+        settings = settings.replace('@GLOBAL_INCLUDE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include'))
+        settings = settings.replace('@NSPR_INCLUDE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include/nspr'))
+        settings = settings.replace('@IPDL_INCLUDE_PATH@', os.path.join(self.environment.topobjdir, 'ipc/ipdl/_ipdlheaders'))
+        settings = settings.replace('@PREINCLUDE_FILE_PATH@', os.path.join(self.environment.topobjdir, 'dist/include/mozilla-config.h'))
+        settings = settings.replace('@DEFINE_MOZILLA_INTERNAL_API@', self._define_entry('MOZILLA_INTERNAL_API', '1'))
+        settings = settings.replace("@COMPILER_FLAGS@", self._cxx + " " + self._cppflags);
+
+        fh.write(settings)
+
+    def _write_launch_files(self, launch_dir):
+        bin_dir = os.path.join(self.environment.topobjdir, 'dist')
+
+        # TODO Improve binary detection
+        if self._macbundle:
+            exe_path = os.path.join(bin_dir, self._macbundle, 'Contents/MacOS')
+        else:
+            exe_path = os.path.join(bin_dir, 'bin')
+
+        exe_path = os.path.join(exe_path, self._appname + self._bin_suffix)
+
+        if self.environment.substs['MOZ_WIDGET_TOOLKIT'] != 'gonk':
+            main_gecko_launch = os.path.join(launch_dir, 'gecko.launch')
+            with open(main_gecko_launch, 'wb') as fh:
+                launch = GECKO_LAUNCH_CONFIG_TEMPLATE
+                launch = launch.replace('@LAUNCH_PROGRAM@', exe_path)
+                launch = launch.replace('@LAUNCH_ARGS@', '-P -no-remote')
+                fh.write(launch)
+
+        if self.environment.substs['MOZ_WIDGET_TOOLKIT'] == 'gonk':
+            b2g_flash = os.path.join(launch_dir, 'b2g-flash.launch')
+            with open(b2g_flash, 'wb') as fh:
+                # We assume that the srcdir is inside the b2g tree.
+                # If that's not the case the user can always adjust the path
+                # from the eclipse IDE.
+                fastxul_path = os.path.join(self.environment.topsrcdir, '..', 'scripts', 'fastxul.sh')
+                launch = B2GFLASH_LAUNCH_CONFIG_TEMPLATE
+                launch = launch.replace('@LAUNCH_PROGRAM@', fastxul_path)
+                launch = launch.replace('@OBJDIR@', self.environment.topobjdir)
+                fh.write(launch)
+
+        #TODO Add more launch configs (and delegate calls to mach)
+
+    def _write_project(self, fh):
+        project = PROJECT_TEMPLATE;
+
+        project = project.replace('@PROJECT_NAME@', self._project_name)
+        project = project.replace('@PROJECT_TOPSRCDIR@', self.environment.topsrcdir)
+        fh.write(project)
+
+    def _write_cproject(self, fh):
+        cproject_header = CPROJECT_TEMPLATE_HEADER
+        cproject_header = cproject_header.replace('@PROJECT_TOPSRCDIR@', self.environment.topobjdir)
+        cproject_header = cproject_header.replace('@MACH_COMMAND@', os.path.join(self.environment.topsrcdir, 'mach'))
+        fh.write(cproject_header)
+
+        for path, defines in self._paths_to_defines.items():
+            folderinfo = CPROJECT_TEMPLATE_FOLDER_INFO_HEADER
+            folderinfo = folderinfo.replace('@FOLDER_ID@', str(random.randint(1000000, 99999999999)))
+            folderinfo = folderinfo.replace('@FOLDER_NAME@', 'tree/' + path)
+            fh.write(folderinfo)
+            for k, v in defines.items():
+                define = ET.Element('listOptionValue')
+                define.set('builtIn', 'false')
+                define.set('value', str(k) + "=" + str(v))
+                fh.write(ET.tostring(define))
+            fh.write(CPROJECT_TEMPLATE_FOLDER_INFO_FOOTER)
+
+
+        fh.write(CPROJECT_TEMPLATE_FOOTER)
+
+
+PROJECT_TEMPLATE = """<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+        <name>@PROJECT_NAME@</name>
+        <comment></comment>
+        <projects>
+        </projects>
+        <buildSpec>
+                <buildCommand>
+                        <name>org.eclipse.cdt.managedbuilder.core.genmakebuilder</name>
+                        <triggers>clean,full,incremental,</triggers>
+                        <arguments>
+                        </arguments>
+                </buildCommand>
+                <buildCommand>
+                        <name>org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder</name>
+                        <triggers></triggers>
+			<arguments>
+			</arguments>
+                </buildCommand>
+        </buildSpec>
+        <natures>
+                <nature>org.eclipse.cdt.core.cnature</nature>
+                <nature>org.eclipse.cdt.core.ccnature</nature>
+                <nature>org.eclipse.cdt.managedbuilder.core.managedBuildNature</nature>
+                <nature>org.eclipse.cdt.managedbuilder.core.ScannerConfigNature</nature>
+        </natures>
+        <linkedResources>
+                <link>
+                        <name>tree</name>
+                        <type>2</type>
+                        <location>@PROJECT_TOPSRCDIR@</location>
+                </link>
+        </linkedResources>
+        <filteredResources>
+                <filter>
+                        <id>17111971</id>
+                        <name>tree</name>
+                        <type>30</type>
+                        <matcher>
+                                <id>org.eclipse.ui.ide.multiFilter</id>
+                                <arguments>1.0-name-matches-false-false-obj-*</arguments>
+                        </matcher>
+                </filter>
+                <filter>
+                        <id>14081994</id>
+                        <name>tree</name>
+                        <type>22</type>
+                        <matcher>
+                                <id>org.eclipse.ui.ide.multiFilter</id>
+                                <arguments>1.0-name-matches-false-false-*.rej</arguments>
+                        </matcher>
+                </filter>
+                <filter>
+                        <id>25121970</id>
+                        <name>tree</name>
+                        <type>22</type>
+                        <matcher>
+                                <id>org.eclipse.ui.ide.multiFilter</id>
+                                <arguments>1.0-name-matches-false-false-*.orig</arguments>
+                        </matcher>
+                </filter>
+                <filter>
+                        <id>10102004</id>
+                        <name>tree</name>
+                        <type>10</type>
+                        <matcher>
+                                <id>org.eclipse.ui.ide.multiFilter</id>
+                                <arguments>1.0-name-matches-false-false-.hg</arguments>
+                        </matcher>
+                </filter>
+                <filter>
+                        <id>23122002</id>
+                        <name>tree</name>
+                        <type>22</type>
+                        <matcher>
+                                <id>org.eclipse.ui.ide.multiFilter</id>
+                                <arguments>1.0-name-matches-false-false-*.pyc</arguments>
+                        </matcher>
+                </filter>
+        </filteredResources>
+</projectDescription>
+"""
+
+CPROJECT_TEMPLATE_HEADER = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?fileVersion 4.0.0?>
+
+<cproject storage_type_id="org.eclipse.cdt.core.XmlProjectDescriptionStorage">
+        <storageModule moduleId="org.eclipse.cdt.core.settings">
+                <cconfiguration id="0.1674256904">
+                        <storageModule buildSystemId="org.eclipse.cdt.managedbuilder.core.configurationDataProvider" id="0.1674256904" moduleId="org.eclipse.cdt.core.settings" name="Default">
+                                <externalSettings/>
+                                <extensions>
+                                        <extension id="org.eclipse.cdt.core.VCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+                                        <extension id="org.eclipse.cdt.core.GmakeErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+                                        <extension id="org.eclipse.cdt.core.CWDLocator" point="org.eclipse.cdt.core.ErrorParser"/>
+                                        <extension id="org.eclipse.cdt.core.GCCErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+                                        <extension id="org.eclipse.cdt.core.GASErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+                                        <extension id="org.eclipse.cdt.core.GLDErrorParser" point="org.eclipse.cdt.core.ErrorParser"/>
+                                </extensions>
+                        </storageModule>
+                        <storageModule moduleId="cdtBuildSystem" version="4.0.0">
+                                <configuration artifactName="${ProjName}" buildProperties="" description="" id="0.1674256904" name="Default" parent="org.eclipse.cdt.build.core.prefbase.cfg">
+                                        <folderInfo id="0.1674256904." name="/" resourcePath="">
+                                                <toolChain id="cdt.managedbuild.toolchain.gnu.cross.exe.debug.1276586933" name="Cross GCC" superClass="cdt.managedbuild.toolchain.gnu.cross.exe.debug">
+                                                        <targetPlatform archList="all" binaryParser="org.eclipse.cdt.core.ELF" id="cdt.managedbuild.targetPlatform.gnu.cross.710759961" isAbstract="false" osList="all" superClass="cdt.managedbuild.targetPlatform.gnu.cross"/>
+							<builder arguments="--log-no-times build" buildPath="@PROJECT_TOPSRCDIR@" command="@MACH_COMMAND@" enableCleanBuild="false" incrementalBuildTarget="binaries" id="org.eclipse.cdt.build.core.settings.default.builder.1437267827" keepEnvironmentInBuildfile="false" name="Gnu Make Builder" superClass="org.eclipse.cdt.build.core.settings.default.builder"/>
+                                                </toolChain>
+                                        </folderInfo>
+"""
+CPROJECT_TEMPLATE_FOLDER_INFO_HEADER = """
+					<folderInfo id="0.1674256904.@FOLDER_ID@" name="/" resourcePath="@FOLDER_NAME@">
+						<toolChain id="org.eclipse.cdt.build.core.prefbase.toolchain.1022318069" name="No ToolChain" superClass="org.eclipse.cdt.build.core.prefbase.toolchain" unusedChildren="">
+							<tool id="org.eclipse.cdt.build.core.settings.holder.libs.1259030812" name="holder for library settings" superClass="org.eclipse.cdt.build.core.settings.holder.libs.1800697532"/>
+							<tool id="org.eclipse.cdt.build.core.settings.holder.1407291069" name="GNU C++" superClass="org.eclipse.cdt.build.core.settings.holder.582514939">
+								<option id="org.eclipse.cdt.build.core.settings.holder.symbols.1907658087" superClass="org.eclipse.cdt.build.core.settings.holder.symbols" valueType="definedSymbols">
+"""
+CPROJECT_TEMPLATE_FOLDER_INFO_DEFINE = """
+									<listOptionValue builtIn="false" value="@FOLDER_DEFINE@"/>
+"""
+CPROJECT_TEMPLATE_FOLDER_INFO_FOOTER = """
+								</option>
+								<inputType id="org.eclipse.cdt.build.core.settings.holder.inType.440601711" languageId="org.eclipse.cdt.core.g++" languageName="GNU C++" sourceContentType="org.eclipse.cdt.core.cxxSource,org.eclipse.cdt.core.cxxHeader" superClass="org.eclipse.cdt.build.core.settings.holder.inType"/>
+							</tool>
+						</toolChain>
+					</folderInfo>
+"""
+CPROJECT_TEMPLATE_FILEINFO = """                                        <fileInfo id="0.1674256904.474736658" name="Layers.cpp" rcbsApplicability="disable" resourcePath="tree/gfx/layers/Layers.cpp" toolsToInvoke="org.eclipse.cdt.build.core.settings.holder.582514939.463639939">
+                                                <tool id="org.eclipse.cdt.build.core.settings.holder.582514939.463639939" name="GNU C++" superClass="org.eclipse.cdt.build.core.settings.holder.582514939">
+                                                        <option id="org.eclipse.cdt.build.core.settings.holder.symbols.232300236" superClass="org.eclipse.cdt.build.core.settings.holder.symbols" valueType="definedSymbols">
+                                                                <listOptionValue builtIn="false" value="BENWA=BENWAVAL"/>
+                                                        </option>
+                                                        <inputType id="org.eclipse.cdt.build.core.settings.holder.inType.1942876228" languageId="org.eclipse.cdt.core.g++" languageName="GNU C++" sourceContentType="org.eclipse.cdt.core.cxxSource,org.eclipse.cdt.core.cxxHeader" superClass="org.eclipse.cdt.build.core.settings.holder.inType"/>
+                                                </tool>
+                                        </fileInfo>
+"""
+CPROJECT_TEMPLATE_FOOTER = """                                </configuration>
+                        </storageModule>
+                        <storageModule moduleId="org.eclipse.cdt.core.externalSettings"/>
+                </cconfiguration>
+        </storageModule>
+        <storageModule moduleId="cdtBuildSystem" version="4.0.0">
+                <project id="Empty.null.1281234804" name="Empty"/>
+        </storageModule>
+        <storageModule moduleId="scannerConfiguration">
+                <autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+                <scannerConfigBuildInfo instanceId="0.1674256904">
+                        <autodiscovery enabled="true" problemReportingEnabled="true" selectedProfileId=""/>
+                </scannerConfigBuildInfo>
+        </storageModule>
+        <storageModule moduleId="refreshScope" versionNumber="2">
+                <configuration configurationName="Default"/>
+        </storageModule>
+        <storageModule moduleId="org.eclipse.cdt.core.LanguageSettingsProviders"/>
+</cproject>
+"""
+
+WORKSPACE_LANGUAGE_SETTINGS_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<plugin>
+        <extension point="org.eclipse.cdt.core.LanguageSettingsProvider">
+                <provider class="org.eclipse.cdt.managedbuilder.language.settings.providers.GCCBuiltinSpecsDetector" console="true" id="org.eclipse.cdt.managedbuilder.core.GCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT GCC Built-in Compiler Settings" parameter="@COMPILER_FLAGS@ -E -P -v -dD &quot;${INPUTS}&quot;">
+                        <language-scope id="org.eclipse.cdt.core.gcc"/>
+                        <language-scope id="org.eclipse.cdt.core.g++"/>
+                </provider>
+        </extension>
+</plugin>
+"""
+
+LANGUAGE_SETTINGS_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<project>
+        <configuration id="0.1674256904" name="Default">
+                <extension point="org.eclipse.cdt.core.LanguageSettingsProvider">
+                        <provider class="org.eclipse.cdt.core.language.settings.providers.LanguageSettingsGenericProvider" id="org.eclipse.cdt.ui.UserLanguageSettingsProvider" name="CDT User Setting Entries" prefer-non-shared="true" store-entries-with-project="true">
+                                <language id="org.eclipse.cdt.core.g++">
+                                        <resource project-relative-path="">
+                                                <entry kind="includePath" name="@GLOBAL_INCLUDE_PATH@">
+                                                        <flag value="LOCAL"/>
+                                                </entry>
+                                                <entry kind="includePath" name="@NSPR_INCLUDE_PATH@">
+                                                        <flag value="LOCAL"/>
+                                                </entry>
+                                                <entry kind="includePath" name="@IPDL_INCLUDE_PATH@">
+                                                        <flag value="LOCAL"/>
+                                                </entry>
+                                                <entry kind="includeFile" name="@PREINCLUDE_FILE_PATH@">
+                                                        <flag value="LOCAL"/>
+                                                </entry>
+                                                <!--
+                                                  Because of https://developer.mozilla.org/en-US/docs/Eclipse_CDT#Headers_are_only_parsed_once
+                                                  we need to make sure headers are parsed with MOZILLA_INTERNAL_API to make sure
+                                                  the indexer gets the version that is used in most of the true. This means that
+                                                  MOZILLA_EXTERNAL_API code will suffer.
+                                                -->
+                                                @DEFINE_MOZILLA_INTERNAL_API@
+                                        </resource>
+                                </language>
+                        </provider>
+                        <provider class="org.eclipse.cdt.internal.build.crossgcc.CrossGCCBuiltinSpecsDetector" console="false" env-hash="-859273372804152468" id="org.eclipse.cdt.build.crossgcc.CrossGCCBuiltinSpecsDetector" keep-relative-paths="false" name="CDT Cross GCC Built-in Compiler Settings" parameter="@COMPILER_FLAGS@ -E -P -v -dD &quot;${INPUTS}&quot; -std=c++11" prefer-non-shared="true" store-entries-with-project="true">
+                             <language-scope id="org.eclipse.cdt.core.gcc"/>
+                             <language-scope id="org.eclipse.cdt.core.g++"/>
+                        </provider>
+                        <provider-reference id="org.eclipse.cdt.managedbuilder.core.MBSLanguageSettingsProvider" ref="shared-provider"/>
+                </extension>
+        </configuration>
+</project>
+"""
+
+GECKO_LAUNCH_CONFIG_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.cdt.launch.applicationLaunchType">
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB" value="true"/>
+<listAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB_LIST"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_NAME" value="lldb"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_ON_FORK" value="false"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.GDB_INIT" value=""/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.NON_STOP" value="false"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.REVERSE" value="false"/>
+<listAttribute key="org.eclipse.cdt.dsf.gdb.SOLIB_PATH"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.TRACEPOINT_MODE" value="TP_NORMAL_ONLY"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.UPDATE_THREADLIST_ON_SUSPEND" value="false"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.internal.ui.launching.LocalApplicationCDebuggerTab.DEFAULTS_SET" value="true"/>
+<intAttribute key="org.eclipse.cdt.launch.ATTR_BUILD_BEFORE_LAUNCH_ATTR" value="2"/>
+<stringAttribute key="org.eclipse.cdt.launch.COREFILE_PATH" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_ID" value="gdb"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_START_MODE" value="run"/>
+<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="false"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value="main"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROGRAM_ARGUMENTS" value="@LAUNCH_ARGS@"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value="@LAUNCH_PROGRAM@"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_ATTR" value="Gecko"/>
+<booleanAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_AUTO_ATTR" value="true"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_ID_ATTR" value=""/>
+<booleanAttribute key="org.eclipse.cdt.launch.use_terminal" value="true"/>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/gecko"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="4"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>
+<stringAttribute key="process_factory_id" value="org.eclipse.cdt.dsf.gdb.GdbProcessFactory"/>
+</launchConfiguration>
+"""
+
+B2GFLASH_LAUNCH_CONFIG_TEMPLATE = """<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.cdt.launch.applicationLaunchType">
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB" value="true"/>
+<listAttribute key="org.eclipse.cdt.dsf.gdb.AUTO_SOLIB_LIST"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_NAME" value="lldb"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.DEBUG_ON_FORK" value="false"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.GDB_INIT" value=""/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.NON_STOP" value="false"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.REVERSE" value="false"/>
+<listAttribute key="org.eclipse.cdt.dsf.gdb.SOLIB_PATH"/>
+<stringAttribute key="org.eclipse.cdt.dsf.gdb.TRACEPOINT_MODE" value="TP_NORMAL_ONLY"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.UPDATE_THREADLIST_ON_SUSPEND" value="false"/>
+<booleanAttribute key="org.eclipse.cdt.dsf.gdb.internal.ui.launching.LocalApplicationCDebuggerTab.DEFAULTS_SET" value="true"/>
+<intAttribute key="org.eclipse.cdt.launch.ATTR_BUILD_BEFORE_LAUNCH_ATTR" value="2"/>
+<stringAttribute key="org.eclipse.cdt.launch.COREFILE_PATH" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_ID" value="gdb"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_START_MODE" value="run"/>
+<booleanAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN" value="false"/>
+<stringAttribute key="org.eclipse.cdt.launch.DEBUGGER_STOP_AT_MAIN_SYMBOL" value="main"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROGRAM_NAME" value="@LAUNCH_PROGRAM@"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_ATTR" value="Gecko"/>
+<booleanAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_AUTO_ATTR" value="true"/>
+<stringAttribute key="org.eclipse.cdt.launch.PROJECT_BUILD_CONFIG_ID_ATTR" value=""/>
+<stringAttribute key="org.eclipse.cdt.launch.WORKING_DIRECTORY" value="@OBJDIR@"/>
+<booleanAttribute key="org.eclipse.cdt.launch.use_terminal" value="true"/>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_PATHS">
+<listEntry value="/gecko"/>
+</listAttribute>
+<listAttribute key="org.eclipse.debug.core.MAPPED_RESOURCE_TYPES">
+<listEntry value="4"/>
+</listAttribute>
+<booleanAttribute key="org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND" value="false"/>
+<stringAttribute key="process_factory_id" value="org.eclipse.cdt.dsf.gdb.GdbProcessFactory"/>
+</launchConfiguration>
+"""
+
+
+EDITOR_SETTINGS = """eclipse.preferences.version=1
+lineNumberRuler=true
+overviewRuler_migration=migrated_3.1
+printMargin=true
+printMarginColumn=80
+showCarriageReturn=false
+showEnclosedSpaces=false
+showLeadingSpaces=false
+showLineFeed=false
+showWhitespaceCharacters=true
+spacesForTabs=true
+tabWidth=2
+undoHistorySize=200
+"""
+
+FORMATTER_SETTINGS = """eclipse.preferences.version=1
+org.eclipse.cdt.core.formatter.alignment_for_arguments_in_method_invocation=16
+org.eclipse.cdt.core.formatter.alignment_for_assignment=16
+org.eclipse.cdt.core.formatter.alignment_for_base_clause_in_type_declaration=80
+org.eclipse.cdt.core.formatter.alignment_for_binary_expression=16
+org.eclipse.cdt.core.formatter.alignment_for_compact_if=16
+org.eclipse.cdt.core.formatter.alignment_for_conditional_expression=34
+org.eclipse.cdt.core.formatter.alignment_for_conditional_expression_chain=18
+org.eclipse.cdt.core.formatter.alignment_for_constructor_initializer_list=48
+org.eclipse.cdt.core.formatter.alignment_for_declarator_list=16
+org.eclipse.cdt.core.formatter.alignment_for_enumerator_list=48
+org.eclipse.cdt.core.formatter.alignment_for_expression_list=0
+org.eclipse.cdt.core.formatter.alignment_for_expressions_in_array_initializer=16
+org.eclipse.cdt.core.formatter.alignment_for_member_access=0
+org.eclipse.cdt.core.formatter.alignment_for_overloaded_left_shift_chain=16
+org.eclipse.cdt.core.formatter.alignment_for_parameters_in_method_declaration=16
+org.eclipse.cdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
+org.eclipse.cdt.core.formatter.brace_position_for_array_initializer=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_block=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_block_in_case=next_line_shifted
+org.eclipse.cdt.core.formatter.brace_position_for_method_declaration=next_line
+org.eclipse.cdt.core.formatter.brace_position_for_namespace_declaration=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_switch=end_of_line
+org.eclipse.cdt.core.formatter.brace_position_for_type_declaration=next_line
+org.eclipse.cdt.core.formatter.comment.min_distance_between_code_and_line_comment=1
+org.eclipse.cdt.core.formatter.comment.never_indent_line_comments_on_first_column=true
+org.eclipse.cdt.core.formatter.comment.preserve_white_space_between_code_and_line_comments=true
+org.eclipse.cdt.core.formatter.compact_else_if=true
+org.eclipse.cdt.core.formatter.continuation_indentation=2
+org.eclipse.cdt.core.formatter.continuation_indentation_for_array_initializer=2
+org.eclipse.cdt.core.formatter.format_guardian_clause_on_one_line=false
+org.eclipse.cdt.core.formatter.indent_access_specifier_compare_to_type_header=false
+org.eclipse.cdt.core.formatter.indent_access_specifier_extra_spaces=0
+org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_access_specifier=true
+org.eclipse.cdt.core.formatter.indent_body_declarations_compare_to_namespace_header=false
+org.eclipse.cdt.core.formatter.indent_breaks_compare_to_cases=true
+org.eclipse.cdt.core.formatter.indent_declaration_compare_to_template_header=true
+org.eclipse.cdt.core.formatter.indent_empty_lines=false
+org.eclipse.cdt.core.formatter.indent_statements_compare_to_block=true
+org.eclipse.cdt.core.formatter.indent_statements_compare_to_body=true
+org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_cases=true
+org.eclipse.cdt.core.formatter.indent_switchstatements_compare_to_switch=false
+org.eclipse.cdt.core.formatter.indentation.size=2
+org.eclipse.cdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_after_template_declaration=insert
+org.eclipse.cdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_colon_in_constructor_initializer_list=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_identifier_in_function_declaration=insert
+org.eclipse.cdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
+org.eclipse.cdt.core.formatter.insert_new_line_in_empty_block=insert
+org.eclipse.cdt.core.formatter.insert_space_after_assignment_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_after_binary_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_arguments=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_angle_bracket_in_template_parameters=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_brace_in_block=insert
+org.eclipse.cdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_base_clause=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_case=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_base_types=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_declarator_list=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_expression_list=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_arguments=insert
+org.eclipse.cdt.core.formatter.insert_space_after_comma_in_template_parameters=insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_angle_bracket_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_bracket=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_exception_specification=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_postfix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_prefix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_after_question_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_after_semicolon_in_for=insert
+org.eclipse.cdt.core.formatter.insert_space_after_unary_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_assignment_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_before_binary_operator=insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_angle_bracket_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_bracket=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_exception_specification=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_base_clause=insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_case=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_default=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_base_types=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_declarator_list=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_expression_list=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_comma_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_arguments=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_angle_bracket_in_template_parameters=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_block=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_namespace_declaration=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_bracket=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_exception_specification=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_for=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_if=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
+org.eclipse.cdt.core.formatter.insert_space_before_opening_paren_in_while=insert
+org.eclipse.cdt.core.formatter.insert_space_before_postfix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_prefix_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_question_in_conditional=insert
+org.eclipse.cdt.core.formatter.insert_space_before_semicolon=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
+org.eclipse.cdt.core.formatter.insert_space_before_unary_operator=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_brackets=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_exception_specification=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
+org.eclipse.cdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
+org.eclipse.cdt.core.formatter.join_wrapped_lines=false
+org.eclipse.cdt.core.formatter.keep_else_statement_on_same_line=false
+org.eclipse.cdt.core.formatter.keep_empty_array_initializer_on_one_line=false
+org.eclipse.cdt.core.formatter.keep_imple_if_on_one_line=false
+org.eclipse.cdt.core.formatter.keep_then_statement_on_same_line=false
+org.eclipse.cdt.core.formatter.lineSplit=80
+org.eclipse.cdt.core.formatter.number_of_empty_lines_to_preserve=1
+org.eclipse.cdt.core.formatter.put_empty_statement_on_new_line=true
+org.eclipse.cdt.core.formatter.tabulation.char=space
+org.eclipse.cdt.core.formatter.tabulation.size=2
+org.eclipse.cdt.core.formatter.use_tabs_only_for_leading_indentations=false
+"""
+
+NOINDEX_TEMPLATE = """eclipse.preferences.version=1
+indexer/indexerId=org.eclipse.cdt.core.nullIndexer
+"""

python/mozbuild/mozbuild/backend/fastermake.py

@@ -0,0 +1,165 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals, print_function
+
+from mozbuild.backend.base import PartialBackend
+from mozbuild.backend.common import CommonBackend
+from mozbuild.frontend.context import (
+    ObjDirPath,
+)
+from mozbuild.frontend.data import (
+    ChromeManifestEntry,
+    FinalTargetPreprocessedFiles,
+    FinalTargetFiles,
+    JARManifest,
+    XPIDLFile,
+)
+from mozbuild.makeutil import Makefile
+from mozbuild.util import OrderedDefaultDict
+from mozpack.manifests import InstallManifest
+import mozpack.path as mozpath
+
+
+class FasterMakeBackend(CommonBackend, PartialBackend):
+    def _init(self):
+        super(FasterMakeBackend, self)._init()
+
+        self._manifest_entries = OrderedDefaultDict(set)
+
+        self._install_manifests = OrderedDefaultDict(InstallManifest)
+
+        self._dependencies = OrderedDefaultDict(list)
+
+        self._has_xpidl = False
+
+    def _add_preprocess(self, obj, path, dest, target=None, **kwargs):
+        if target is None:
+            target = mozpath.basename(path)
+        # This matches what PP_TARGETS do in config/rules.
+        if target.endswith('.in'):
+            target = target[:-3]
+        if target.endswith('.css'):
+            kwargs['marker'] = '%'
+        depfile = mozpath.join(
+            self.environment.topobjdir, 'faster', '.deps',
+            mozpath.join(obj.install_target, dest, target).replace('/', '_'))
+        self._install_manifests[obj.install_target].add_preprocess(
+            mozpath.join(obj.srcdir, path),
+            mozpath.join(dest, target),
+            depfile,
+            **kwargs)
+
+    def consume_object(self, obj):
+        if isinstance(obj, JARManifest) and \
+                obj.install_target.startswith('dist/bin'):
+            self._consume_jar_manifest(obj)
+
+        elif isinstance(obj, (FinalTargetFiles,
+                              FinalTargetPreprocessedFiles)) and \
+                obj.install_target.startswith('dist/bin'):
+            defines = obj.defines or {}
+            if defines:
+                defines = defines.defines
+            for path, files in obj.files.walk():
+                for f in files:
+                    if isinstance(obj, FinalTargetPreprocessedFiles):
+                        self._add_preprocess(obj, f.full_path, path,
+                                             target=f.target_basename,
+                                             defines=defines)
+                    elif '*' in f:
+                        def _prefix(s):
+                            for p in mozpath.split(s):
+                                if '*' not in p:
+                                    yield p + '/'
+                        prefix = ''.join(_prefix(f.full_path))
+
+                        self._install_manifests[obj.install_target] \
+                            .add_pattern_symlink(
+                                prefix,
+                                f.full_path[len(prefix):],
+                                mozpath.join(path, f.target_basename))
+                    else:
+                        self._install_manifests[obj.install_target].add_symlink(
+                            f.full_path,
+                            mozpath.join(path, f.target_basename)
+                        )
+                    if isinstance(f, ObjDirPath):
+                        dep_target = 'install-%s' % obj.install_target
+                        self._dependencies[dep_target].append(
+                            mozpath.relpath(f.full_path,
+                                            self.environment.topobjdir))
+
+        elif isinstance(obj, ChromeManifestEntry) and \
+                obj.install_target.startswith('dist/bin'):
+            top_level = mozpath.join(obj.install_target, 'chrome.manifest')
+            if obj.path != top_level:
+                entry = 'manifest %s' % mozpath.relpath(obj.path,
+                                                        obj.install_target)
+                self._manifest_entries[top_level].add(entry)
+            self._manifest_entries[obj.path].add(str(obj.entry))
+
+        elif isinstance(obj, XPIDLFile):
+            self._has_xpidl = True
+            # We're not actually handling XPIDL files.
+            return False
+
+        else:
+            return False
+
+        return True
+
+    def consume_finished(self):
+        mk = Makefile()
+        # Add the default rule at the very beginning.
+        mk.create_rule(['default'])
+        mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
+        mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
+        if not self._has_xpidl:
+            mk.add_statement('NO_XPIDL = 1')
+
+        # Add a few necessary variables inherited from configure
+        for var in (
+            'PYTHON',
+            'ACDEFINES',
+            'MOZ_BUILD_APP',
+            'MOZ_WIDGET_TOOLKIT',
+        ):
+            value = self.environment.substs.get(var)
+            if value is not None:
+                mk.add_statement('%s = %s' % (var, value))
+
+        install_manifests_bases = self._install_manifests.keys()
+
+        # Add information for chrome manifest generation
+        manifest_targets = []
+
+        for target, entries in self._manifest_entries.iteritems():
+            manifest_targets.append(target)
+            install_target = mozpath.basedir(target, install_manifests_bases)
+            self._install_manifests[install_target].add_content(
+                ''.join('%s\n' % e for e in sorted(entries)),
+                mozpath.relpath(target, install_target))
+
+        # Add information for install manifests.
+        mk.add_statement('INSTALL_MANIFESTS = %s'
+                         % ' '.join(self._install_manifests.keys()))
+
+        # Add dependencies we infered:
+        for target, deps in self._dependencies.iteritems():
+            mk.create_rule([target]).add_dependencies(
+                '$(TOPOBJDIR)/%s' % d for d in deps)
+
+        mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
+
+        for base, install_manifest in self._install_manifests.iteritems():
+            with self._write_file(
+                    mozpath.join(self.environment.topobjdir, 'faster',
+                                 'install_%s' % base.replace('/', '_'))) as fh:
+                install_manifest.write(fileobj=fh)
+
+        with self._write_file(
+                mozpath.join(self.environment.topobjdir, 'faster',
+                             'Makefile')) as fh:
+            mk.dump(fh, removal_guard=False)

python/mozbuild/mozbuild/backend/mach_commands.py

@@ -0,0 +1,132 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import os
+import sys
+import subprocess
+import which
+
+from mozbuild.base import (
+    MachCommandBase,
+)
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+@CommandProvider
+class MachCommands(MachCommandBase):
+    @Command('ide', category='devenv',
+        description='Generate a project and launch an IDE.')
+    @CommandArgument('ide', choices=['eclipse', 'visualstudio', 'androidstudio', 'intellij'])
+    @CommandArgument('args', nargs=argparse.REMAINDER)
+    def eclipse(self, ide, args):
+        if ide == 'eclipse':
+            backend = 'CppEclipse'
+        elif ide == 'visualstudio':
+            backend = 'VisualStudio'
+        elif ide == 'androidstudio' or ide == 'intellij':
+            # The build backend for Android Studio and IntelliJ is just the regular one.
+            backend = 'RecursiveMake'
+
+        if ide == 'eclipse':
+            try:
+                which.which('eclipse')
+            except which.WhichError:
+                print('Eclipse CDT 8.4 or later must be installed in your PATH.')
+                print('Download: http://www.eclipse.org/cdt/downloads.php')
+                return 1
+        elif ide == 'androidstudio' or ide =='intellij':
+            studio = ['studio'] if ide == 'androidstudio' else ['idea']
+            if sys.platform != 'darwin':
+                try:
+                    which.which(studio[0])
+                except:
+                    self.print_ide_error(ide)
+                    return 1
+            else:
+                # In order of preference!
+                for d in self.get_mac_ide_preferences(ide):
+                    if os.path.isdir(d):
+                        studio = ['open', '-a', d]
+                        break
+                else:
+                    print('Android Studio or IntelliJ IDEA 14 is not installed in /Applications.')
+                    return 1
+
+        # Here we refresh the whole build. 'build export' is sufficient here and is probably more
+        # correct but it's also nice having a single target to get a fully built and indexed
+        # project (gives a easy target to use before go out to lunch).
+        res = self._mach_context.commands.dispatch('build', self._mach_context)
+        if res != 0:
+            return 1
+
+        if ide in ('androidstudio', 'intellij'):
+            res = self._mach_context.commands.dispatch('package', self._mach_context)
+            if res != 0:
+                return 1
+        else:
+            # Generate or refresh the IDE backend.
+            python = self.virtualenv_manager.python_path
+            config_status = os.path.join(self.topobjdir, 'config.status')
+            args = [python, config_status, '--backend=%s' % backend]
+            res = self._run_command_in_objdir(args=args, pass_thru=True, ensure_exit_code=False)
+            if res != 0:
+                return 1
+
+
+        if ide == 'eclipse':
+            eclipse_workspace_dir = self.get_eclipse_workspace_path()
+            process = subprocess.check_call(['eclipse', '-data', eclipse_workspace_dir])
+        elif ide == 'visualstudio':
+            visual_studio_workspace_dir = self.get_visualstudio_workspace_path()
+            process = subprocess.check_call(['explorer.exe', visual_studio_workspace_dir])
+        elif ide == 'androidstudio' or ide == 'intellij':
+            gradle_dir = None
+            if self.is_gradle_project_already_imported():
+                gradle_dir = self.get_gradle_project_path()
+            else:
+                gradle_dir = self.get_gradle_import_path()
+            process = subprocess.check_call(studio + [gradle_dir])
+
+    def get_eclipse_workspace_path(self):
+        from mozbuild.backend.cpp_eclipse import CppEclipseBackend
+        return CppEclipseBackend.get_workspace_path(self.topsrcdir, self.topobjdir)
+
+    def get_visualstudio_workspace_path(self):
+        return os.path.join(self.topobjdir, 'msvc', 'mozilla.sln')
+
+    def get_gradle_project_path(self):
+        return os.path.join(self.topobjdir, 'mobile', 'android', 'gradle')
+
+    def get_gradle_import_path(self):
+        return os.path.join(self.get_gradle_project_path(), 'build.gradle')
+
+    def is_gradle_project_already_imported(self):
+        gradle_project_path = os.path.join(self.get_gradle_project_path(), '.idea')
+        return os.path.exists(gradle_project_path)
+
+    def get_mac_ide_preferences(self, ide):
+        if sys.platform == 'darwin':
+            if ide == 'androidstudio':
+                return ['/Applications/Android Studio.app']
+            else:
+                return [
+                    '/Applications/IntelliJ IDEA 14 EAP.app',
+                    '/Applications/IntelliJ IDEA 14.app',
+                    '/Applications/IntelliJ IDEA 14 CE EAP.app',
+                    '/Applications/IntelliJ IDEA 14 CE.app']
+
+    def print_ide_error(self, ide):
+        if ide == 'androidstudio':
+            print('Android Studio is not installed in your PATH.')
+            print('You can generate a command-line launcher from Android Studio->Tools->Create Command-line launcher with script name \'studio\'')
+        elif ide == 'intellij':
+            print('IntelliJ is not installed in your PATH.')
+            print('You can generate a command-line launcher from IntelliJ IDEA->Tools->Create Command-line launcher with script name \'idea\'')

python/mozbuild/mozbuild/backend/templates/android_eclipse/.classpath

@@ -0,0 +1,10 @@
+#filter substitution
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+	<classpathentry kind="src" path="gen"/>
+	<classpathentry exported="true" kind="con" path="com.android.ide.eclipse.adt.ANDROID_FRAMEWORK"/>
+	<classpathentry exported="true" kind="con" path="com.android.ide.eclipse.adt.LIBRARIES"/>
+	<classpathentry exported="true" kind="con" path="com.android.ide.eclipse.adt.DEPENDENCIES"/>
+	<classpathentry kind="output" path="bin/classes"/>
+@IDE_CLASSPATH_ENTRIES@
+</classpath>

python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ApkBuilder.launch

@@ -0,0 +1,8 @@
+#filter substitution
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="false"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_DISABLED_BUILDER" value="com.android.ide.eclipse.adt.ApkBuilder"/>
+<mapAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS"/>
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
+</launchConfiguration>

python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.PreCompilerBuilder.launch

@@ -0,0 +1,8 @@
+#filter substitution
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="false"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_DISABLED_BUILDER" value="com.android.ide.eclipse.adt.PreCompilerBuilder"/>
+<mapAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS"/>
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
+</launchConfiguration>

python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/com.android.ide.eclipse.adt.ResourceManagerBuilder.launch

@@ -0,0 +1,8 @@
+#filter substitution
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="false"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_DISABLED_BUILDER" value="com.android.ide.eclipse.adt.ResourceManagerBuilder"/>
+<mapAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS"/>
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
+</launchConfiguration>

python/mozbuild/mozbuild/backend/templates/android_eclipse/.externalToolBuilders/org.eclipse.jdt.core.javabuilder.launch

@@ -0,0 +1,8 @@
+#filter substitution
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<launchConfiguration type="org.eclipse.ant.AntBuilderLaunchConfigurationType">
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_BUILDER_ENABLED" value="true"/>
+<stringAttribute key="org.eclipse.ui.externaltools.ATTR_DISABLED_BUILDER" value="org.eclipse.jdt.core.javabuilder"/>
+<mapAttribute key="org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS"/>
+<booleanAttribute key="org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED" value="true"/>
+</launchConfiguration>

python/mozbuild/mozbuild/backend/templates/android_eclipse/AndroidManifest.xml

@@ -0,0 +1,11 @@
+#filter substitution
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="@IDE_PACKAGE_NAME@"
+    android:versionCode="1"
+    android:versionName="1.0" >
+
+    <uses-sdk
+        android:minSdkVersion="@MOZ_ANDROID_MIN_SDK_VERSION@"
+        android:targetSdkVersion="@ANDROID_TARGET_SDK@" />
+
+</manifest>

python/mozbuild/mozbuild/backend/templates/android_eclipse/gen/tmp

@@ -0,0 +1 @@
+#filter substitution

python/mozbuild/mozbuild/backend/templates/android_eclipse/lint.xml

@@ -0,0 +1,5 @@
+#filter substitution
+<?xml version="1.0" encoding="UTF-8"?>
+<lint>
+    <issue id="NewApi" severity="ignore" />
+</lint>

python/mozbuild/mozbuild/backend/templates/android_eclipse/project.properties

@@ -0,0 +1,14 @@
+#filter substitution
+# This file is automatically generated by Android Tools.
+# Do not modify this file -- YOUR CHANGES WILL BE ERASED!
+#
+# This file must be checked in Version Control Systems.
+#
+# To customize properties used by the Ant build system edit
+# "ant.properties", and override values to adapt the script to your
+# project structure.
+
+# Project target.
+target=android-L
+@IDE_PROJECT_LIBRARY_SETTING@
+@IDE_PROJECT_LIBRARY_REFERENCES@

python/mozbuild/mozbuild/backend/templates/android_eclipse_empty_resource_directory/.not_an_android_resource

@@ -0,0 +1,5 @@
+This file is named such that it is ignored by Android aapt. The file
+itself ensures that the AndroidEclipse build backend can create an
+empty res/ directory for projects explicitly specifying that it has no
+resource directory.  This is necessary because the Android Eclipse
+plugin requires that each project have a res/ directory.

python/mozbuild/mozbuild/backend/tup.py

@@ -0,0 +1,344 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+
+import mozpack.path as mozpath
+from mozbuild.base import MozbuildObject
+from mozbuild.backend.base import PartialBackend, HybridBackend
+from mozbuild.backend.recursivemake import RecursiveMakeBackend
+from mozbuild.shellutil import quote as shell_quote
+
+from .common import CommonBackend
+from ..frontend.data import (
+    ContextDerived,
+    Defines,
+    FinalTargetPreprocessedFiles,
+    GeneratedFile,
+    HostDefines,
+    ObjdirPreprocessedFiles,
+)
+from ..util import (
+    FileAvoidWrite,
+)
+
+
+class BackendTupfile(object):
+    """Represents a generated Tupfile.
+    """
+
+    def __init__(self, srcdir, objdir, environment, topsrcdir, topobjdir):
+        self.topsrcdir = topsrcdir
+        self.srcdir = srcdir
+        self.objdir = objdir
+        self.relobjdir = mozpath.relpath(objdir, topobjdir)
+        self.environment = environment
+        self.name = mozpath.join(objdir, 'Tupfile')
+        self.rules_included = False
+        self.shell_exported = False
+        self.defines = []
+        self.host_defines = []
+        self.delayed_generated_files = []
+
+        self.fh = FileAvoidWrite(self.name, capture_diff=True)
+        self.fh.write('# THIS FILE WAS AUTOMATICALLY GENERATED. DO NOT EDIT.\n')
+        self.fh.write('\n')
+
+    def write(self, buf):
+        self.fh.write(buf)
+
+    def include_rules(self):
+        if not self.rules_included:
+            self.write('include_rules\n')
+            self.rules_included = True
+
+    def rule(self, cmd, inputs=None, outputs=None, display=None, extra_outputs=None, check_unchanged=False):
+        inputs = inputs or []
+        outputs = outputs or []
+        display = display or ""
+        self.include_rules()
+        flags = ""
+        if check_unchanged:
+            # This flag causes tup to compare the outputs with the previous run
+            # of the command, and skip the rest of the DAG for any that are the
+            # same.
+            flags += "o"
+
+        if display:
+            caret_text = flags + ' ' + display
+        else:
+            caret_text = flags
+
+        self.write(': %(inputs)s |> %(display)s%(cmd)s |> %(outputs)s%(extra_outputs)s\n' % {
+            'inputs': ' '.join(inputs),
+            'display': '^%s^ ' % caret_text if caret_text else '',
+            'cmd': ' '.join(cmd),
+            'outputs': ' '.join(outputs),
+            'extra_outputs': ' | ' + ' '.join(extra_outputs) if extra_outputs else '',
+        })
+
+    def export_shell(self):
+        if not self.shell_exported:
+            # These are used by mach/mixin/process.py to determine the current
+            # shell.
+            for var in ('SHELL', 'MOZILLABUILD', 'COMSPEC'):
+                self.write('export %s\n' % var)
+            self.shell_exported = True
+
+    def close(self):
+        return self.fh.close()
+
+    @property
+    def diff(self):
+        return self.fh.diff
+
+
+class TupOnly(CommonBackend, PartialBackend):
+    """Backend that generates Tupfiles for the tup build system.
+    """
+
+    def _init(self):
+        CommonBackend._init(self)
+
+        self._backend_files = {}
+        self._cmd = MozbuildObject.from_environment()
+
+    def _get_backend_file(self, relativedir):
+        objdir = mozpath.join(self.environment.topobjdir, relativedir)
+        srcdir = mozpath.join(self.environment.topsrcdir, relativedir)
+        if objdir not in self._backend_files:
+            self._backend_files[objdir] = \
+                    BackendTupfile(srcdir, objdir, self.environment,
+                                   self.environment.topsrcdir, self.environment.topobjdir)
+        return self._backend_files[objdir]
+
+    def _get_backend_file_for(self, obj):
+        return self._get_backend_file(obj.relativedir)
+
+    def _py_action(self, action):
+        cmd = [
+            '$(PYTHON)',
+            '-m',
+            'mozbuild.action.%s' % action,
+        ]
+        return cmd
+
+    def consume_object(self, obj):
+        """Write out build files necessary to build with tup."""
+
+        if not isinstance(obj, ContextDerived):
+            return False
+
+        consumed = CommonBackend.consume_object(self, obj)
+
+        # Even if CommonBackend acknowledged the object, we still need to let
+        # the RecursiveMake backend also handle these objects.
+        if consumed:
+            return False
+
+        backend_file = self._get_backend_file_for(obj)
+
+        if isinstance(obj, GeneratedFile):
+            # These files are already generated by make before tup runs.
+            skip_files = (
+                'buildid.h',
+                'source-repo.h',
+            )
+            if any(f in skip_files for f in obj.outputs):
+                # Let the RecursiveMake backend handle these.
+                return False
+
+            if 'application.ini.h' in obj.outputs:
+                # application.ini.h is a special case since we need to process
+                # the FINAL_TARGET_PP_FILES for application.ini before running
+                # the GENERATED_FILES script, and tup doesn't handle the rules
+                # out of order.
+                backend_file.delayed_generated_files.append(obj)
+            else:
+                self._process_generated_file(backend_file, obj)
+        elif isinstance(obj, Defines):
+            self._process_defines(backend_file, obj)
+        elif isinstance(obj, HostDefines):
+            self._process_defines(backend_file, obj, host=True)
+        elif isinstance(obj, FinalTargetPreprocessedFiles):
+            self._process_final_target_pp_files(obj, backend_file)
+        elif isinstance(obj, ObjdirPreprocessedFiles):
+            self._process_final_target_pp_files(obj, backend_file)
+
+        return True
+
+    def consume_finished(self):
+        CommonBackend.consume_finished(self)
+
+        for objdir, backend_file in sorted(self._backend_files.items()):
+            for obj in backend_file.delayed_generated_files:
+                self._process_generated_file(backend_file, obj)
+            with self._write_file(fh=backend_file):
+                pass
+
+        with self._write_file(mozpath.join(self.environment.topobjdir, 'Tuprules.tup')) as fh:
+            acdefines = [name for name in self.environment.defines
+                if not name in self.environment.non_global_defines]
+            acdefines_flags = ' '.join(['-D%s=%s' % (name,
+                shell_quote(self.environment.defines[name]))
+                for name in sorted(acdefines)])
+            # TODO: AB_CD only exists in Makefiles at the moment.
+            acdefines_flags += ' -DAB_CD=en-US'
+
+            fh.write('MOZ_OBJ_ROOT = $(TUP_CWD)\n')
+            fh.write('DIST = $(MOZ_OBJ_ROOT)/dist\n')
+            fh.write('ACDEFINES = %s\n' % acdefines_flags)
+            fh.write('topsrcdir = $(MOZ_OBJ_ROOT)/%s\n' % (
+                os.path.relpath(self.environment.topsrcdir, self.environment.topobjdir)
+            ))
+            fh.write('PYTHON = $(MOZ_OBJ_ROOT)/_virtualenv/bin/python -B\n')
+            fh.write('PYTHON_PATH = $(PYTHON) $(topsrcdir)/config/pythonpath.py\n')
+            fh.write('PLY_INCLUDE = -I$(topsrcdir)/other-licenses/ply\n')
+            fh.write('IDL_PARSER_DIR = $(topsrcdir)/xpcom/idl-parser\n')
+            fh.write('IDL_PARSER_CACHE_DIR = $(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl\n')
+
+        # Run 'tup init' if necessary.
+        if not os.path.exists(mozpath.join(self.environment.topsrcdir, ".tup")):
+            tup = self.environment.substs.get('TUP', 'tup')
+            self._cmd.run_process(cwd=self.environment.topsrcdir, log_name='tup', args=[tup, 'init'])
+
+    def _process_generated_file(self, backend_file, obj):
+        # TODO: These are directories that don't work in the tup backend
+        # yet, because things they depend on aren't built yet.
+        skip_directories = (
+            'layout/style/test', # HostSimplePrograms
+            'toolkit/library', # libxul.so
+        )
+        if obj.script and obj.method and obj.relobjdir not in skip_directories:
+            backend_file.export_shell()
+            cmd = self._py_action('file_generate')
+            cmd.extend([
+                obj.script,
+                obj.method,
+                obj.outputs[0],
+                '%s.pp' % obj.outputs[0], # deps file required
+            ])
+            full_inputs = [f.full_path for f in obj.inputs]
+            cmd.extend(full_inputs)
+
+            outputs = []
+            outputs.extend(obj.outputs)
+            outputs.append('%s.pp' % obj.outputs[0])
+
+            backend_file.rule(
+                display='python {script}:{method} -> [%o]'.format(script=obj.script, method=obj.method),
+                cmd=cmd,
+                inputs=full_inputs,
+                outputs=outputs,
+            )
+
+    def _process_defines(self, backend_file, obj, host=False):
+        defines = list(obj.get_defines())
+        if defines:
+            if host:
+                backend_file.host_defines = defines
+            else:
+                backend_file.defines = defines
+
+    def _process_final_target_pp_files(self, obj, backend_file):
+        for i, (path, files) in enumerate(obj.files.walk()):
+            for f in files:
+                self._preprocess(backend_file, f.full_path,
+                                 destdir=mozpath.join(self.environment.topobjdir, obj.install_target, path))
+
+    def _handle_idl_manager(self, manager):
+        backend_file = self._get_backend_file('xpcom/xpidl')
+        backend_file.export_shell()
+
+        for module, data in sorted(manager.modules.iteritems()):
+            dest, idls = data
+            cmd = [
+                '$(PYTHON_PATH)',
+                '$(PLY_INCLUDE)',
+                '-I$(IDL_PARSER_DIR)',
+                '-I$(IDL_PARSER_CACHE_DIR)',
+                '$(topsrcdir)/python/mozbuild/mozbuild/action/xpidl-process.py',
+                '--cache-dir', '$(IDL_PARSER_CACHE_DIR)',
+                '$(DIST)/idl',
+                '$(DIST)/include',
+                '$(MOZ_OBJ_ROOT)/%s/components' % dest,
+                module,
+            ]
+            cmd.extend(sorted(idls))
+
+            outputs = ['$(MOZ_OBJ_ROOT)/%s/components/%s.xpt' % (dest, module)]
+            outputs.extend(['$(MOZ_OBJ_ROOT)/dist/include/%s.h' % f for f in sorted(idls)])
+            backend_file.rule(
+                inputs=[
+                    '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidllex.py',
+                    '$(MOZ_OBJ_ROOT)/xpcom/idl-parser/xpidl/xpidlyacc.py',
+                ],
+                display='XPIDL %s' % module,
+                cmd=cmd,
+                outputs=outputs,
+            )
+
+    def _preprocess(self, backend_file, input_file, destdir=None):
+        cmd = self._py_action('preprocessor')
+        cmd.extend(backend_file.defines)
+        cmd.extend(['$(ACDEFINES)', '%f', '-o', '%o'])
+
+        base_input = mozpath.basename(input_file)
+        if base_input.endswith('.in'):
+            base_input = mozpath.splitext(base_input)[0]
+        output = mozpath.join(destdir, base_input) if destdir else base_input
+
+        backend_file.rule(
+            inputs=[input_file],
+            display='Preprocess %o',
+            cmd=cmd,
+            outputs=[output],
+        )
+
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+                             unified_ipdl_cppsrcs_mapping):
+        # TODO: This isn't implemented yet in the tup backend, but it is called
+        # by the CommonBackend.
+        pass
+
+    def _handle_webidl_build(self, bindings_dir, unified_source_mapping,
+                             webidls, expected_build_output_files,
+                             global_define_files):
+        backend_file = self._get_backend_file('dom/bindings')
+        backend_file.export_shell()
+
+        for source in sorted(webidls.all_preprocessed_sources()):
+            self._preprocess(backend_file, source)
+
+        cmd = self._py_action('webidl')
+        cmd.append(mozpath.join(self.environment.topsrcdir, 'dom', 'bindings'))
+
+        # The WebIDLCodegenManager knows all of the .cpp and .h files that will
+        # be created (expected_build_output_files), but there are a few
+        # additional files that are also created by the webidl py_action.
+        outputs = [
+            '_cache/webidlyacc.py',
+            'codegen.json',
+            'codegen.pp',
+            'parser.out',
+        ]
+        outputs.extend(expected_build_output_files)
+
+        backend_file.rule(
+            display='WebIDL code generation',
+            cmd=cmd,
+            inputs=webidls.all_non_static_basenames(),
+            outputs=outputs,
+            check_unchanged=True,
+        )
+
+
+class TupBackend(HybridBackend(TupOnly, RecursiveMakeBackend)):
+    def build(self, config, output, jobs, verbose):
+        status = config._run_make(directory=self.environment.topobjdir, target='tup',
+                                  line_handler=output.on_line, log=False, print_directory=False,
+                                  ensure_exit_code=False, num_jobs=jobs, silent=not verbose)
+        return status

python/mozbuild/mozbuild/backend/visualstudio.py

@@ -0,0 +1,582 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains a build backend for generating Visual Studio project
+# files.
+
+from __future__ import absolute_import, unicode_literals
+
+import errno
+import os
+import re
+import types
+import uuid
+
+from xml.dom import getDOMImplementation
+
+from mozpack.files import FileFinder
+
+from .common import CommonBackend
+from ..frontend.data import (
+    Defines,
+    GeneratedSources,
+    HostProgram,
+    HostSources,
+    Library,
+    LocalInclude,
+    Program,
+    Sources,
+    UnifiedSources,
+)
+from mozbuild.base import ExecutionSummary
+
+
+MSBUILD_NAMESPACE = 'http://schemas.microsoft.com/developer/msbuild/2003'
+
+def get_id(name):
+    return str(uuid.uuid5(uuid.NAMESPACE_URL, name)).upper()
+
+def visual_studio_product_to_solution_version(version):
+    if version == '2015':
+        return '12.00', '14'
+    else:
+        raise Exception('Unknown version seen: %s' % version)
+
+def visual_studio_product_to_platform_toolset_version(version):
+    if version == '2015':
+        return 'v140'
+    else:
+        raise Exception('Unknown version seen: %s' % version)
+
+class VisualStudioBackend(CommonBackend):
+    """Generate Visual Studio project files.
+
+    This backend is used to produce Visual Studio projects and a solution
+    to foster developing Firefox with Visual Studio.
+
+    This backend is currently considered experimental. There are many things
+    not optimal about how it works.
+    """
+
+    def _init(self):
+        CommonBackend._init(self)
+
+        # These should eventually evolve into parameters.
+        self._out_dir = os.path.join(self.environment.topobjdir, 'msvc')
+        self._projsubdir = 'projects'
+
+        self._version = self.environment.substs.get('MSVS_VERSION', '2015')
+
+        self._paths_to_sources = {}
+        self._paths_to_includes = {}
+        self._paths_to_defines = {}
+        self._paths_to_configs = {}
+        self._libs_to_paths = {}
+        self._progs_to_paths = {}
+
+    def summary(self):
+        return ExecutionSummary(
+            'VisualStudio backend executed in {execution_time:.2f}s\n'
+            'Generated Visual Studio solution at {path:s}',
+            execution_time=self._execution_time,
+            path=os.path.join(self._out_dir, 'mozilla.sln'))
+
+    def consume_object(self, obj):
+        reldir = getattr(obj, 'relativedir', None)
+
+        if hasattr(obj, 'config') and reldir not in self._paths_to_configs:
+            self._paths_to_configs[reldir] = obj.config
+
+        if isinstance(obj, Sources):
+            self._add_sources(reldir, obj)
+
+        elif isinstance(obj, HostSources):
+            self._add_sources(reldir, obj)
+
+        elif isinstance(obj, GeneratedSources):
+            self._add_sources(reldir, obj)
+
+        elif isinstance(obj, UnifiedSources):
+            # XXX we should be letting CommonBackend.consume_object call this
+            # for us instead.
+            self._process_unified_sources(obj);
+
+        elif isinstance(obj, Library):
+            self._libs_to_paths[obj.basename] = reldir
+
+        elif isinstance(obj, Program) or isinstance(obj, HostProgram):
+            self._progs_to_paths[obj.program] = reldir
+
+        elif isinstance(obj, Defines):
+            self._paths_to_defines.setdefault(reldir, {}).update(obj.defines)
+
+        elif isinstance(obj, LocalInclude):
+            includes = self._paths_to_includes.setdefault(reldir, [])
+            includes.append(obj.path.full_path)
+
+        # Just acknowledge everything.
+        return True
+
+    def _add_sources(self, reldir, obj):
+        s = self._paths_to_sources.setdefault(reldir, set())
+        s.update(obj.files)
+
+    def _process_unified_sources(self, obj):
+        reldir = getattr(obj, 'relativedir', None)
+
+        s = self._paths_to_sources.setdefault(reldir, set())
+        s.update(obj.files)
+
+    def consume_finished(self):
+        out_dir = self._out_dir
+        out_proj_dir = os.path.join(self._out_dir, self._projsubdir)
+
+        projects = self._write_projects_for_sources(self._libs_to_paths,
+            "library", out_proj_dir)
+        projects.update(self._write_projects_for_sources(self._progs_to_paths,
+            "binary", out_proj_dir))
+
+        # Generate projects that can be used to build common targets.
+        for target in ('export', 'binaries', 'tools', 'full'):
+            basename = 'target_%s' % target
+            command = '$(SolutionDir)\\mach.bat build'
+            if target != 'full':
+                command += ' %s' % target
+
+            project_id = self._write_vs_project(out_proj_dir, basename, target,
+                build_command=command,
+                clean_command='$(SolutionDir)\\mach.bat build clean')
+
+            projects[basename] = (project_id, basename, target)
+
+        # A project that can be used to regenerate the visual studio projects.
+        basename = 'target_vs'
+        project_id = self._write_vs_project(out_proj_dir, basename, 'visual-studio',
+            build_command='$(SolutionDir)\\mach.bat build-backend -b VisualStudio')
+        projects[basename] = (project_id, basename, 'visual-studio')
+
+        # Write out a shared property file with common variables.
+        props_path = os.path.join(out_proj_dir, 'mozilla.props')
+        with self._write_file(props_path, mode='rb') as fh:
+            self._write_props(fh)
+
+        # Generate some wrapper scripts that allow us to invoke mach inside
+        # a MozillaBuild-like environment. We currently only use the batch
+        # script. We'd like to use the PowerShell script. However, it seems
+        # to buffer output from within Visual Studio (surely this is
+        # configurable) and the default execution policy of PowerShell doesn't
+        # allow custom scripts to be executed.
+        with self._write_file(os.path.join(out_dir, 'mach.bat'), mode='rb') as fh:
+            self._write_mach_batch(fh)
+
+        with self._write_file(os.path.join(out_dir, 'mach.ps1'), mode='rb') as fh:
+            self._write_mach_powershell(fh)
+
+        # Write out a solution file to tie it all together.
+        solution_path = os.path.join(out_dir, 'mozilla.sln')
+        with self._write_file(solution_path, mode='rb') as fh:
+            self._write_solution(fh, projects)
+
+    def _write_projects_for_sources(self, sources, prefix, out_dir):
+        projects = {}
+        for item, path in sorted(sources.items()):
+            config = self._paths_to_configs.get(path, None)
+            sources = self._paths_to_sources.get(path, set())
+            sources = set(os.path.join('$(TopSrcDir)', path, s) for s in sources)
+            sources = set(os.path.normpath(s) for s in sources)
+
+            finder = FileFinder(os.path.join(self.environment.topsrcdir, path),
+                find_executables=False)
+
+            headers = [t[0] for t in finder.find('*.h')]
+            headers = [os.path.normpath(os.path.join('$(TopSrcDir)',
+                path, f)) for f in headers]
+
+            includes = [
+                os.path.join('$(TopSrcDir)', path),
+                os.path.join('$(TopObjDir)', path),
+            ]
+            includes.extend(self._paths_to_includes.get(path, []))
+            includes.append('$(TopObjDir)\\dist\\include\\nss')
+            includes.append('$(TopObjDir)\\dist\\include')
+
+            for v in ('NSPR_CFLAGS', 'NSS_CFLAGS', 'MOZ_JPEG_CFLAGS',
+                    'MOZ_PNG_CFLAGS', 'MOZ_ZLIB_CFLAGS', 'MOZ_PIXMAN_CFLAGS'):
+                if not config:
+                    break
+
+                args = config.substs.get(v, [])
+
+                for i, arg in enumerate(args):
+                    if arg.startswith('-I'):
+                        includes.append(os.path.normpath(arg[2:]))
+
+            # Pull in system defaults.
+            includes.append('$(DefaultIncludes)')
+
+            includes = [os.path.normpath(i) for i in includes]
+
+            defines = []
+            for k, v in self._paths_to_defines.get(path, {}).items():
+                if v is True:
+                    defines.append(k)
+                else:
+                    defines.append('%s=%s' % (k, v))
+
+            debugger=None
+            if prefix == 'binary':
+                if item.startswith(self.environment.substs['MOZ_APP_NAME']):
+                    debugger = ('$(TopObjDir)\\dist\\bin\\%s' % item, '-no-remote')
+                else:
+                    debugger = ('$(TopObjDir)\\dist\\bin\\%s' % item, '')
+
+            basename = '%s_%s' % (prefix, item)
+
+            project_id = self._write_vs_project(out_dir, basename, item,
+                includes=includes,
+                forced_includes=['$(TopObjDir)\\dist\\include\\mozilla-config.h'],
+                defines=defines,
+                headers=headers,
+                sources=sources,
+                debugger=debugger)
+
+            projects[basename] = (project_id, basename, item)
+
+        return projects
+
+    def _write_solution(self, fh, projects):
+        # Visual Studio appears to write out its current version in the
+        # solution file. Instead of trying to figure out what version it will
+        # write, try to parse the version out of the existing file and use it
+        # verbatim.
+        vs_version = None
+        try:
+            with open(fh.name, 'rb') as sfh:
+                for line in sfh:
+                    if line.startswith(b'VisualStudioVersion = '):
+                        vs_version = line.split(b' = ', 1)[1].strip()
+        except IOError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+        format_version, comment_version = visual_studio_product_to_solution_version(self._version)
+        # This is a Visual C++ Project type.
+        project_type = '8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942'
+
+        # Visual Studio seems to require this header.
+        fh.write('Microsoft Visual Studio Solution File, Format Version %s\r\n' %
+                 format_version)
+        fh.write('# Visual Studio %s\r\n' % comment_version)
+
+        if vs_version:
+            fh.write('VisualStudioVersion = %s\r\n' % vs_version)
+
+        # Corresponds to VS2013.
+        fh.write('MinimumVisualStudioVersion = 12.0.31101.0\r\n')
+
+        binaries_id = projects['target_binaries'][0]
+
+        # Write out entries for each project.
+        for key in sorted(projects):
+            project_id, basename, name = projects[key]
+            path = os.path.join(self._projsubdir, '%s.vcxproj' % basename)
+
+            fh.write('Project("{%s}") = "%s", "%s", "{%s}"\r\n' % (
+                project_type, name, path, project_id))
+
+            # Make all libraries depend on the binaries target.
+            if key.startswith('library_'):
+                fh.write('\tProjectSection(ProjectDependencies) = postProject\r\n')
+                fh.write('\t\t{%s} = {%s}\r\n' % (binaries_id, binaries_id))
+                fh.write('\tEndProjectSection\r\n')
+
+            fh.write('EndProject\r\n')
+
+        # Write out solution folders for organizing things.
+
+        # This is the UUID you use for solution folders.
+        container_id = '2150E333-8FDC-42A3-9474-1A3956D46DE8'
+
+        def write_container(desc):
+            cid = get_id(desc.encode('utf-8'))
+            fh.write('Project("{%s}") = "%s", "%s", "{%s}"\r\n' % (
+                container_id, desc, desc, cid))
+            fh.write('EndProject\r\n')
+
+            return cid
+
+        library_id = write_container('Libraries')
+        target_id = write_container('Build Targets')
+        binary_id = write_container('Binaries')
+
+        fh.write('Global\r\n')
+
+        # Make every project a member of our one configuration.
+        fh.write('\tGlobalSection(SolutionConfigurationPlatforms) = preSolution\r\n')
+        fh.write('\t\tBuild|Win32 = Build|Win32\r\n')
+        fh.write('\tEndGlobalSection\r\n')
+
+        # Set every project's active configuration to the one configuration and
+        # set up the default build project.
+        fh.write('\tGlobalSection(ProjectConfigurationPlatforms) = postSolution\r\n')
+        for name, project in sorted(projects.items()):
+            fh.write('\t\t{%s}.Build|Win32.ActiveCfg = Build|Win32\r\n' % project[0])
+
+            # Only build the full build target by default.
+            # It's important we don't write multiple entries here because they
+            # conflict!
+            if name == 'target_full':
+                fh.write('\t\t{%s}.Build|Win32.Build.0 = Build|Win32\r\n' % project[0])
+
+        fh.write('\tEndGlobalSection\r\n')
+
+        fh.write('\tGlobalSection(SolutionProperties) = preSolution\r\n')
+        fh.write('\t\tHideSolutionNode = FALSE\r\n')
+        fh.write('\tEndGlobalSection\r\n')
+
+        # Associate projects with containers.
+        fh.write('\tGlobalSection(NestedProjects) = preSolution\r\n')
+        for key in sorted(projects):
+            project_id = projects[key][0]
+
+            if key.startswith('library_'):
+                container_id = library_id
+            elif key.startswith('target_'):
+                container_id = target_id
+            elif key.startswith('binary_'):
+                container_id = binary_id
+            else:
+                raise Exception('Unknown project type: %s' % key)
+
+            fh.write('\t\t{%s} = {%s}\r\n' % (project_id, container_id))
+        fh.write('\tEndGlobalSection\r\n')
+
+        fh.write('EndGlobal\r\n')
+
+    def _write_props(self, fh):
+        impl = getDOMImplementation()
+        doc = impl.createDocument(MSBUILD_NAMESPACE, 'Project', None)
+
+        project = doc.documentElement
+        project.setAttribute('xmlns', MSBUILD_NAMESPACE)
+        project.setAttribute('ToolsVersion', '4.0')
+
+        ig = project.appendChild(doc.createElement('ImportGroup'))
+        ig.setAttribute('Label', 'PropertySheets')
+
+        pg = project.appendChild(doc.createElement('PropertyGroup'))
+        pg.setAttribute('Label', 'UserMacros')
+
+        ig = project.appendChild(doc.createElement('ItemGroup'))
+
+        def add_var(k, v):
+            e = pg.appendChild(doc.createElement(k))
+            e.appendChild(doc.createTextNode(v))
+
+            e = ig.appendChild(doc.createElement('BuildMacro'))
+            e.setAttribute('Include', k)
+
+            e = e.appendChild(doc.createElement('Value'))
+            e.appendChild(doc.createTextNode('$(%s)' % k))
+
+        add_var('TopObjDir', os.path.normpath(self.environment.topobjdir))
+        add_var('TopSrcDir', os.path.normpath(self.environment.topsrcdir))
+        add_var('PYTHON', '$(TopObjDir)\\_virtualenv\\Scripts\\python.exe')
+        add_var('MACH', '$(TopSrcDir)\\mach')
+
+        # From MozillaBuild.
+        add_var('DefaultIncludes', os.environ.get('INCLUDE', ''))
+
+        fh.write(b'\xef\xbb\xbf')
+        doc.writexml(fh, addindent='  ', newl='\r\n')
+
+    def _relevant_environment_variables(self):
+        # Write out the environment variables, presumably coming from
+        # MozillaBuild.
+        for k, v in sorted(os.environ.items()):
+            if not re.match('^[a-zA-Z0-9_]+$', k):
+                continue
+
+            if k in ('OLDPWD', 'PS1'):
+                continue
+
+            if k.startswith('_'):
+                continue
+
+            yield k, v
+
+        yield 'TOPSRCDIR', self.environment.topsrcdir
+        yield 'TOPOBJDIR', self.environment.topobjdir
+
+    def _write_mach_powershell(self, fh):
+        for k, v in self._relevant_environment_variables():
+            fh.write(b'$env:%s = "%s"\r\n' % (k, v))
+
+        relpath = os.path.relpath(self.environment.topsrcdir,
+            self.environment.topobjdir).replace('\\', '/')
+
+        fh.write(b'$bashargs = "%s/mach", "--log-no-times"\r\n' % relpath)
+        fh.write(b'$bashargs = $bashargs + $args\r\n')
+
+        fh.write(b"$expanded = $bashargs -join ' '\r\n")
+        fh.write(b'$procargs = "-c", $expanded\r\n')
+
+        fh.write(b'Start-Process -WorkingDirectory $env:TOPOBJDIR '
+            b'-FilePath $env:MOZILLABUILD\\msys\\bin\\bash '
+            b'-ArgumentList $procargs '
+            b'-Wait -NoNewWindow\r\n')
+
+    def _write_mach_batch(self, fh):
+        """Write out a batch script that builds the tree.
+
+        The script "bootstraps" into the MozillaBuild environment by setting
+        the environment variables that are active in the current MozillaBuild
+        environment. Then, it builds the tree.
+        """
+        for k, v in self._relevant_environment_variables():
+            fh.write(b'SET "%s=%s"\r\n' % (k, v))
+
+        fh.write(b'cd %TOPOBJDIR%\r\n')
+
+        # We need to convert Windows-native paths to msys paths. Easiest way is
+        # relative paths, since munging c:\ to /c/ is slightly more
+        # complicated.
+        relpath = os.path.relpath(self.environment.topsrcdir,
+            self.environment.topobjdir).replace('\\', '/')
+
+        # We go through mach because it has the logic for choosing the most
+        # appropriate build tool.
+        fh.write(b'"%%MOZILLABUILD%%\\msys\\bin\\bash" '
+            b'-c "%s/mach --log-no-times %%1 %%2 %%3 %%4 %%5 %%6 %%7"' % relpath)
+
+    def _write_vs_project(self, out_dir, basename, name, **kwargs):
+        root = '%s.vcxproj' % basename
+        project_id = get_id(basename.encode('utf-8'))
+
+        with self._write_file(os.path.join(out_dir, root), mode='rb') as fh:
+            project_id, name = VisualStudioBackend.write_vs_project(fh,
+                self._version, project_id, name, **kwargs)
+
+        with self._write_file(os.path.join(out_dir, '%s.user' % root), mode='rb') as fh:
+            fh.write('<?xml version="1.0" encoding="utf-8"?>\r\n')
+            fh.write('<Project ToolsVersion="4.0" xmlns="%s">\r\n' %
+                MSBUILD_NAMESPACE)
+            fh.write('</Project>\r\n')
+
+        return project_id
+
+    @staticmethod
+    def write_vs_project(fh, version, project_id, name, includes=[],
+        forced_includes=[], defines=[],
+        build_command=None, clean_command=None,
+        debugger=None, headers=[], sources=[]):
+
+        impl = getDOMImplementation()
+        doc = impl.createDocument(MSBUILD_NAMESPACE, 'Project', None)
+
+        project = doc.documentElement
+        project.setAttribute('DefaultTargets', 'Build')
+        project.setAttribute('ToolsVersion', '4.0')
+        project.setAttribute('xmlns', MSBUILD_NAMESPACE)
+
+        ig = project.appendChild(doc.createElement('ItemGroup'))
+        ig.setAttribute('Label', 'ProjectConfigurations')
+
+        pc = ig.appendChild(doc.createElement('ProjectConfiguration'))
+        pc.setAttribute('Include', 'Build|Win32')
+
+        c = pc.appendChild(doc.createElement('Configuration'))
+        c.appendChild(doc.createTextNode('Build'))
+
+        p = pc.appendChild(doc.createElement('Platform'))
+        p.appendChild(doc.createTextNode('Win32'))
+
+        pg = project.appendChild(doc.createElement('PropertyGroup'))
+        pg.setAttribute('Label', 'Globals')
+
+        n = pg.appendChild(doc.createElement('ProjectName'))
+        n.appendChild(doc.createTextNode(name))
+
+        k = pg.appendChild(doc.createElement('Keyword'))
+        k.appendChild(doc.createTextNode('MakeFileProj'))
+
+        g = pg.appendChild(doc.createElement('ProjectGuid'))
+        g.appendChild(doc.createTextNode('{%s}' % project_id))
+
+        rn = pg.appendChild(doc.createElement('RootNamespace'))
+        rn.appendChild(doc.createTextNode('mozilla'))
+
+        pts = pg.appendChild(doc.createElement('PlatformToolset'))
+        pts.appendChild(doc.createTextNode(visual_studio_product_to_platform_toolset_version(version)))
+
+        i = project.appendChild(doc.createElement('Import'))
+        i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.Default.props')
+
+        ig = project.appendChild(doc.createElement('ImportGroup'))
+        ig.setAttribute('Label', 'ExtensionTargets')
+
+        ig = project.appendChild(doc.createElement('ImportGroup'))
+        ig.setAttribute('Label', 'ExtensionSettings')
+
+        ig = project.appendChild(doc.createElement('ImportGroup'))
+        ig.setAttribute('Label', 'PropertySheets')
+        i = ig.appendChild(doc.createElement('Import'))
+        i.setAttribute('Project', 'mozilla.props')
+
+        pg = project.appendChild(doc.createElement('PropertyGroup'))
+        pg.setAttribute('Label', 'Configuration')
+        ct = pg.appendChild(doc.createElement('ConfigurationType'))
+        ct.appendChild(doc.createTextNode('Makefile'))
+
+        pg = project.appendChild(doc.createElement('PropertyGroup'))
+        pg.setAttribute('Condition', "'$(Configuration)|$(Platform)'=='Build|Win32'")
+
+        if build_command:
+            n = pg.appendChild(doc.createElement('NMakeBuildCommandLine'))
+            n.appendChild(doc.createTextNode(build_command))
+
+        if clean_command:
+            n = pg.appendChild(doc.createElement('NMakeCleanCommandLine'))
+            n.appendChild(doc.createTextNode(clean_command))
+
+        if includes:
+            n = pg.appendChild(doc.createElement('NMakeIncludeSearchPath'))
+            n.appendChild(doc.createTextNode(';'.join(includes)))
+
+        if forced_includes:
+            n = pg.appendChild(doc.createElement('NMakeForcedIncludes'))
+            n.appendChild(doc.createTextNode(';'.join(forced_includes)))
+
+        if defines:
+            n = pg.appendChild(doc.createElement('NMakePreprocessorDefinitions'))
+            n.appendChild(doc.createTextNode(';'.join(defines)))
+
+        if debugger:
+            n = pg.appendChild(doc.createElement('LocalDebuggerCommand'))
+            n.appendChild(doc.createTextNode(debugger[0]))
+
+            n = pg.appendChild(doc.createElement('LocalDebuggerCommandArguments'))
+            n.appendChild(doc.createTextNode(debugger[1]))
+
+        i = project.appendChild(doc.createElement('Import'))
+        i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.props')
+
+        i = project.appendChild(doc.createElement('Import'))
+        i.setAttribute('Project', '$(VCTargetsPath)\\Microsoft.Cpp.targets')
+
+        # Now add files to the project.
+        ig = project.appendChild(doc.createElement('ItemGroup'))
+        for header in sorted(headers or []):
+            n = ig.appendChild(doc.createElement('ClInclude'))
+            n.setAttribute('Include', header)
+
+        ig = project.appendChild(doc.createElement('ItemGroup'))
+        for source in sorted(sources or []):
+            n = ig.appendChild(doc.createElement('ClCompile'))
+            n.setAttribute('Include', source)
+
+        fh.write(b'\xef\xbb\xbf')
+        doc.writexml(fh, addindent='  ', newl='\r\n')
+
+        return project_id, name

python/mozbuild/mozbuild/base.py

@@ -0,0 +1,850 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import json
+import logging
+import mozpack.path as mozpath
+import multiprocessing
+import os
+import subprocess
+import sys
+import which
+
+from mach.mixin.logging import LoggingMixin
+from mach.mixin.process import ProcessExecutionMixin
+from mozversioncontrol import get_repository_object
+
+from .backend.configenvironment import ConfigEnvironment
+from .controller.clobber import Clobberer
+from .mozconfig import (
+    MozconfigFindException,
+    MozconfigLoadException,
+    MozconfigLoader,
+)
+from .util import memoized_property
+from .virtualenv import VirtualenvManager
+
+
+_config_guess_output = []
+
+
+def ancestors(path):
+    """Emit the parent directories of a path."""
+    while path:
+        yield path
+        newpath = os.path.dirname(path)
+        if newpath == path:
+            break
+        path = newpath
+
+def samepath(path1, path2):
+    if hasattr(os.path, 'samefile'):
+        return os.path.samefile(path1, path2)
+    return os.path.normcase(os.path.realpath(path1)) == \
+        os.path.normcase(os.path.realpath(path2))
+
+class BadEnvironmentException(Exception):
+    """Base class for errors raised when the build environment is not sane."""
+
+
+class BuildEnvironmentNotFoundException(BadEnvironmentException):
+    """Raised when we could not find a build environment."""
+
+
+class ObjdirMismatchException(BadEnvironmentException):
+    """Raised when the current dir is an objdir and doesn't match the mozconfig."""
+    def __init__(self, objdir1, objdir2):
+        self.objdir1 = objdir1
+        self.objdir2 = objdir2
+
+    def __str__(self):
+        return "Objdir mismatch: %s != %s" % (self.objdir1, self.objdir2)
+
+
+class MozbuildObject(ProcessExecutionMixin):
+    """Base class providing basic functionality useful to many modules.
+
+    Modules in this package typically require common functionality such as
+    accessing the current config, getting the location of the source directory,
+    running processes, etc. This classes provides that functionality. Other
+    modules can inherit from this class to obtain this functionality easily.
+    """
+    def __init__(self, topsrcdir, settings, log_manager, topobjdir=None,
+                 mozconfig=MozconfigLoader.AUTODETECT):
+        """Create a new Mozbuild object instance.
+
+        Instances are bound to a source directory, a ConfigSettings instance,
+        and a LogManager instance. The topobjdir may be passed in as well. If
+        it isn't, it will be calculated from the active mozconfig.
+        """
+        self.topsrcdir = mozpath.normsep(topsrcdir)
+        self.settings = settings
+
+        self.populate_logger()
+        self.log_manager = log_manager
+
+        self._make = None
+        self._topobjdir = mozpath.normsep(topobjdir) if topobjdir else topobjdir
+        self._mozconfig = mozconfig
+        self._config_environment = None
+        self._virtualenv_manager = None
+
+    @classmethod
+    def from_environment(cls, cwd=None, detect_virtualenv_mozinfo=True):
+        """Create a MozbuildObject by detecting the proper one from the env.
+
+        This examines environment state like the current working directory and
+        creates a MozbuildObject from the found source directory, mozconfig, etc.
+
+        The role of this function is to identify a topsrcdir, topobjdir, and
+        mozconfig file.
+
+        If the current working directory is inside a known objdir, we always
+        use the topsrcdir and mozconfig associated with that objdir.
+
+        If the current working directory is inside a known srcdir, we use that
+        topsrcdir and look for mozconfigs using the default mechanism, which
+        looks inside environment variables.
+
+        If the current Python interpreter is running from a virtualenv inside
+        an objdir, we use that as our objdir.
+
+        If we're not inside a srcdir or objdir, an exception is raised.
+
+        detect_virtualenv_mozinfo determines whether we should look for a
+        mozinfo.json file relative to the virtualenv directory. This was
+        added to facilitate testing. Callers likely shouldn't change the
+        default.
+        """
+
+        cwd = cwd or os.getcwd()
+        topsrcdir = None
+        topobjdir = None
+        mozconfig = MozconfigLoader.AUTODETECT
+
+        def load_mozinfo(path):
+            info = json.load(open(path, 'rt'))
+            topsrcdir = info.get('topsrcdir')
+            topobjdir = os.path.dirname(path)
+            mozconfig = info.get('mozconfig')
+            return topsrcdir, topobjdir, mozconfig
+
+        for dir_path in ancestors(cwd):
+            # If we find a mozinfo.json, we are in the objdir.
+            mozinfo_path = os.path.join(dir_path, 'mozinfo.json')
+            if os.path.isfile(mozinfo_path):
+                topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path)
+                break
+
+            # We choose an arbitrary file as an indicator that this is a
+            # srcdir. We go with ourself because why not!
+            our_path = os.path.join(dir_path, 'python', 'mozbuild', 'mozbuild', 'base.py')
+            if os.path.isfile(our_path):
+                topsrcdir = dir_path
+                break
+
+        # See if we're running from a Python virtualenv that's inside an objdir.
+        mozinfo_path = os.path.join(os.path.dirname(sys.prefix), "mozinfo.json")
+        if detect_virtualenv_mozinfo and os.path.isfile(mozinfo_path):
+            topsrcdir, topobjdir, mozconfig = load_mozinfo(mozinfo_path)
+
+        # If we were successful, we're only guaranteed to find a topsrcdir. If
+        # we couldn't find that, there's nothing we can do.
+        if not topsrcdir:
+            raise BuildEnvironmentNotFoundException(
+                'Could not find Mozilla source tree or build environment.')
+
+        topsrcdir = mozpath.normsep(topsrcdir)
+        if topobjdir:
+            topobjdir = mozpath.normsep(os.path.normpath(topobjdir))
+
+            if topsrcdir == topobjdir:
+                raise BadEnvironmentException('The object directory appears '
+                    'to be the same as your source directory (%s). This build '
+                    'configuration is not supported.' % topsrcdir)
+
+        # If we can't resolve topobjdir, oh well. We'll figure out when we need
+        # one.
+        return cls(topsrcdir, None, None, topobjdir=topobjdir,
+                   mozconfig=mozconfig)
+
+    def resolve_mozconfig_topobjdir(self, default=None):
+        topobjdir = self.mozconfig['topobjdir'] or default
+        if not topobjdir:
+            return None
+
+        if '@CONFIG_GUESS@' in topobjdir:
+            topobjdir = topobjdir.replace('@CONFIG_GUESS@',
+                self.resolve_config_guess())
+
+        if not os.path.isabs(topobjdir):
+            topobjdir = os.path.abspath(os.path.join(self.topsrcdir, topobjdir))
+
+        return mozpath.normsep(os.path.normpath(topobjdir))
+
+    @property
+    def topobjdir(self):
+        if self._topobjdir is None:
+            self._topobjdir = self.resolve_mozconfig_topobjdir(
+                default='obj-@CONFIG_GUESS@')
+
+        return self._topobjdir
+
+    @property
+    def virtualenv_manager(self):
+        if self._virtualenv_manager is None:
+            self._virtualenv_manager = VirtualenvManager(self.topsrcdir,
+                self.topobjdir, os.path.join(self.topobjdir, '_virtualenv'),
+                sys.stdout, os.path.join(self.topsrcdir, 'build',
+                'virtualenv_packages.txt'))
+
+        return self._virtualenv_manager
+
+    @property
+    def mozconfig(self):
+        """Returns information about the current mozconfig file.
+
+        This a dict as returned by MozconfigLoader.read_mozconfig()
+        """
+        if not isinstance(self._mozconfig, dict):
+            loader = MozconfigLoader(self.topsrcdir)
+            self._mozconfig = loader.read_mozconfig(path=self._mozconfig,
+                moz_build_app=os.environ.get('MOZ_CURRENT_PROJECT'))
+
+        return self._mozconfig
+
+    @property
+    def config_environment(self):
+        """Returns the ConfigEnvironment for the current build configuration.
+
+        This property is only available once configure has executed.
+
+        If configure's output is not available, this will raise.
+        """
+        if self._config_environment:
+            return self._config_environment
+
+        config_status = os.path.join(self.topobjdir, 'config.status')
+
+        if not os.path.exists(config_status):
+            raise BuildEnvironmentNotFoundException('config.status not available. Run configure.')
+
+        self._config_environment = \
+            ConfigEnvironment.from_config_status(config_status)
+
+        return self._config_environment
+
+    @property
+    def defines(self):
+        return self.config_environment.defines
+
+    @property
+    def non_global_defines(self):
+        return self.config_environment.non_global_defines
+
+    @property
+    def substs(self):
+        return self.config_environment.substs
+
+    @property
+    def distdir(self):
+        return os.path.join(self.topobjdir, 'dist')
+
+    @property
+    def bindir(self):
+        return os.path.join(self.topobjdir, 'dist', 'bin')
+
+    @property
+    def includedir(self):
+        return os.path.join(self.topobjdir, 'dist', 'include')
+
+    @property
+    def statedir(self):
+        return os.path.join(self.topobjdir, '.mozbuild')
+
+    @memoized_property
+    def extra_environment_variables(self):
+        '''Some extra environment variables are stored in .mozconfig.mk.
+        This functions extracts and returns them.'''
+        from mozbuild import shellutil
+        mozconfig_mk = os.path.join(self.topobjdir, '.mozconfig.mk')
+        env = {}
+        with open(mozconfig_mk) as fh:
+            for line in fh:
+                if line.startswith('export '):
+                    exports = shellutil.split(line)[1:]
+                    for e in exports:
+                        if '=' in e:
+                            key, value = e.split('=')
+                            env[key] = value
+        return env
+
+    @memoized_property
+    def repository(self):
+        '''Get a `mozversioncontrol.Repository` object for the
+        top source directory.'''
+        return get_repository_object(self.topsrcdir)
+
+    def is_clobber_needed(self):
+        if not os.path.exists(self.topobjdir):
+            return False
+        return Clobberer(self.topsrcdir, self.topobjdir).clobber_needed()
+
+    def get_binary_path(self, what='app', validate_exists=True, where='default'):
+        """Obtain the path to a compiled binary for this build configuration.
+
+        The what argument is the program or tool being sought after. See the
+        code implementation for supported values.
+
+        If validate_exists is True (the default), we will ensure the found path
+        exists before returning, raising an exception if it doesn't.
+
+        If where is 'staged-package', we will return the path to the binary in
+        the package staging directory.
+
+        If no arguments are specified, we will return the main binary for the
+        configured XUL application.
+        """
+
+        if where not in ('default', 'staged-package'):
+            raise Exception("Don't know location %s" % where)
+
+        substs = self.substs
+
+        stem = self.distdir
+        if where == 'staged-package':
+            stem = os.path.join(stem, substs['MOZ_APP_NAME'])
+
+        if substs['OS_ARCH'] == 'Darwin':
+            if substs['MOZ_BUILD_APP'] == 'xulrunner':
+                stem = os.path.join(stem, 'XUL.framework');
+            else:
+                stem = os.path.join(stem, substs['MOZ_MACBUNDLE_NAME'], 'Contents',
+                    'MacOS')
+        elif where == 'default':
+            stem = os.path.join(stem, 'bin')
+
+        leaf = None
+
+        leaf = (substs['MOZ_APP_NAME'] if what == 'app' else what) + substs['BIN_SUFFIX']
+        path = os.path.join(stem, leaf)
+
+        if validate_exists and not os.path.exists(path):
+            raise Exception('Binary expected at %s does not exist.' % path)
+
+        return path
+
+    def resolve_config_guess(self):
+        make_extra = self.mozconfig['make_extra'] or []
+        make_extra = dict(m.split('=', 1) for m in make_extra)
+
+        config_guess = make_extra.get('CONFIG_GUESS', None)
+
+        if config_guess:
+            return config_guess
+
+        # config.guess results should be constant for process lifetime. Cache
+        # it.
+        if _config_guess_output:
+            return _config_guess_output[0]
+
+        p = os.path.join(self.topsrcdir, 'build', 'autoconf', 'config.guess')
+
+        # This is a little kludgy. We need access to the normalize_command
+        # function. However, that's a method of a mach mixin, so we need a
+        # class instance. Ideally the function should be accessible as a
+        # standalone function.
+        o = MozbuildObject(self.topsrcdir, None, None, None)
+        args = o._normalize_command([p], True)
+
+        _config_guess_output.append(
+                subprocess.check_output(args, cwd=self.topsrcdir).strip())
+        return _config_guess_output[0]
+
+    def notify(self, msg):
+        """Show a desktop notification with the supplied message
+
+        On Linux and Mac, this will show a desktop notification with the message,
+        but on Windows we can only flash the screen.
+        """
+        moz_nospam = os.environ.get('MOZ_NOSPAM')
+        if moz_nospam:
+            return
+
+        try:
+            if sys.platform.startswith('darwin'):
+                try:
+                    notifier = which.which('terminal-notifier')
+                except which.WhichError:
+                    raise Exception('Install terminal-notifier to get '
+                        'a notification when the build finishes.')
+                self.run_process([notifier, '-title',
+                    'Mozilla Build System', '-group', 'mozbuild',
+                    '-message', msg], ensure_exit_code=False)
+            elif sys.platform.startswith('linux'):
+                try:
+                    import dbus
+                except ImportError:
+                    raise Exception('Install the python dbus module to '
+                        'get a notification when the build finishes.')
+                bus = dbus.SessionBus()
+                notify = bus.get_object('org.freedesktop.Notifications',
+                                        '/org/freedesktop/Notifications')
+                method = notify.get_dbus_method('Notify',
+                                                'org.freedesktop.Notifications')
+                method('Mozilla Build System', 0, '', msg, '', [], [], -1)
+            elif sys.platform.startswith('win'):
+                from ctypes import Structure, windll, POINTER, sizeof
+                from ctypes.wintypes import DWORD, HANDLE, WINFUNCTYPE, BOOL, UINT
+                class FLASHWINDOW(Structure):
+                    _fields_ = [("cbSize", UINT),
+                                ("hwnd", HANDLE),
+                                ("dwFlags", DWORD),
+                                ("uCount", UINT),
+                                ("dwTimeout", DWORD)]
+                FlashWindowExProto = WINFUNCTYPE(BOOL, POINTER(FLASHWINDOW))
+                FlashWindowEx = FlashWindowExProto(("FlashWindowEx", windll.user32))
+                FLASHW_CAPTION = 0x01
+                FLASHW_TRAY = 0x02
+                FLASHW_TIMERNOFG = 0x0C
+
+                # GetConsoleWindows returns NULL if no console is attached. We
+                # can't flash nothing.
+                console = windll.kernel32.GetConsoleWindow()
+                if not console:
+                    return
+
+                params = FLASHWINDOW(sizeof(FLASHWINDOW),
+                                    console,
+                                    FLASHW_CAPTION | FLASHW_TRAY | FLASHW_TIMERNOFG, 3, 0)
+                FlashWindowEx(params)
+        except Exception as e:
+            self.log(logging.WARNING, 'notifier-failed', {'error':
+                e.message}, 'Notification center failed: {error}')
+
+    def _ensure_objdir_exists(self):
+        if os.path.isdir(self.statedir):
+            return
+
+        os.makedirs(self.statedir)
+
+    def _ensure_state_subdir_exists(self, subdir):
+        path = os.path.join(self.statedir, subdir)
+
+        if os.path.isdir(path):
+            return
+
+        os.makedirs(path)
+
+    def _get_state_filename(self, filename, subdir=None):
+        path = self.statedir
+
+        if subdir:
+            path = os.path.join(path, subdir)
+
+        return os.path.join(path, filename)
+
+    def _wrap_path_argument(self, arg):
+        return PathArgument(arg, self.topsrcdir, self.topobjdir)
+
+    def _run_make(self, directory=None, filename=None, target=None, log=True,
+            srcdir=False, allow_parallel=True, line_handler=None,
+            append_env=None, explicit_env=None, ignore_errors=False,
+            ensure_exit_code=0, silent=True, print_directory=True,
+            pass_thru=False, num_jobs=0):
+        """Invoke make.
+
+        directory -- Relative directory to look for Makefile in.
+        filename -- Explicit makefile to run.
+        target -- Makefile target(s) to make. Can be a string or iterable of
+            strings.
+        srcdir -- If True, invoke make from the source directory tree.
+            Otherwise, make will be invoked from the object directory.
+        silent -- If True (the default), run make in silent mode.
+        print_directory -- If True (the default), have make print directories
+        while doing traversal.
+        """
+        self._ensure_objdir_exists()
+
+        args = self._make_path()
+
+        if directory:
+            args.extend(['-C', directory.replace(os.sep, '/')])
+
+        if filename:
+            args.extend(['-f', filename])
+
+        if num_jobs == 0 and self.mozconfig['make_flags']:
+            flags = iter(self.mozconfig['make_flags'])
+            for flag in flags:
+                if flag == '-j':
+                    try:
+                        flag = flags.next()
+                    except StopIteration:
+                        break
+                    try:
+                        num_jobs = int(flag)
+                    except ValueError:
+                        args.append(flag)
+                elif flag.startswith('-j'):
+                    try:
+                        num_jobs = int(flag[2:])
+                    except (ValueError, IndexError):
+                        break
+                else:
+                    args.append(flag)
+
+        if allow_parallel:
+            if num_jobs > 0:
+                args.append('-j%d' % num_jobs)
+            else:
+                args.append('-j%d' % multiprocessing.cpu_count())
+        elif num_jobs > 0:
+            args.append('MOZ_PARALLEL_BUILD=%d' % num_jobs)
+
+        if ignore_errors:
+            args.append('-k')
+
+        if silent:
+            args.append('-s')
+
+        # Print entering/leaving directory messages. Some consumers look at
+        # these to measure progress.
+        if print_directory:
+            args.append('-w')
+
+        if isinstance(target, list):
+            args.extend(target)
+        elif target:
+            args.append(target)
+
+        fn = self._run_command_in_objdir
+
+        if srcdir:
+            fn = self._run_command_in_srcdir
+
+        append_env = dict(append_env or ())
+        append_env[b'MACH'] = '1'
+
+        params = {
+            'args': args,
+            'line_handler': line_handler,
+            'append_env': append_env,
+            'explicit_env': explicit_env,
+            'log_level': logging.INFO,
+            'require_unix_environment': False,
+            'ensure_exit_code': ensure_exit_code,
+            'pass_thru': pass_thru,
+
+            # Make manages its children, so mozprocess doesn't need to bother.
+            # Having mozprocess manage children can also have side-effects when
+            # building on Windows. See bug 796840.
+            'ignore_children': True,
+        }
+
+        if log:
+            params['log_name'] = 'make'
+
+        return fn(**params)
+
+    def _make_path(self):
+        baseconfig = os.path.join(self.topsrcdir, 'config', 'baseconfig.mk')
+
+        def is_xcode_lisense_error(output):
+            return self._is_osx() and 'Agreeing to the Xcode' in output
+
+        def validate_make(make):
+            if os.path.exists(baseconfig) and os.path.exists(make):
+                cmd = [make, '-f', baseconfig]
+                if self._is_windows():
+                    cmd.append('HOST_OS_ARCH=WINNT')
+                try:
+                    subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+                except subprocess.CalledProcessError as e:
+                    return False, is_xcode_lisense_error(e.output)
+                return True, False
+            return False, False
+
+        xcode_lisense_error = False
+        possible_makes = ['gmake', 'make', 'mozmake', 'gnumake', 'mingw32-make']
+
+        if 'MAKE' in os.environ:
+            make = os.environ['MAKE']
+            possible_makes.insert(0, make)
+
+        for test in possible_makes:
+            if os.path.isabs(test):
+                make = test
+            else:
+                try:
+                    make = which.which(test)
+                except which.WhichError:
+                    continue
+            result, xcode_lisense_error_tmp = validate_make(make)
+            if result:
+                return [make]
+            if xcode_lisense_error_tmp:
+                xcode_lisense_error = True
+
+        if xcode_lisense_error:
+            raise Exception('Xcode requires accepting to the license agreement.\n'
+                'Please run Xcode and accept the license agreement.')
+
+        if self._is_windows():
+            raise Exception('Could not find a suitable make implementation.\n'
+                'Please use MozillaBuild 1.9 or newer')
+        else:
+            raise Exception('Could not find a suitable make implementation.')
+
+    def _run_command_in_srcdir(self, **args):
+        return self.run_process(cwd=self.topsrcdir, **args)
+
+    def _run_command_in_objdir(self, **args):
+        return self.run_process(cwd=self.topobjdir, **args)
+
+    def _is_windows(self):
+        return os.name in ('nt', 'ce')
+
+    def _is_osx(self):
+        return 'darwin' in str(sys.platform).lower()
+
+    def _spawn(self, cls):
+        """Create a new MozbuildObject-derived class instance from ourselves.
+
+        This is used as a convenience method to create other
+        MozbuildObject-derived class instances. It can only be used on
+        classes that have the same constructor arguments as us.
+        """
+
+        return cls(self.topsrcdir, self.settings, self.log_manager,
+            topobjdir=self.topobjdir)
+
+    def _activate_virtualenv(self):
+        self.virtualenv_manager.ensure()
+        self.virtualenv_manager.activate()
+
+
+class MachCommandBase(MozbuildObject):
+    """Base class for mach command providers that wish to be MozbuildObjects.
+
+    This provides a level of indirection so MozbuildObject can be refactored
+    without having to change everything that inherits from it.
+    """
+
+    def __init__(self, context):
+        # Attempt to discover topobjdir through environment detection, as it is
+        # more reliable than mozconfig when cwd is inside an objdir.
+        topsrcdir = context.topdir
+        topobjdir = None
+        detect_virtualenv_mozinfo = True
+        if hasattr(context, 'detect_virtualenv_mozinfo'):
+            detect_virtualenv_mozinfo = getattr(context,
+                'detect_virtualenv_mozinfo')
+        try:
+            dummy = MozbuildObject.from_environment(cwd=context.cwd,
+                detect_virtualenv_mozinfo=detect_virtualenv_mozinfo)
+            topsrcdir = dummy.topsrcdir
+            topobjdir = dummy._topobjdir
+            if topobjdir:
+                # If we're inside a objdir and the found mozconfig resolves to
+                # another objdir, we abort. The reasoning here is that if you
+                # are inside an objdir you probably want to perform actions on
+                # that objdir, not another one. This prevents accidental usage
+                # of the wrong objdir when the current objdir is ambiguous.
+                config_topobjdir = dummy.resolve_mozconfig_topobjdir()
+
+                try:
+                    universal_bin = dummy.substs.get('UNIVERSAL_BINARY')
+                except:
+                    universal_bin = False
+
+                if config_topobjdir and not (samepath(topobjdir, config_topobjdir) or
+                        universal_bin and topobjdir.startswith(config_topobjdir)):
+                    raise ObjdirMismatchException(topobjdir, config_topobjdir)
+        except BuildEnvironmentNotFoundException:
+            pass
+        except ObjdirMismatchException as e:
+            print('Ambiguous object directory detected. We detected that '
+                'both %s and %s could be object directories. This is '
+                'typically caused by having a mozconfig pointing to a '
+                'different object directory from the current working '
+                'directory. To solve this problem, ensure you do not have a '
+                'default mozconfig in searched paths.' % (e.objdir1,
+                    e.objdir2))
+            sys.exit(1)
+
+        except MozconfigLoadException as e:
+            print('Error loading mozconfig: ' + e.path)
+            print('')
+            print(e.message)
+            if e.output:
+                print('')
+                print('mozconfig output:')
+                print('')
+                for line in e.output:
+                    print(line)
+
+            sys.exit(1)
+
+        MozbuildObject.__init__(self, topsrcdir, context.settings,
+            context.log_manager, topobjdir=topobjdir)
+
+        self._mach_context = context
+
+        # Incur mozconfig processing so we have unified error handling for
+        # errors. Otherwise, the exceptions could bubble back to mach's error
+        # handler.
+        try:
+            self.mozconfig
+
+        except MozconfigFindException as e:
+            print(e.message)
+            sys.exit(1)
+
+        except MozconfigLoadException as e:
+            print('Error loading mozconfig: ' + e.path)
+            print('')
+            print(e.message)
+            if e.output:
+                print('')
+                print('mozconfig output:')
+                print('')
+                for line in e.output:
+                    print(line)
+
+            sys.exit(1)
+
+        # Always keep a log of the last command, but don't do that for mach
+        # invokations from scripts (especially not the ones done by the build
+        # system itself).
+        if (os.isatty(sys.stdout.fileno()) and
+                not getattr(self, 'NO_AUTO_LOG', False)):
+            self._ensure_state_subdir_exists('.')
+            logfile = self._get_state_filename('last_log.json')
+            try:
+                fd = open(logfile, "wb")
+                self.log_manager.add_json_handler(fd)
+            except Exception as e:
+                self.log(logging.WARNING, 'mach', {'error': e},
+                         'Log will not be kept for this command: {error}.')
+
+
+class MachCommandConditions(object):
+    """A series of commonly used condition functions which can be applied to
+    mach commands with providers deriving from MachCommandBase.
+    """
+    @staticmethod
+    def is_firefox(cls):
+        """Must have a Firefox build."""
+        if hasattr(cls, 'substs'):
+            return cls.substs.get('MOZ_BUILD_APP') == 'browser'
+        return False
+
+    @staticmethod
+    def is_mulet(cls):
+        """Must have a Mulet build."""
+        if hasattr(cls, 'substs'):
+            return cls.substs.get('MOZ_BUILD_APP') == 'b2g/dev'
+        return False
+
+    @staticmethod
+    def is_b2g(cls):
+        """Must have a B2G build."""
+        if hasattr(cls, 'substs'):
+            return cls.substs.get('MOZ_WIDGET_TOOLKIT') == 'gonk'
+        return False
+
+    @staticmethod
+    def is_b2g_desktop(cls):
+        """Must have a B2G desktop build."""
+        if hasattr(cls, 'substs'):
+            return cls.substs.get('MOZ_BUILD_APP') == 'b2g' and \
+                   cls.substs.get('MOZ_WIDGET_TOOLKIT') != 'gonk'
+        return False
+
+    @staticmethod
+    def is_emulator(cls):
+        """Must have a B2G build with an emulator configured."""
+        try:
+            return MachCommandConditions.is_b2g(cls) and \
+                   cls.device_name.startswith('emulator')
+        except AttributeError:
+            return False
+
+    @staticmethod
+    def is_android(cls):
+        """Must have an Android build."""
+        if hasattr(cls, 'substs'):
+            return cls.substs.get('MOZ_WIDGET_TOOLKIT') == 'android'
+        return False
+
+    @staticmethod
+    def is_hg(cls):
+        """Must have a mercurial source checkout."""
+        if hasattr(cls, 'substs'):
+            top_srcdir = cls.substs.get('top_srcdir')
+            return top_srcdir and os.path.isdir(os.path.join(top_srcdir, '.hg'))
+        return False
+
+    @staticmethod
+    def is_git(cls):
+        """Must have a git source checkout."""
+        if hasattr(cls, 'substs'):
+            top_srcdir = cls.substs.get('top_srcdir')
+            return top_srcdir and os.path.isdir(os.path.join(top_srcdir, '.git'))
+        return False
+
+
+class PathArgument(object):
+    """Parse a filesystem path argument and transform it in various ways."""
+
+    def __init__(self, arg, topsrcdir, topobjdir, cwd=None):
+        self.arg = arg
+        self.topsrcdir = topsrcdir
+        self.topobjdir = topobjdir
+        self.cwd = os.getcwd() if cwd is None else cwd
+
+    def relpath(self):
+        """Return a path relative to the topsrcdir or topobjdir.
+
+        If the argument is a path to a location in one of the base directories
+        (topsrcdir or topobjdir), then strip off the base directory part and
+        just return the path within the base directory."""
+
+        abspath = os.path.abspath(os.path.join(self.cwd, self.arg))
+
+        # If that path is within topsrcdir or topobjdir, return an equivalent
+        # path relative to that base directory.
+        for base_dir in [self.topobjdir, self.topsrcdir]:
+            if abspath.startswith(os.path.abspath(base_dir)):
+                return mozpath.relpath(abspath, base_dir)
+
+        return mozpath.normsep(self.arg)
+
+    def srcdir_path(self):
+        return mozpath.join(self.topsrcdir, self.relpath())
+
+    def objdir_path(self):
+        return mozpath.join(self.topobjdir, self.relpath())
+
+
+class ExecutionSummary(dict):
+    """Helper for execution summaries."""
+
+    def __init__(self, summary_format, **data):
+        self._summary_format = ''
+        assert 'execution_time' in data
+        self.extend(summary_format, **data)
+
+    def extend(self, summary_format, **data):
+        self._summary_format += summary_format
+        self.update(data)
+
+    def __str__(self):
+        return self._summary_format.format(**self)
+
+    def __getattr__(self, key):
+        return self[key]

python/mozbuild/mozbuild/codecoverage/chrome_map.py

@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from collections import defaultdict
+import json
+import os
+import urlparse
+
+from mach.config import ConfigSettings
+from mach.logging import LoggingManager
+from mozbuild.backend.common import CommonBackend
+from mozbuild.base import MozbuildObject
+from mozbuild.frontend.data import (
+    FinalTargetFiles,
+    FinalTargetPreprocessedFiles,
+)
+from mozbuild.frontend.data import JARManifest, ChromeManifestEntry
+from mozpack.chrome.manifest import (
+    Manifest,
+    ManifestChrome,
+    ManifestOverride,
+    ManifestResource,
+    parse_manifest,
+)
+import mozpack.path as mozpath
+
+
+class ChromeManifestHandler(object):
+    def __init__(self):
+        self.overrides = {}
+        self.chrome_mapping = defaultdict(set)
+
+    def handle_manifest_entry(self, entry):
+        format_strings = {
+            "content": "chrome://%s/content/",
+            "resource": "resource://%s/",
+            "locale": "chrome://%s/locale/",
+            "skin": "chrome://%s/skin/",
+        }
+
+        if isinstance(entry, (ManifestChrome, ManifestResource)):
+            if isinstance(entry, ManifestResource):
+                dest = entry.target
+                url = urlparse.urlparse(dest)
+                if not url.scheme:
+                    dest = mozpath.normpath(mozpath.join(entry.base, dest))
+                if url.scheme == 'file':
+                    dest = mozpath.normpath(url.path)
+            else:
+                dest = mozpath.normpath(entry.path)
+
+            base_uri = format_strings[entry.type] % entry.name
+            self.chrome_mapping[base_uri].add(dest)
+        if isinstance(entry, ManifestOverride):
+            self.overrides[entry.overloaded] = entry.overload
+        if isinstance(entry, Manifest):
+            for e in parse_manifest(None, entry.path):
+                self.handle_manifest_entry(e)
+
+class ChromeMapBackend(CommonBackend):
+    def _init(self):
+        CommonBackend._init(self)
+
+        log_manager = LoggingManager()
+        self._cmd = MozbuildObject(self.environment.topsrcdir, ConfigSettings(),
+                                   log_manager, self.environment.topobjdir)
+        self._install_mapping = {}
+        self.manifest_handler = ChromeManifestHandler()
+
+    def consume_object(self, obj):
+        if isinstance(obj, JARManifest):
+            self._consume_jar_manifest(obj)
+        if isinstance(obj, ChromeManifestEntry):
+            self.manifest_handler.handle_manifest_entry(obj.entry)
+        if isinstance(obj, (FinalTargetFiles,
+                            FinalTargetPreprocessedFiles)):
+            self._handle_final_target_files(obj)
+        return True
+
+    def _handle_final_target_files(self, obj):
+        for path, files in obj.files.walk():
+            for f in files:
+                dest = mozpath.join(obj.install_target, path, f.target_basename)
+                is_pp = isinstance(obj,
+                                   FinalTargetPreprocessedFiles)
+                self._install_mapping[dest] = f.full_path, is_pp
+
+    def consume_finished(self):
+        # Our result has three parts:
+        #  A map from url prefixes to objdir directories:
+        #  { "chrome://mozapps/content/": [ "dist/bin/chrome/toolkit/content/mozapps" ], ... }
+        #  A map of overrides.
+        #  A map from objdir paths to sourcedir paths, and a flag for whether the source was preprocessed:
+        #  { "dist/bin/browser/chrome/browser/content/browser/aboutSessionRestore.js":
+        #    [ "$topsrcdir/browser/components/sessionstore/content/aboutSessionRestore.js", false ], ... }
+        outputfile = os.path.join(self.environment.topobjdir, 'chrome-map.json')
+        with self._write_file(outputfile) as fh:
+            chrome_mapping = self.manifest_handler.chrome_mapping
+            overrides = self.manifest_handler.overrides
+            json.dump([
+                {k: list(v) for k, v in chrome_mapping.iteritems()},
+                overrides,
+                self._install_mapping,
+            ], fh, sort_keys=True, indent=2)

python/mozbuild/mozbuild/codecoverage/packager.py

@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+import argparse
+import sys
+
+from mozpack.files import FileFinder
+from mozpack.copier import Jarrer
+
+def package_gcno_tree(root, output_file):
+    # XXX JarWriter doesn't support unicode strings, see bug 1056859
+    if isinstance(root, unicode):
+        root = root.encode('utf-8')
+
+    finder = FileFinder(root)
+    jarrer = Jarrer(optimize=False)
+    for p, f in finder.find("**/*.gcno"):
+        jarrer.add(p, f)
+    jarrer.copy(output_file)
+
+
+def cli(args=sys.argv[1:]):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--output-file',
+                        dest='output_file',
+                        help='Path to save packaged data to.')
+    parser.add_argument('--root',
+                        dest='root',
+                        default=None,
+                        help='Root directory to search from.')
+    args = parser.parse_args(args)
+
+    if not args.root:
+        from buildconfig import topobjdir
+        args.root = topobjdir
+
+    return package_gcno_tree(args.root, args.output_file)
+
+if __name__ == '__main__':
+    sys.exit(cli())

python/mozbuild/mozbuild/compilation/codecomplete.py

@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This modules provides functionality for dealing with code completion.
+
+from __future__ import absolute_import
+
+import os
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from mozbuild.base import MachCommandBase
+from mozbuild.shellutil import (
+    split as shell_split,
+    quote as shell_quote,
+)
+
+
+@CommandProvider
+class Introspection(MachCommandBase):
+    """Instropection commands."""
+
+    @Command('compileflags', category='devenv',
+        description='Display the compilation flags for a given source file')
+    @CommandArgument('what', default=None,
+        help='Source file to display compilation flags for')
+    def compileflags(self, what):
+        from mozbuild.util import resolve_target_to_make
+        from mozbuild.compilation import util
+
+        if not util.check_top_objdir(self.topobjdir):
+            return 1
+
+        path_arg = self._wrap_path_argument(what)
+
+        make_dir, make_target = resolve_target_to_make(self.topobjdir,
+            path_arg.relpath())
+
+        if make_dir is None and make_target is None:
+            return 1
+
+        build_vars = util.get_build_vars(make_dir, self)
+
+        if what.endswith('.c'):
+            cc = 'CC'
+            name = 'COMPILE_CFLAGS'
+        else:
+            cc = 'CXX'
+            name = 'COMPILE_CXXFLAGS'
+
+        if name not in build_vars:
+            return
+
+        # Drop the first flag since that is the pathname of the compiler.
+        flags = (shell_split(build_vars[cc]) + shell_split(build_vars[name]))[1:]
+
+        print(' '.join(shell_quote(arg)
+                       for arg in util.sanitize_cflags(flags)))

python/mozbuild/mozbuild/compilation/database.py

@@ -0,0 +1,252 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This modules provides functionality for dealing with code completion.
+
+import os
+import types
+
+from mozbuild.compilation import util
+from mozbuild.backend.common import CommonBackend
+from mozbuild.frontend.data import (
+    Sources,
+    GeneratedSources,
+    DirectoryTraversal,
+    Defines,
+    Linkable,
+    LocalInclude,
+    VariablePassthru,
+    SimpleProgram,
+)
+from mozbuild.shellutil import (
+    quote as shell_quote,
+)
+from mozbuild.util import expand_variables
+import mozpack.path as mozpath
+from collections import (
+    defaultdict,
+    OrderedDict,
+)
+
+
+class CompileDBBackend(CommonBackend):
+    def _init(self):
+        CommonBackend._init(self)
+        if not util.check_top_objdir(self.environment.topobjdir):
+            raise Exception()
+
+        # The database we're going to dump out to.
+        self._db = OrderedDict()
+
+        # The cache for per-directory flags
+        self._flags = {}
+
+        self._envs = {}
+        self._includes = defaultdict(list)
+        self._defines = defaultdict(list)
+        self._local_flags = defaultdict(dict)
+        self._extra_includes = defaultdict(list)
+        self._gyp_dirs = set()
+        self._dist_include_testing = '-I%s' % mozpath.join(
+            self.environment.topobjdir, 'dist', 'include', 'testing')
+
+    def consume_object(self, obj):
+        # Those are difficult directories, that will be handled later.
+        if obj.relativedir in (
+                'build/unix/elfhack',
+                'build/unix/elfhack/inject',
+                'build/clang-plugin',
+                'build/clang-plugin/tests',
+                'security/sandbox/win/wow_helper',
+                'toolkit/crashreporter/google-breakpad/src/common'):
+            return True
+
+        consumed = CommonBackend.consume_object(self, obj)
+
+        if consumed:
+            return True
+
+        if isinstance(obj, DirectoryTraversal):
+            self._envs[obj.objdir] = obj.config
+            for var in ('STL_FLAGS', 'VISIBILITY_FLAGS', 'WARNINGS_AS_ERRORS'):
+                value = obj.config.substs.get(var)
+                if value:
+                    self._local_flags[obj.objdir][var] = value
+
+        elif isinstance(obj, (Sources, GeneratedSources)):
+            # For other sources, include each source file.
+            for f in obj.files:
+                self._build_db_line(obj.objdir, obj.relativedir, obj.config, f,
+                                    obj.canonical_suffix)
+
+        elif isinstance(obj, LocalInclude):
+            self._includes[obj.objdir].append('-I%s' % mozpath.normpath(
+                obj.path.full_path))
+
+        elif isinstance(obj, Linkable):
+            if isinstance(obj.defines, Defines): # As opposed to HostDefines
+                for d in obj.defines.get_defines():
+                    if d not in self._defines[obj.objdir]:
+                        self._defines[obj.objdir].append(d)
+            self._defines[obj.objdir].extend(obj.lib_defines.get_defines())
+            if isinstance(obj, SimpleProgram) and obj.is_unit_test:
+                if (self._dist_include_testing not in
+                        self._extra_includes[obj.objdir]):
+                    self._extra_includes[obj.objdir].append(
+                        self._dist_include_testing)
+
+        elif isinstance(obj, VariablePassthru):
+            if obj.variables.get('IS_GYP_DIR'):
+                self._gyp_dirs.add(obj.objdir)
+            for var in ('MOZBUILD_CFLAGS', 'MOZBUILD_CXXFLAGS',
+                        'MOZBUILD_CMFLAGS', 'MOZBUILD_CMMFLAGS',
+                        'RTL_FLAGS', 'VISIBILITY_FLAGS'):
+                if var in obj.variables:
+                    self._local_flags[obj.objdir][var] = obj.variables[var]
+            if (obj.variables.get('DISABLE_STL_WRAPPING') and
+                    'STL_FLAGS' in self._local_flags[obj.objdir]):
+                del self._local_flags[obj.objdir]['STL_FLAGS']
+            if (obj.variables.get('ALLOW_COMPILER_WARNINGS') and
+                    'WARNINGS_AS_ERRORS' in self._local_flags[obj.objdir]):
+                del self._local_flags[obj.objdir]['WARNINGS_AS_ERRORS']
+
+        return True
+
+    def consume_finished(self):
+        CommonBackend.consume_finished(self)
+
+        db = []
+
+        for (directory, filename), cmd in self._db.iteritems():
+            env = self._envs[directory]
+            cmd = list(cmd)
+            cmd.append(filename)
+            local_extra = list(self._extra_includes[directory])
+            if directory not in self._gyp_dirs:
+                for var in (
+                    'NSPR_CFLAGS',
+                    'NSS_CFLAGS',
+                    'MOZ_JPEG_CFLAGS',
+                    'MOZ_PNG_CFLAGS',
+                    'MOZ_ZLIB_CFLAGS',
+                    'MOZ_PIXMAN_CFLAGS',
+                ):
+                    f = env.substs.get(var)
+                    if f:
+                        local_extra.extend(f)
+            variables = {
+                'LOCAL_INCLUDES': self._includes[directory],
+                'DEFINES': self._defines[directory],
+                'EXTRA_INCLUDES': local_extra,
+                'DIST': mozpath.join(env.topobjdir, 'dist'),
+                'DEPTH': env.topobjdir,
+                'MOZILLA_DIR': env.topsrcdir,
+                'topsrcdir': env.topsrcdir,
+                'topobjdir': env.topobjdir,
+            }
+            variables.update(self._local_flags[directory])
+            c = []
+            for a in cmd:
+                a = expand_variables(a, variables).split()
+                if not a:
+                    continue
+                if isinstance(a, types.StringTypes):
+                    c.append(a)
+                else:
+                    c.extend(a)
+            db.append({
+                'directory': directory,
+                'command': ' '.join(shell_quote(a) for a in c),
+                'file': filename,
+            })
+
+        import json
+        # Output the database (a JSON file) to objdir/compile_commands.json
+        outputfile = os.path.join(self.environment.topobjdir, 'compile_commands.json')
+        with self._write_file(outputfile) as jsonout:
+            json.dump(db, jsonout, indent=0)
+
+    def _process_unified_sources(self, obj):
+        # For unified sources, only include the unified source file.
+        # Note that unified sources are never used for host sources.
+        for f in obj.unified_source_mapping:
+            self._build_db_line(obj.objdir, obj.relativedir, obj.config, f[0],
+                                obj.canonical_suffix)
+
+    def _handle_idl_manager(self, idl_manager):
+        pass
+
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+                             unified_ipdl_cppsrcs_mapping):
+        for f in unified_ipdl_cppsrcs_mapping:
+            self._build_db_line(ipdl_dir, None, self.environment, f[0],
+                                '.cpp')
+
+    def _handle_webidl_build(self, bindings_dir, unified_source_mapping,
+                             webidls, expected_build_output_files,
+                             global_define_files):
+        for f in unified_source_mapping:
+            self._build_db_line(bindings_dir, None, self.environment, f[0],
+                                '.cpp')
+
+    COMPILERS = {
+        '.c': 'CC',
+        '.cpp': 'CXX',
+        '.m': 'CC',
+        '.mm': 'CXX',
+    }
+
+    CFLAGS = {
+        '.c': 'CFLAGS',
+        '.cpp': 'CXXFLAGS',
+        '.m': 'CFLAGS',
+        '.mm': 'CXXFLAGS',
+    }
+
+    def _build_db_line(self, objdir, reldir, cenv, filename, canonical_suffix):
+        if canonical_suffix not in self.COMPILERS:
+            return
+        db = self._db.setdefault((objdir, filename),
+            cenv.substs[self.COMPILERS[canonical_suffix]].split() +
+            ['-o', '/dev/null', '-c'])
+        reldir = reldir or mozpath.relpath(objdir, cenv.topobjdir)
+
+        def append_var(name):
+            value = cenv.substs.get(name)
+            if not value:
+                return
+            if isinstance(value, types.StringTypes):
+                value = value.split()
+            db.extend(value)
+
+        if canonical_suffix in ('.mm', '.cpp'):
+            db.append('$(STL_FLAGS)')
+
+        db.extend((
+            '$(VISIBILITY_FLAGS)',
+            '$(DEFINES)',
+            '-I%s' % mozpath.join(cenv.topsrcdir, reldir),
+            '-I%s' % objdir,
+            '$(LOCAL_INCLUDES)',
+            '-I%s/dist/include' % cenv.topobjdir,
+            '$(EXTRA_INCLUDES)',
+        ))
+        append_var('DSO_CFLAGS')
+        append_var('DSO_PIC_CFLAGS')
+        if canonical_suffix in ('.c', '.cpp'):
+            db.append('$(RTL_FLAGS)')
+        append_var('OS_COMPILE_%s' % self.CFLAGS[canonical_suffix])
+        append_var('OS_CPPFLAGS')
+        append_var('OS_%s' % self.CFLAGS[canonical_suffix])
+        append_var('MOZ_DEBUG_FLAGS')
+        append_var('MOZ_OPTIMIZE_FLAGS')
+        append_var('MOZ_FRAMEPTR_FLAGS')
+        db.append('$(WARNINGS_AS_ERRORS)')
+        db.append('$(MOZBUILD_%s)' % self.CFLAGS[canonical_suffix])
+        if canonical_suffix == '.m':
+            append_var('OS_COMPILE_CMFLAGS')
+            db.append('$(MOZBUILD_CMFLAGS)')
+        elif canonical_suffix == '.mm':
+            append_var('OS_COMPILE_CMMFLAGS')
+            db.append('$(MOZBUILD_CMMFLAGS)')

python/mozbuild/mozbuild/compilation/util.py

@@ -0,0 +1,54 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+from mozbuild import shellutil
+
+def check_top_objdir(topobjdir):
+    top_make = os.path.join(topobjdir, 'Makefile')
+    if not os.path.exists(top_make):
+        print('Your tree has not been built yet. Please run '
+            '|mach build| with no arguments.')
+        return False
+    return True
+
+def get_build_vars(directory, cmd):
+    build_vars = {}
+
+    def on_line(line):
+        elements = [s.strip() for s in line.split('=', 1)]
+
+        if len(elements) != 2:
+            return
+
+        build_vars[elements[0]] = elements[1]
+
+    try:
+        old_logger = cmd.log_manager.replace_terminal_handler(None)
+        cmd._run_make(directory=directory, target='showbuild', log=False,
+                print_directory=False, allow_parallel=False, silent=True,
+                line_handler=on_line)
+    finally:
+        cmd.log_manager.replace_terminal_handler(old_logger)
+
+    return build_vars
+
+def sanitize_cflags(flags):
+    # We filter out -Xclang arguments as clang based tools typically choke on
+    # passing these flags down to the clang driver.  -Xclang tells the clang
+    # driver driver to pass whatever comes after it down to clang cc1, which is
+    # why we skip -Xclang and the argument immediately after it.  Here is an
+    # example: the following two invocations pass |-foo -bar -baz| to cc1:
+    # clang -cc1 -foo -bar -baz
+    # clang -Xclang -foo -Xclang -bar -Xclang -baz
+    sanitized = []
+    saw_xclang = False
+    for flag in flags:
+        if flag == '-Xclang':
+            saw_xclang = True
+        elif saw_xclang:
+            saw_xclang = False
+        else:
+            sanitized.append(flag)
+    return sanitized

python/mozbuild/mozbuild/compilation/warnings.py

@@ -0,0 +1,376 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This modules provides functionality for dealing with compiler warnings.
+
+from __future__ import absolute_import, unicode_literals
+
+import errno
+import json
+import os
+import re
+
+from mozbuild.util import hash_file
+import mozpack.path as mozpath
+
+
+# Regular expression to strip ANSI color sequences from a string. This is
+# needed to properly analyze Clang compiler output, which may be colorized.
+# It assumes ANSI escape sequences.
+RE_STRIP_COLORS = re.compile(r'\x1b\[[\d;]+m')
+
+# This captures Clang diagnostics with the standard formatting.
+RE_CLANG_WARNING = re.compile(r"""
+    (?P<file>[^:]+)
+    :
+    (?P<line>\d+)
+    :
+    (?P<column>\d+)
+    :
+    \swarning:\s
+    (?P<message>.+)
+    \[(?P<flag>[^\]]+)
+    """, re.X)
+
+# This captures Visual Studio's warning format.
+RE_MSVC_WARNING = re.compile(r"""
+    (?P<file>.*)
+    \((?P<line>\d+)\)
+    \s?:\swarning\s
+    (?P<flag>[^:]+)
+    :\s
+    (?P<message>.*)
+    """, re.X)
+
+IN_FILE_INCLUDED_FROM = 'In file included from '
+
+
+class CompilerWarning(dict):
+    """Represents an individual compiler warning."""
+
+    def __init__(self):
+        dict.__init__(self)
+
+        self['filename'] = None
+        self['line'] = None
+        self['column'] = None
+        self['message'] = None
+        self['flag'] = None
+
+    # Since we inherit from dict, functools.total_ordering gets confused.
+    # Thus, we define a key function, a generic comparison, and then
+    # implement all the rich operators with those; approach is from:
+    # http://regebro.wordpress.com/2010/12/13/python-implementing-rich-comparison-the-correct-way/
+    def _cmpkey(self):
+        return (self['filename'], self['line'], self['column'])
+
+    def _compare(self, other, func):
+        if not isinstance(other, CompilerWarning):
+            return NotImplemented
+
+        return func(self._cmpkey(), other._cmpkey())
+
+    def __eq__(self, other):
+        return self._compare(other, lambda s,o: s == o)
+
+    def __neq__(self, other):
+        return self._compare(other, lambda s,o: s != o)
+
+    def __lt__(self, other):
+        return self._compare(other, lambda s,o: s < o)
+
+    def __le__(self, other):
+        return self._compare(other, lambda s,o: s <= o)
+
+    def __gt__(self, other):
+        return self._compare(other, lambda s,o: s > o)
+
+    def __ge__(self, other):
+        return self._compare(other, lambda s,o: s >= o)
+
+    def __hash__(self):
+        """Define so this can exist inside a set, etc."""
+        return hash(tuple(sorted(self.items())))
+
+
+class WarningsDatabase(object):
+    """Holds a collection of warnings.
+
+    The warnings database is a semi-intelligent container that holds warnings
+    encountered during builds.
+
+    The warnings database is backed by a JSON file. But, that is transparent
+    to consumers.
+
+    Under most circumstances, the warnings database is insert only. When a
+    warning is encountered, the caller simply blindly inserts it into the
+    database. The database figures out whether it is a dupe, etc.
+
+    During the course of development, it is common for warnings to change
+    slightly as source code changes. For example, line numbers will disagree.
+    The WarningsDatabase handles this by storing the hash of a file a warning
+    occurred in. At warning insert time, if the hash of the file does not match
+    what is stored in the database, the existing warnings for that file are
+    purged from the database.
+
+    Callers should periodically prune old, invalid warnings from the database
+    by calling prune(). A good time to do this is at the end of a build.
+    """
+    def __init__(self):
+        """Create an empty database."""
+        self._files = {}
+
+    def __len__(self):
+        i = 0
+        for value in self._files.values():
+            i += len(value['warnings'])
+
+        return i
+
+    def __iter__(self):
+        for value in self._files.values():
+            for warning in value['warnings']:
+                yield warning
+
+    def __contains__(self, item):
+        for value in self._files.values():
+            for warning in value['warnings']:
+                if warning == item:
+                    return True
+
+        return False
+
+    @property
+    def warnings(self):
+        """All the CompilerWarning instances in this database."""
+        for value in self._files.values():
+            for w in value['warnings']:
+                yield w
+
+    def type_counts(self, dirpath=None):
+        """Returns a mapping of warning types to their counts."""
+
+        types = {}
+        for value in self._files.values():
+            for warning in value['warnings']:
+                if dirpath and not mozpath.normsep(warning['filename']).startswith(dirpath):
+                    continue
+                flag = warning['flag']
+                count = types.get(flag, 0)
+                count += 1
+
+                types[flag] = count
+
+        return types
+
+    def has_file(self, filename):
+        """Whether we have any warnings for the specified file."""
+        return filename in self._files
+
+    def warnings_for_file(self, filename):
+        """Obtain the warnings for the specified file."""
+        f = self._files.get(filename, {'warnings': []})
+
+        for warning in f['warnings']:
+            yield warning
+
+    def insert(self, warning, compute_hash=True):
+        assert isinstance(warning, CompilerWarning)
+
+        filename = warning['filename']
+
+        new_hash = None
+
+        if compute_hash:
+            new_hash = hash_file(filename)
+
+        if filename in self._files:
+            if new_hash != self._files[filename]['hash']:
+                del self._files[filename]
+
+        value = self._files.get(filename, {
+            'hash': new_hash,
+            'warnings': set(),
+        })
+
+        value['warnings'].add(warning)
+
+        self._files[filename] = value
+
+    def prune(self):
+        """Prune the contents of the database.
+
+        This removes warnings that are no longer valid. A warning is no longer
+        valid if the file it was in no longer exists or if the content has
+        changed.
+
+        The check for changed content catches the case where a file previously
+        contained warnings but no longer does.
+        """
+
+        # Need to calculate up front since we are mutating original object.
+        filenames = self._files.keys()
+        for filename in filenames:
+            if not os.path.exists(filename):
+                del self._files[filename]
+                continue
+
+            if self._files[filename]['hash'] is None:
+                continue
+
+            current_hash = hash_file(filename)
+            if current_hash != self._files[filename]['hash']:
+                del self._files[filename]
+                continue
+
+    def serialize(self, fh):
+        """Serialize the database to an open file handle."""
+        obj = {'files': {}}
+
+        # All this hackery because JSON can't handle sets.
+        for k, v in self._files.iteritems():
+            obj['files'][k] = {}
+
+            for k2, v2 in v.iteritems():
+                normalized = v2
+
+                if k2 == 'warnings':
+                    normalized = [w for w in v2]
+
+                obj['files'][k][k2] = normalized
+
+        json.dump(obj, fh, indent=2)
+
+    def deserialize(self, fh):
+        """Load serialized content from a handle into the current instance."""
+        obj = json.load(fh)
+
+        self._files = obj['files']
+
+        # Normalize data types.
+        for filename, value in self._files.iteritems():
+            for k, v in value.iteritems():
+                if k != 'warnings':
+                    continue
+
+                normalized = set()
+                for d in v:
+                    w = CompilerWarning()
+                    w.update(d)
+                    normalized.add(w)
+
+                self._files[filename]['warnings'] = normalized
+
+    def load_from_file(self, filename):
+        """Load the database from a file."""
+        with open(filename, 'rb') as fh:
+            self.deserialize(fh)
+
+    def save_to_file(self, filename):
+        """Save the database to a file."""
+        try:
+            # Ensure the directory exists
+            os.makedirs(os.path.dirname(filename))
+        except OSError as e:
+            if e.errno != errno.EEXIST:
+                raise
+        with open(filename, 'wb') as fh:
+            self.serialize(fh)
+
+
+class WarningsCollector(object):
+    """Collects warnings from text data.
+
+    Instances of this class receive data (usually the output of compiler
+    invocations) and parse it into warnings and add these warnings to a
+    database.
+
+    The collector works by incrementally receiving data, usually line-by-line
+    output from the compiler. Therefore, it can maintain state to parse
+    multi-line warning messages.
+    """
+    def __init__(self, database=None, objdir=None, resolve_files=True):
+        self.database = database
+        self.objdir = objdir
+        self.resolve_files = resolve_files
+        self.included_from = []
+
+        if database is None:
+            self.database = WarningsDatabase()
+
+    def process_line(self, line):
+        """Take a line of text and process it for a warning."""
+
+        filtered = RE_STRIP_COLORS.sub('', line)
+
+        # Clang warnings in files included from the one(s) being compiled will
+        # start with "In file included from /path/to/file:line:". Here, we
+        # record those.
+        if filtered.startswith(IN_FILE_INCLUDED_FROM):
+            included_from = filtered[len(IN_FILE_INCLUDED_FROM):]
+
+            parts = included_from.split(':')
+
+            self.included_from.append(parts[0])
+
+            return
+
+        warning = CompilerWarning()
+        filename = None
+
+        # TODO make more efficient so we run minimal regexp matches.
+        match_clang = RE_CLANG_WARNING.match(filtered)
+        match_msvc = RE_MSVC_WARNING.match(filtered)
+        if match_clang:
+            d = match_clang.groupdict()
+
+            filename = d['file']
+            warning['line'] = int(d['line'])
+            warning['column'] = int(d['column'])
+            warning['flag'] = d['flag']
+            warning['message'] = d['message'].rstrip()
+
+        elif match_msvc:
+            d = match_msvc.groupdict()
+
+            filename = d['file']
+            warning['line'] = int(d['line'])
+            warning['flag'] = d['flag']
+            warning['message'] = d['message'].rstrip()
+        else:
+            self.included_from = []
+            return None
+
+        filename = os.path.normpath(filename)
+
+        # Sometimes we get relative includes. These typically point to files in
+        # the object directory. We try to resolve the relative path.
+        if not os.path.isabs(filename):
+            filename = self._normalize_relative_path(filename)
+
+        if not os.path.exists(filename) and self.resolve_files:
+            raise Exception('Could not find file containing warning: %s' %
+                    filename)
+
+        warning['filename'] = filename
+
+        self.database.insert(warning, compute_hash=self.resolve_files)
+
+        return warning
+
+    def _normalize_relative_path(self, filename):
+        # Special case files in dist/include.
+        idx = filename.find('/dist/include')
+        if idx != -1:
+            return self.objdir + filename[idx:]
+
+        for included_from in self.included_from:
+            source_dir = os.path.dirname(included_from)
+
+            candidate = os.path.normpath(os.path.join(source_dir, filename))
+
+            if os.path.exists(candidate):
+                return candidate
+
+        return filename

python/mozbuild/mozbuild/config_status.py

@@ -0,0 +1,182 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# Combined with build/autoconf/config.status.m4, ConfigStatus is an almost
+# drop-in replacement for autoconf 2.13's config.status, with features
+# borrowed from autoconf > 2.5, and additional features.
+
+from __future__ import absolute_import, print_function
+
+import logging
+import os
+import subprocess
+import sys
+import time
+
+from argparse import ArgumentParser
+
+from mach.logging import LoggingManager
+from mozbuild.backend.configenvironment import ConfigEnvironment
+from mozbuild.base import MachCommandConditions
+from mozbuild.frontend.emitter import TreeMetadataEmitter
+from mozbuild.frontend.reader import BuildReader
+from mozbuild.mozinfo import write_mozinfo
+from itertools import chain
+
+from mozbuild.backend import (
+    backends,
+    get_backend_class,
+)
+
+
+log_manager = LoggingManager()
+
+
+ANDROID_IDE_ADVERTISEMENT = '''
+=============
+ADVERTISEMENT
+
+You are building Firefox for Android. After your build completes, you can open
+the top source directory in IntelliJ or Android Studio directly and build using
+Gradle.  See the documentation at
+
+https://developer.mozilla.org/en-US/docs/Simple_Firefox_for_Android_build
+
+PLEASE BE AWARE THAT GRADLE AND INTELLIJ/ANDROID STUDIO SUPPORT IS EXPERIMENTAL.
+You should verify any changes using |mach build|.
+=============
+'''.strip()
+
+VISUAL_STUDIO_ADVERTISEMENT = '''
+===============================
+Visual Studio Support Available
+
+You are building Firefox on Windows. You can generate Visual Studio
+files by running:
+
+   mach build-backend --backend=VisualStudio
+
+===============================
+'''.strip()
+
+
+def config_status(topobjdir='.', topsrcdir='.', defines=None,
+                  non_global_defines=None, substs=None, source=None,
+                  mozconfig=None, args=sys.argv[1:]):
+    '''Main function, providing config.status functionality.
+
+    Contrary to config.status, it doesn't use CONFIG_FILES or CONFIG_HEADERS
+    variables.
+
+    Without the -n option, this program acts as config.status and considers
+    the current directory as the top object directory, even when config.status
+    is in a different directory. It will, however, treat the directory
+    containing config.status as the top object directory with the -n option.
+
+    The options to this function are passed when creating the
+    ConfigEnvironment. These lists, as well as the actual wrapper script
+    around this function, are meant to be generated by configure.
+    See build/autoconf/config.status.m4.
+    '''
+
+    if 'CONFIG_FILES' in os.environ:
+        raise Exception('Using the CONFIG_FILES environment variable is not '
+            'supported.')
+    if 'CONFIG_HEADERS' in os.environ:
+        raise Exception('Using the CONFIG_HEADERS environment variable is not '
+            'supported.')
+
+    if not os.path.isabs(topsrcdir):
+        raise Exception('topsrcdir must be defined as an absolute directory: '
+            '%s' % topsrcdir)
+
+    default_backends = ['RecursiveMake']
+    default_backends = (substs or {}).get('BUILD_BACKENDS', ['RecursiveMake'])
+
+    parser = ArgumentParser()
+    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
+                        help='display verbose output')
+    parser.add_argument('-n', dest='not_topobjdir', action='store_true',
+                        help='do not consider current directory as top object directory')
+    parser.add_argument('-d', '--diff', action='store_true',
+                        help='print diffs of changed files.')
+    parser.add_argument('-b', '--backend', nargs='+', choices=sorted(backends),
+                        default=default_backends,
+                        help='what backend to build (default: %s).' %
+                        ' '.join(default_backends))
+    parser.add_argument('--dry-run', action='store_true',
+                        help='do everything except writing files out.')
+    options = parser.parse_args(args)
+
+    # Without -n, the current directory is meant to be the top object directory
+    if not options.not_topobjdir:
+        topobjdir = os.path.abspath('.')
+
+    env = ConfigEnvironment(topsrcdir, topobjdir, defines=defines,
+            non_global_defines=non_global_defines, substs=substs,
+            source=source, mozconfig=mozconfig)
+
+    # mozinfo.json only needs written if configure changes and configure always
+    # passes this environment variable.
+    if 'WRITE_MOZINFO' in os.environ:
+        write_mozinfo(os.path.join(topobjdir, 'mozinfo.json'), env, os.environ)
+
+    cpu_start = time.clock()
+    time_start = time.time()
+
+    # Make appropriate backend instances, defaulting to RecursiveMakeBackend,
+    # or what is in BUILD_BACKENDS.
+    selected_backends = [get_backend_class(b)(env) for b in options.backend]
+
+    if options.dry_run:
+        for b in selected_backends:
+            b.dry_run = True
+
+    reader = BuildReader(env)
+    emitter = TreeMetadataEmitter(env)
+    # This won't actually do anything because of the magic of generators.
+    definitions = emitter.emit(reader.read_topsrcdir())
+
+    log_level = logging.DEBUG if options.verbose else logging.INFO
+    log_manager.add_terminal_logging(level=log_level)
+    log_manager.enable_unstructured()
+
+    print('Reticulating splines...', file=sys.stderr)
+    if len(selected_backends) > 1:
+        definitions = list(definitions)
+
+    for the_backend in selected_backends:
+        the_backend.consume(definitions)
+
+    execution_time = 0.0
+    for obj in chain((reader, emitter), selected_backends):
+        summary = obj.summary()
+        print(summary, file=sys.stderr)
+        execution_time += summary.execution_time
+
+    cpu_time = time.clock() - cpu_start
+    wall_time = time.time() - time_start
+    efficiency = cpu_time / wall_time if wall_time else 100
+    untracked = wall_time - execution_time
+
+    print(
+        'Total wall time: {:.2f}s; CPU time: {:.2f}s; Efficiency: '
+        '{:.0%}; Untracked: {:.2f}s'.format(
+            wall_time, cpu_time, efficiency, untracked),
+        file=sys.stderr
+    )
+
+    if options.diff:
+        for the_backend in selected_backends:
+            for path, diff in sorted(the_backend.file_diffs.items()):
+                print('\n'.join(diff))
+
+    # Advertise Visual Studio if appropriate.
+    if os.name == 'nt' and 'VisualStudio' not in options.backend:
+        print(VISUAL_STUDIO_ADVERTISEMENT)
+
+    # Advertise Eclipse if it is appropriate.
+    if MachCommandConditions.is_android(env):
+        if 'AndroidEclipse' not in options.backend:
+            print(ANDROID_IDE_ADVERTISEMENT)

python/mozbuild/mozbuild/configure/__init__.py

@@ -0,0 +1,935 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import inspect
+import logging
+import os
+import re
+import sys
+import types
+from collections import OrderedDict
+from contextlib import contextmanager
+from functools import wraps
+from mozbuild.configure.options import (
+    CommandLineHelper,
+    ConflictingOptionError,
+    InvalidOptionError,
+    NegativeOptionValue,
+    Option,
+    OptionValue,
+    PositiveOptionValue,
+)
+from mozbuild.configure.help import HelpFormatter
+from mozbuild.configure.util import (
+    ConfigureOutputHandler,
+    getpreferredencoding,
+    LineIO,
+)
+from mozbuild.util import (
+    exec_,
+    memoize,
+    memoized_property,
+    ReadOnlyDict,
+    ReadOnlyNamespace,
+)
+
+import mozpack.path as mozpath
+
+
+class ConfigureError(Exception):
+    pass
+
+
+class SandboxDependsFunction(object):
+    '''Sandbox-visible representation of @depends functions.'''
+    def __call__(self, *arg, **kwargs):
+        raise ConfigureError('The `%s` function may not be called'
+                             % self.__name__)
+
+
+class DependsFunction(object):
+    __slots__ = (
+        'func', 'dependencies', 'when', 'sandboxed', 'sandbox', '_result')
+
+    def __init__(self, sandbox, func, dependencies, when=None):
+        assert isinstance(sandbox, ConfigureSandbox)
+        self.func = func
+        self.dependencies = dependencies
+        self.sandboxed = wraps(func)(SandboxDependsFunction())
+        self.sandbox = sandbox
+        self.when = when
+        sandbox._depends[self.sandboxed] = self
+
+        # Only @depends functions with a dependency on '--help' are executed
+        # immediately. Everything else is queued for later execution.
+        if sandbox._help_option in dependencies:
+            sandbox._value_for(self)
+        elif not sandbox._help:
+            sandbox._execution_queue.append((sandbox._value_for, (self,)))
+
+    @property
+    def name(self):
+        return self.func.__name__
+
+    @property
+    def sandboxed_dependencies(self):
+        return [
+            d.sandboxed if isinstance(d, DependsFunction) else d
+            for d in self.dependencies
+        ]
+
+    @memoized_property
+    def result(self):
+        if self.when and not self.sandbox._value_for(self.when):
+            return None
+
+        resolved_args = [self.sandbox._value_for(d) for d in self.dependencies]
+        return self.func(*resolved_args)
+
+    def __repr__(self):
+        return '<%s.%s %s(%s)>' % (
+            self.__class__.__module__,
+            self.__class__.__name__,
+            self.name,
+            ', '.join(repr(d) for d in self.dependencies),
+        )
+
+
+class CombinedDependsFunction(DependsFunction):
+    def __init__(self, sandbox, func, dependencies):
+        @memoize
+        @wraps(func)
+        def wrapper(*args):
+            return func(args)
+
+        flatten_deps = []
+        for d in dependencies:
+            if isinstance(d, CombinedDependsFunction) and d.func == wrapper:
+                for d2 in d.dependencies:
+                    if d2 not in flatten_deps:
+                        flatten_deps.append(d2)
+            elif d not in flatten_deps:
+                flatten_deps.append(d)
+
+        # Automatically add a --help dependency if one of the dependencies
+        # depends on it.
+        for d in flatten_deps:
+            if (isinstance(d, DependsFunction) and
+                sandbox._help_option in d.dependencies):
+                flatten_deps.insert(0, sandbox._help_option)
+                break
+
+        super(CombinedDependsFunction, self).__init__(
+            sandbox, wrapper, flatten_deps)
+
+    @memoized_property
+    def result(self):
+        # Ignore --help for the combined result
+        deps = self.dependencies
+        if deps[0] == self.sandbox._help_option:
+            deps = deps[1:]
+        resolved_args = [self.sandbox._value_for(d) for d in deps]
+        return self.func(*resolved_args)
+
+    def __eq__(self, other):
+        return (isinstance(other, self.__class__) and
+                self.func == other.func and
+                set(self.dependencies) == set(other.dependencies))
+
+    def __ne__(self, other):
+        return not self == other
+
+class SandboxedGlobal(dict):
+    '''Identifiable dict type for use as function global'''
+
+
+def forbidden_import(*args, **kwargs):
+    raise ImportError('Importing modules is forbidden')
+
+
+class ConfigureSandbox(dict):
+    """Represents a sandbox for executing Python code for build configuration.
+    This is a different kind of sandboxing than the one used for moz.build
+    processing.
+
+    The sandbox has 9 primitives:
+    - option
+    - depends
+    - template
+    - imports
+    - include
+    - set_config
+    - set_define
+    - imply_option
+    - only_when
+
+    `option`, `include`, `set_config`, `set_define` and `imply_option` are
+    functions. `depends`, `template`, and `imports` are decorators. `only_when`
+    is a context_manager.
+
+    These primitives are declared as name_impl methods to this class and
+    the mapping name -> name_impl is done automatically in __getitem__.
+
+    Additional primitives should be frowned upon to keep the sandbox itself as
+    simple as possible. Instead, helpers should be created within the sandbox
+    with the existing primitives.
+
+    The sandbox is given, at creation, a dict where the yielded configuration
+    will be stored.
+
+        config = {}
+        sandbox = ConfigureSandbox(config)
+        sandbox.run(path)
+        do_stuff(config)
+    """
+
+    # The default set of builtins. We expose unicode as str to make sandboxed
+    # files more python3-ready.
+    BUILTINS = ReadOnlyDict({
+        b: __builtins__[b]
+        for b in ('None', 'False', 'True', 'int', 'bool', 'any', 'all', 'len',
+                  'list', 'tuple', 'set', 'dict', 'isinstance', 'getattr',
+                  'hasattr', 'enumerate', 'range', 'zip')
+    }, __import__=forbidden_import, str=unicode)
+
+    # Expose a limited set of functions from os.path
+    OS = ReadOnlyNamespace(path=ReadOnlyNamespace(**{
+        k: getattr(mozpath, k, getattr(os.path, k))
+        for k in ('abspath', 'basename', 'dirname', 'isabs', 'join',
+                  'normcase', 'normpath', 'realpath', 'relpath')
+    }))
+
+    def __init__(self, config, environ=os.environ, argv=sys.argv,
+                 stdout=sys.stdout, stderr=sys.stderr, logger=None):
+        dict.__setitem__(self, '__builtins__', self.BUILTINS)
+
+        self._paths = []
+        self._all_paths = set()
+        self._templates = set()
+        # Associate SandboxDependsFunctions to DependsFunctions.
+        self._depends = {}
+        self._seen = set()
+        # Store the @imports added to a given function.
+        self._imports = {}
+
+        self._options = OrderedDict()
+        # Store raw option (as per command line or environment) for each Option
+        self._raw_options = OrderedDict()
+
+        # Store options added with `imply_option`, and the reason they were
+        # added (which can either have been given to `imply_option`, or
+        # inferred. Their order matters, so use a list.
+        self._implied_options = []
+
+        # Store all results from _prepare_function
+        self._prepared_functions = set()
+
+        # Queue of functions to execute, with their arguments
+        self._execution_queue = []
+
+        # Store the `when`s associated to some options.
+        self._conditions = {}
+
+        # A list of conditions to apply as a default `when` for every *_impl()
+        self._default_conditions = []
+
+        self._helper = CommandLineHelper(environ, argv)
+
+        assert isinstance(config, dict)
+        self._config = config
+
+        if logger is None:
+            logger = moz_logger = logging.getLogger('moz.configure')
+            logger.setLevel(logging.DEBUG)
+            formatter = logging.Formatter('%(levelname)s: %(message)s')
+            handler = ConfigureOutputHandler(stdout, stderr)
+            handler.setFormatter(formatter)
+            queue_debug = handler.queue_debug
+            logger.addHandler(handler)
+
+        else:
+            assert isinstance(logger, logging.Logger)
+            moz_logger = None
+            @contextmanager
+            def queue_debug():
+                yield
+
+        # Some callers will manage to log a bytestring with characters in it
+        # that can't be converted to ascii. Make our log methods robust to this
+        # by detecting the encoding that a producer is likely to have used.
+        encoding = getpreferredencoding()
+        def wrapped_log_method(logger, key):
+            method = getattr(logger, key)
+            if not encoding:
+                return method
+            def wrapped(*args, **kwargs):
+                out_args = [
+                    arg.decode(encoding) if isinstance(arg, str) else arg
+                    for arg in args
+                ]
+                return method(*out_args, **kwargs)
+            return wrapped
+
+        log_namespace = {
+            k: wrapped_log_method(logger, k)
+            for k in ('debug', 'info', 'warning', 'error')
+        }
+        log_namespace['queue_debug'] = queue_debug
+        self.log_impl = ReadOnlyNamespace(**log_namespace)
+
+        self._help = None
+        self._help_option = self.option_impl('--help',
+                                             help='print this message')
+        self._seen.add(self._help_option)
+
+        self._always = DependsFunction(self, lambda: True, [])
+        self._never = DependsFunction(self, lambda: False, [])
+
+        if self._value_for(self._help_option):
+            self._help = HelpFormatter(argv[0])
+            self._help.add(self._help_option)
+        elif moz_logger:
+            handler = logging.FileHandler('config.log', mode='w', delay=True)
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+
+    def include_file(self, path):
+        '''Include one file in the sandbox. Users of this class probably want
+
+        Note: this will execute all template invocations, as well as @depends
+        functions that depend on '--help', but nothing else.
+        '''
+
+        if self._paths:
+            path = mozpath.join(mozpath.dirname(self._paths[-1]), path)
+            path = mozpath.normpath(path)
+            if not mozpath.basedir(path, (mozpath.dirname(self._paths[0]),)):
+                raise ConfigureError(
+                    'Cannot include `%s` because it is not in a subdirectory '
+                    'of `%s`' % (path, mozpath.dirname(self._paths[0])))
+        else:
+            path = mozpath.realpath(mozpath.abspath(path))
+        if path in self._all_paths:
+            raise ConfigureError(
+                'Cannot include `%s` because it was included already.' % path)
+        self._paths.append(path)
+        self._all_paths.add(path)
+
+        source = open(path, 'rb').read()
+
+        code = compile(source, path, 'exec')
+
+        exec_(code, self)
+
+        self._paths.pop(-1)
+
+    def run(self, path=None):
+        '''Executes the given file within the sandbox, as well as everything
+        pending from any other included file, and ensure the overall
+        consistency of the executed script(s).'''
+        if path:
+            self.include_file(path)
+
+        for option in self._options.itervalues():
+            # All options must be referenced by some @depends function
+            if option not in self._seen:
+                raise ConfigureError(
+                    'Option `%s` is not handled ; reference it with a @depends'
+                    % option.option
+                )
+
+            self._value_for(option)
+
+        # All implied options should exist.
+        for implied_option in self._implied_options:
+            value = self._resolve(implied_option.value,
+                                  need_help_dependency=False)
+            if value is not None:
+                raise ConfigureError(
+                    '`%s`, emitted from `%s` line %d, is unknown.'
+                    % (implied_option.option, implied_option.caller[1],
+                       implied_option.caller[2]))
+
+        # All options should have been removed (handled) by now.
+        for arg in self._helper:
+            without_value = arg.split('=', 1)[0]
+            raise InvalidOptionError('Unknown option: %s' % without_value)
+
+        # Run the execution queue
+        for func, args in self._execution_queue:
+            func(*args)
+
+        if self._help:
+            with LineIO(self.log_impl.info) as out:
+                self._help.usage(out)
+
+    def __getitem__(self, key):
+        impl = '%s_impl' % key
+        func = getattr(self, impl, None)
+        if func:
+            return func
+
+        return super(ConfigureSandbox, self).__getitem__(key)
+
+    def __setitem__(self, key, value):
+        if (key in self.BUILTINS or key == '__builtins__' or
+                hasattr(self, '%s_impl' % key)):
+            raise KeyError('Cannot reassign builtins')
+
+        if inspect.isfunction(value) and value not in self._templates:
+            value, _ = self._prepare_function(value)
+
+        elif (not isinstance(value, SandboxDependsFunction) and
+                value not in self._templates and
+                not (inspect.isclass(value) and issubclass(value, Exception))):
+            raise KeyError('Cannot assign `%s` because it is neither a '
+                           '@depends nor a @template' % key)
+
+        return super(ConfigureSandbox, self).__setitem__(key, value)
+
+    def _resolve(self, arg, need_help_dependency=True):
+        if isinstance(arg, SandboxDependsFunction):
+            return self._value_for_depends(self._depends[arg],
+                                           need_help_dependency)
+        return arg
+
+    def _value_for(self, obj, need_help_dependency=False):
+        if isinstance(obj, SandboxDependsFunction):
+            assert obj in self._depends
+            return self._value_for_depends(self._depends[obj],
+                                           need_help_dependency)
+
+        elif isinstance(obj, DependsFunction):
+            return self._value_for_depends(obj, need_help_dependency)
+
+        elif isinstance(obj, Option):
+            return self._value_for_option(obj)
+
+        assert False
+
+    @memoize
+    def _value_for_depends(self, obj, need_help_dependency=False):
+        assert not inspect.isgeneratorfunction(obj.func)
+        return obj.result
+
+    @memoize
+    def _value_for_option(self, option):
+        implied = {}
+        for implied_option in self._implied_options[:]:
+            if implied_option.name not in (option.name, option.env):
+                continue
+            self._implied_options.remove(implied_option)
+
+            if (implied_option.when and
+                not self._value_for(implied_option.when)):
+                continue
+
+            value = self._resolve(implied_option.value,
+                                  need_help_dependency=False)
+
+            if value is not None:
+                if isinstance(value, OptionValue):
+                    pass
+                elif value is True:
+                    value = PositiveOptionValue()
+                elif value is False or value == ():
+                    value = NegativeOptionValue()
+                elif isinstance(value, types.StringTypes):
+                    value = PositiveOptionValue((value,))
+                elif isinstance(value, tuple):
+                    value = PositiveOptionValue(value)
+                else:
+                    raise TypeError("Unexpected type: '%s'"
+                                    % type(value).__name__)
+
+                opt = value.format(implied_option.option)
+                self._helper.add(opt, 'implied')
+                implied[opt] = implied_option
+
+        try:
+            value, option_string = self._helper.handle(option)
+        except ConflictingOptionError as e:
+            reason = implied[e.arg].reason
+            if isinstance(reason, Option):
+                reason = self._raw_options.get(reason) or reason.option
+                reason = reason.split('=', 1)[0]
+            raise InvalidOptionError(
+                "'%s' implied by '%s' conflicts with '%s' from the %s"
+                % (e.arg, reason, e.old_arg, e.old_origin))
+
+        if option_string:
+            self._raw_options[option] = option_string
+
+        when = self._conditions.get(option)
+        if (when and not self._value_for(when, need_help_dependency=True) and
+            value is not None and value.origin != 'default'):
+            if value.origin == 'environment':
+                # The value we return doesn't really matter, because of the
+                # requirement for @depends to have the same when.
+                return None
+            raise InvalidOptionError(
+                '%s is not available in this configuration'
+                % option_string.split('=', 1)[0])
+
+        return value
+
+    def _dependency(self, arg, callee_name, arg_name=None):
+        if isinstance(arg, types.StringTypes):
+            prefix, name, values = Option.split_option(arg)
+            if values != ():
+                raise ConfigureError("Option must not contain an '='")
+            if name not in self._options:
+                raise ConfigureError("'%s' is not a known option. "
+                                     "Maybe it's declared too late?"
+                                     % arg)
+            arg = self._options[name]
+            self._seen.add(arg)
+        elif isinstance(arg, SandboxDependsFunction):
+            assert arg in self._depends
+            arg = self._depends[arg]
+        else:
+            raise TypeError(
+                "Cannot use object of type '%s' as %sargument to %s"
+                % (type(arg).__name__, '`%s` ' % arg_name if arg_name else '',
+                   callee_name))
+        return arg
+
+    def _normalize_when(self, when, callee_name):
+        if when is True:
+            when = self._always
+        elif when is False:
+            when = self._never
+        elif when is not None:
+            when = self._dependency(when, callee_name, 'when')
+
+        if self._default_conditions:
+            # Create a pseudo @depends function for the combination of all
+            # default conditions and `when`.
+            dependencies = [when] if when else []
+            dependencies.extend(self._default_conditions)
+            if len(dependencies) == 1:
+                return dependencies[0]
+            return CombinedDependsFunction(self, all, dependencies)
+        return when
+
+    @contextmanager
+    def only_when_impl(self, when):
+        '''Implementation of only_when()
+
+        `only_when` is a context manager that essentially makes calls to
+        other sandbox functions within the context block ignored.
+        '''
+        when = self._normalize_when(when, 'only_when')
+        if when and self._default_conditions[-1:] != [when]:
+            self._default_conditions.append(when)
+            yield
+            self._default_conditions.pop()
+        else:
+            yield
+
+    def option_impl(self, *args, **kwargs):
+        '''Implementation of option()
+        This function creates and returns an Option() object, passing it the
+        resolved arguments (uses the result of functions when functions are
+        passed). In most cases, the result of this function is not expected to
+        be used.
+        Command line argument/environment variable parsing for this Option is
+        handled here.
+        '''
+        when = self._normalize_when(kwargs.get('when'), 'option')
+        args = [self._resolve(arg) for arg in args]
+        kwargs = {k: self._resolve(v) for k, v in kwargs.iteritems()
+                                      if k != 'when'}
+        option = Option(*args, **kwargs)
+        if when:
+            self._conditions[option] = when
+        if option.name in self._options:
+            raise ConfigureError('Option `%s` already defined' % option.option)
+        if option.env in self._options:
+            raise ConfigureError('Option `%s` already defined' % option.env)
+        if option.name:
+            self._options[option.name] = option
+        if option.env:
+            self._options[option.env] = option
+
+        if self._help and (when is None or
+                           self._value_for(when, need_help_dependency=True)):
+            self._help.add(option)
+
+        return option
+
+    def depends_impl(self, *args, **kwargs):
+        '''Implementation of @depends()
+        This function is a decorator. It returns a function that subsequently
+        takes a function and returns a dummy function. The dummy function
+        identifies the actual function for the sandbox, while preventing
+        further function calls from within the sandbox.
+
+        @depends() takes a variable number of option strings or dummy function
+        references. The decorated function is called as soon as the decorator
+        is called, and the arguments it receives are the OptionValue or
+        function results corresponding to each of the arguments to @depends.
+        As an exception, when a HelpFormatter is attached, only functions that
+        have '--help' in their @depends argument list are called.
+
+        The decorated function is altered to use a different global namespace
+        for its execution. This different global namespace exposes a limited
+        set of functions from os.path.
+        '''
+        for k in kwargs:
+            if k != 'when':
+                raise TypeError(
+                    "depends_impl() got an unexpected keyword argument '%s'"
+                    % k)
+
+        when = self._normalize_when(kwargs.get('when'), '@depends')
+
+        if not when and not args:
+            raise ConfigureError('@depends needs at least one argument')
+
+        dependencies = tuple(self._dependency(arg, '@depends') for arg in args)
+
+        conditions = [
+            self._conditions[d]
+            for d in dependencies
+            if d in self._conditions and isinstance(d, Option)
+        ]
+        for c in conditions:
+            if c != when:
+                raise ConfigureError('@depends function needs the same `when` '
+                                     'as options it depends on')
+
+        def decorator(func):
+            if inspect.isgeneratorfunction(func):
+                raise ConfigureError(
+                    'Cannot decorate generator functions with @depends')
+            func, glob = self._prepare_function(func)
+            depends = DependsFunction(self, func, dependencies, when=when)
+            return depends.sandboxed
+
+        return decorator
+
+    def include_impl(self, what, when=None):
+        '''Implementation of include().
+        Allows to include external files for execution in the sandbox.
+        It is possible to use a @depends function as argument, in which case
+        the result of the function is the file name to include. This latter
+        feature is only really meant for --enable-application/--enable-project.
+        '''
+        with self.only_when_impl(when):
+            what = self._resolve(what)
+            if what:
+                if not isinstance(what, types.StringTypes):
+                    raise TypeError("Unexpected type: '%s'" % type(what).__name__)
+                self.include_file(what)
+
+    def template_impl(self, func):
+        '''Implementation of @template.
+        This function is a decorator. Template functions are called
+        immediately. They are altered so that their global namespace exposes
+        a limited set of functions from os.path, as well as `depends` and
+        `option`.
+        Templates allow to simplify repetitive constructs, or to implement
+        helper decorators and somesuch.
+        '''
+        template, glob = self._prepare_function(func)
+        glob.update(
+            (k[:-len('_impl')], getattr(self, k))
+            for k in dir(self) if k.endswith('_impl') and k != 'template_impl'
+        )
+        glob.update((k, v) for k, v in self.iteritems() if k not in glob)
+
+        # Any function argument to the template must be prepared to be sandboxed.
+        # If the template itself returns a function (in which case, it's very
+        # likely a decorator), that function must be prepared to be sandboxed as
+        # well.
+        def wrap_template(template):
+            isfunction = inspect.isfunction
+
+            def maybe_prepare_function(obj):
+                if isfunction(obj):
+                    func, _ = self._prepare_function(obj)
+                    return func
+                return obj
+
+            # The following function may end up being prepared to be sandboxed,
+            # so it mustn't depend on anything from the global scope in this
+            # file. It can however depend on variables from the closure, thus
+            # maybe_prepare_function and isfunction are declared above to be
+            # available there.
+            @wraps(template)
+            def wrapper(*args, **kwargs):
+                args = [maybe_prepare_function(arg) for arg in args]
+                kwargs = {k: maybe_prepare_function(v)
+                          for k, v in kwargs.iteritems()}
+                ret = template(*args, **kwargs)
+                if isfunction(ret):
+                    # We can't expect the sandboxed code to think about all the
+                    # details of implementing decorators, so do some of the
+                    # work for them. If the function takes exactly one function
+                    # as argument and returns a function, it must be a
+                    # decorator, so mark the returned function as wrapping the
+                    # function passed in.
+                    if len(args) == 1 and not kwargs and isfunction(args[0]):
+                        ret = wraps(args[0])(ret)
+                    return wrap_template(ret)
+                return ret
+            return wrapper
+
+        wrapper = wrap_template(template)
+        self._templates.add(wrapper)
+        return wrapper
+
+    RE_MODULE = re.compile('^[a-zA-Z0-9_\.]+$')
+
+    def imports_impl(self, _import, _from=None, _as=None):
+        '''Implementation of @imports.
+        This decorator imports the given _import from the given _from module
+        optionally under a different _as name.
+        The options correspond to the various forms for the import builtin.
+            @imports('sys')
+            @imports(_from='mozpack', _import='path', _as='mozpath')
+        '''
+        for value, required in (
+                (_import, True), (_from, False), (_as, False)):
+
+            if not isinstance(value, types.StringTypes) and (
+                    required or value is not None):
+                raise TypeError("Unexpected type: '%s'" % type(value).__name__)
+            if value is not None and not self.RE_MODULE.match(value):
+                raise ValueError("Invalid argument to @imports: '%s'" % value)
+        if _as and '.' in _as:
+            raise ValueError("Invalid argument to @imports: '%s'" % _as)
+
+        def decorator(func):
+            if func in self._templates:
+                raise ConfigureError(
+                    '@imports must appear after @template')
+            if func in self._depends:
+                raise ConfigureError(
+                    '@imports must appear after @depends')
+            # For the imports to apply in the order they appear in the
+            # .configure file, we accumulate them in reverse order and apply
+            # them later.
+            imports = self._imports.setdefault(func, [])
+            imports.insert(0, (_from, _import, _as))
+            return func
+
+        return decorator
+
+    def _apply_imports(self, func, glob):
+        for _from, _import, _as in self._imports.get(func, ()):
+            _from = '%s.' % _from if _from else ''
+            if _as:
+                glob[_as] = self._get_one_import('%s%s' % (_from, _import))
+            else:
+                what = _import.split('.')[0]
+                glob[what] = self._get_one_import('%s%s' % (_from, what))
+
+    def _get_one_import(self, what):
+        # The special `__sandbox__` module gives access to the sandbox
+        # instance.
+        if what == '__sandbox__':
+            return self
+        # Special case for the open() builtin, because otherwise, using it
+        # fails with "IOError: file() constructor not accessible in
+        # restricted mode"
+        if what == '__builtin__.open':
+            return lambda *args, **kwargs: open(*args, **kwargs)
+        # Until this proves to be a performance problem, just construct an
+        # import statement and execute it.
+        import_line = ''
+        if '.' in what:
+            _from, what = what.rsplit('.', 1)
+            import_line += 'from %s ' % _from
+        import_line += 'import %s as imported' % what
+        glob = {}
+        exec_(import_line, {}, glob)
+        return glob['imported']
+
+    def _resolve_and_set(self, data, name, value, when=None):
+        # Don't set anything when --help was on the command line
+        if self._help:
+            return
+        if when and not self._value_for(when):
+            return
+        name = self._resolve(name, need_help_dependency=False)
+        if name is None:
+            return
+        if not isinstance(name, types.StringTypes):
+            raise TypeError("Unexpected type: '%s'" % type(name).__name__)
+        if name in data:
+            raise ConfigureError(
+                "Cannot add '%s' to configuration: Key already "
+                "exists" % name)
+        value = self._resolve(value, need_help_dependency=False)
+        if value is not None:
+            data[name] = value
+
+    def set_config_impl(self, name, value, when=None):
+        '''Implementation of set_config().
+        Set the configuration items with the given name to the given value.
+        Both `name` and `value` can be references to @depends functions,
+        in which case the result from these functions is used. If the result
+        of either function is None, the configuration item is not set.
+        '''
+        when = self._normalize_when(when, 'set_config')
+
+        self._execution_queue.append((
+            self._resolve_and_set, (self._config, name, value, when)))
+
+    def set_define_impl(self, name, value, when=None):
+        '''Implementation of set_define().
+        Set the define with the given name to the given value. Both `name` and
+        `value` can be references to @depends functions, in which case the
+        result from these functions is used. If the result of either function
+        is None, the define is not set. If the result is False, the define is
+        explicitly undefined (-U).
+        '''
+        when = self._normalize_when(when, 'set_define')
+
+        defines = self._config.setdefault('DEFINES', {})
+        self._execution_queue.append((
+            self._resolve_and_set, (defines, name, value, when)))
+
+    def imply_option_impl(self, option, value, reason=None, when=None):
+        '''Implementation of imply_option().
+        Injects additional options as if they had been passed on the command
+        line. The `option` argument is a string as in option()'s `name` or
+        `env`. The option must be declared after `imply_option` references it.
+        The `value` argument indicates the value to pass to the option.
+        It can be:
+        - True. In this case `imply_option` injects the positive option
+          (--enable-foo/--with-foo).
+              imply_option('--enable-foo', True)
+              imply_option('--disable-foo', True)
+          are both equivalent to `--enable-foo` on the command line.
+
+        - False. In this case `imply_option` injects the negative option
+          (--disable-foo/--without-foo).
+              imply_option('--enable-foo', False)
+              imply_option('--disable-foo', False)
+          are both equivalent to `--disable-foo` on the command line.
+
+        - None. In this case `imply_option` does nothing.
+              imply_option('--enable-foo', None)
+              imply_option('--disable-foo', None)
+          are both equivalent to not passing any flag on the command line.
+
+        - a string or a tuple. In this case `imply_option` injects the positive
+          option with the given value(s).
+              imply_option('--enable-foo', 'a')
+              imply_option('--disable-foo', 'a')
+          are both equivalent to `--enable-foo=a` on the command line.
+              imply_option('--enable-foo', ('a', 'b'))
+              imply_option('--disable-foo', ('a', 'b'))
+          are both equivalent to `--enable-foo=a,b` on the command line.
+
+        Because imply_option('--disable-foo', ...) can be misleading, it is
+        recommended to use the positive form ('--enable' or '--with') for
+        `option`.
+
+        The `value` argument can also be (and usually is) a reference to a
+        @depends function, in which case the result of that function will be
+        used as per the descripted mapping above.
+
+        The `reason` argument indicates what caused the option to be implied.
+        It is necessary when it cannot be inferred from the `value`.
+        '''
+        # Don't do anything when --help was on the command line
+        if self._help:
+            return
+        if not reason and isinstance(value, SandboxDependsFunction):
+            deps = self._depends[value].dependencies
+            possible_reasons = [d for d in deps if d != self._help_option]
+            if len(possible_reasons) == 1:
+                if isinstance(possible_reasons[0], Option):
+                    reason = possible_reasons[0]
+        if not reason and (isinstance(value, (bool, tuple)) or
+                           isinstance(value, types.StringTypes)):
+            # A reason can be provided automatically when imply_option
+            # is called with an immediate value.
+            _, filename, line, _, _, _ = inspect.stack()[1]
+            reason = "imply_option at %s:%s" % (filename, line)
+
+        if not reason:
+            raise ConfigureError(
+                "Cannot infer what implies '%s'. Please add a `reason` to "
+                "the `imply_option` call."
+                % option)
+
+        when = self._normalize_when(when, 'imply_option')
+
+        prefix, name, values = Option.split_option(option)
+        if values != ():
+            raise ConfigureError("Implied option must not contain an '='")
+
+        self._implied_options.append(ReadOnlyNamespace(
+            option=option,
+            prefix=prefix,
+            name=name,
+            value=value,
+            caller=inspect.stack()[1],
+            reason=reason,
+            when=when,
+        ))
+
+    def _prepare_function(self, func):
+        '''Alter the given function global namespace with the common ground
+        for @depends, and @template.
+        '''
+        if not inspect.isfunction(func):
+            raise TypeError("Unexpected type: '%s'" % type(func).__name__)
+        if func in self._prepared_functions:
+            return func, func.func_globals
+
+        glob = SandboxedGlobal(
+            (k, v) for k, v in func.func_globals.iteritems()
+            if (inspect.isfunction(v) and v not in self._templates) or (
+                inspect.isclass(v) and issubclass(v, Exception))
+        )
+        glob.update(
+            __builtins__=self.BUILTINS,
+            __file__=self._paths[-1] if self._paths else '',
+            __name__=self._paths[-1] if self._paths else '',
+            os=self.OS,
+            log=self.log_impl,
+        )
+
+        # The execution model in the sandbox doesn't guarantee the execution
+        # order will always be the same for a given function, and if it uses
+        # variables from a closure that are changed after the function is
+        # declared, depending when the function is executed, the value of the
+        # variable can differ. For consistency, we force the function to use
+        # the value from the earliest it can be run, which is at declaration.
+        # Note this is not entirely bullet proof (if the value is e.g. a list,
+        # the list contents could have changed), but covers the bases.
+        closure = None
+        if func.func_closure:
+            def makecell(content):
+                def f():
+                    content
+                return f.func_closure[0]
+
+            closure = tuple(makecell(cell.cell_contents)
+                            for cell in func.func_closure)
+
+        new_func = wraps(func)(types.FunctionType(
+            func.func_code,
+            glob,
+            func.__name__,
+            func.func_defaults,
+            closure
+        ))
+        @wraps(new_func)
+        def wrapped(*args, **kwargs):
+            if func in self._imports:
+                self._apply_imports(func, glob)
+                del self._imports[func]
+            return new_func(*args, **kwargs)
+
+        self._prepared_functions.add(wrapped)
+        return wrapped, glob

python/mozbuild/mozbuild/configure/check_debug_ranges.py

@@ -0,0 +1,62 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script returns the number of items for the DW_AT_ranges corresponding
+# to a given compilation unit. This is used as a helper to find a bug in some
+# versions of GNU ld.
+
+from __future__ import absolute_import
+
+import subprocess
+import sys
+import re
+
+def get_range_for(compilation_unit, debug_info):
+    '''Returns the range offset for a given compilation unit
+       in a given debug_info.'''
+    name = ranges = ''
+    search_cu = False
+    for nfo in debug_info.splitlines():
+        if 'DW_TAG_compile_unit' in nfo:
+            search_cu = True
+        elif 'DW_TAG_' in nfo or not nfo.strip():
+            if name == compilation_unit and ranges != '':
+                return int(ranges, 16)
+            name = ranges = ''
+            search_cu = False
+        if search_cu:
+            if 'DW_AT_name' in nfo:
+                name = nfo.rsplit(None, 1)[1]
+            elif 'DW_AT_ranges' in nfo:
+                ranges = nfo.rsplit(None, 1)[1]
+    return None
+
+def get_range_length(range, debug_ranges):
+    '''Returns the number of items in the range starting at the
+       given offset.'''
+    length = 0
+    for line in debug_ranges.splitlines():
+        m = re.match('\s*([0-9a-fA-F]+)\s+([0-9a-fA-F]+)\s+([0-9a-fA-F]+)', line)
+        if m and int(m.group(1), 16) == range:
+            length += 1
+    return length
+
+def main(bin, compilation_unit):
+    p = subprocess.Popen(['objdump', '-W', bin], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
+    (out, err) = p.communicate()
+    sections = re.split('\n(Contents of the|The section) ', out)
+    debug_info = [s for s in sections if s.startswith('.debug_info')]
+    debug_ranges = [s for s in sections if s.startswith('.debug_ranges')]
+    if not debug_ranges or not debug_info:
+        return 0
+
+    range = get_range_for(compilation_unit, debug_info[0])
+    if range is not None:
+        return get_range_length(range, debug_ranges[0])
+
+    return -1
+
+
+if __name__ == '__main__':
+    print main(*sys.argv[1:])

python/mozbuild/mozbuild/configure/constants.py

@@ -0,0 +1,103 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from mozbuild.util import EnumString
+from collections import OrderedDict
+
+
+CompilerType = EnumString.subclass(
+    'clang',
+    'clang-cl',
+    'gcc',
+    'msvc',
+)
+
+OS = EnumString.subclass(
+    'Android',
+    'DragonFly',
+    'FreeBSD',
+    'GNU',
+    'iOS',
+    'NetBSD',
+    'OpenBSD',
+    'OSX',
+    'WINNT',
+)
+
+Kernel = EnumString.subclass(
+    'Darwin',
+    'DragonFly',
+    'FreeBSD',
+    'kFreeBSD',
+    'Linux',
+    'NetBSD',
+    'OpenBSD',
+    'WINNT',
+)
+
+CPU_bitness = {
+    'aarch64': 64,
+    'Alpha': 32,
+    'arm': 32,
+    'hppa': 32,
+    'ia64': 64,
+    'mips32': 32,
+    'mips64': 64,
+    'ppc': 32,
+    'ppc64': 64,
+    's390': 32,
+    's390x': 64,
+    'sparc': 32,
+    'sparc64': 64,
+    'x86': 32,
+    'x86_64': 64,
+}
+
+CPU = EnumString.subclass(*CPU_bitness.keys())
+
+Endianness = EnumString.subclass(
+    'big',
+    'little',
+)
+
+WindowsBinaryType = EnumString.subclass(
+    'win32',
+    'win64',
+)
+
+# The order of those checks matter
+CPU_preprocessor_checks = OrderedDict((
+    ('x86', '__i386__ || _M_IX86'),
+    ('x86_64', '__x86_64__ || _M_X64'),
+    ('arm', '__arm__ || _M_ARM'),
+    ('aarch64', '__aarch64__'),
+    ('ia64', '__ia64__'),
+    ('s390x', '__s390x__'),
+    ('s390', '__s390__'),
+    ('ppc64', '__powerpc64__'),
+    ('ppc', '__powerpc__'),
+    ('Alpha', '__alpha__'),
+    ('hppa', '__hppa__'),
+    ('sparc64', '__sparc__ && __arch64__'),
+    ('sparc', '__sparc__'),
+    ('mips64', '__mips64'),
+    ('mips32', '__mips__'),
+))
+
+assert sorted(CPU_preprocessor_checks.keys()) == sorted(CPU.POSSIBLE_VALUES)
+
+kernel_preprocessor_checks = {
+    'Darwin': '__APPLE__',
+    'DragonFly': '__DragonFly__',
+    'FreeBSD': '__FreeBSD__',
+    'kFreeBSD': '__FreeBSD_kernel__',
+    'Linux': '__linux__',
+    'NetBSD': '__NetBSD__',
+    'OpenBSD': '__OpenBSD__',
+    'WINNT': '_WIN32 || __CYGWIN__',
+}
+
+assert sorted(kernel_preprocessor_checks.keys()) == sorted(Kernel.POSSIBLE_VALUES)

python/mozbuild/mozbuild/configure/help.py

@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+from mozbuild.configure.options import Option
+
+
+class HelpFormatter(object):
+    def __init__(self, argv0):
+        self.intro = ['Usage: %s [options]' % os.path.basename(argv0)]
+        self.options = ['Options: [defaults in brackets after descriptions]']
+        self.env = ['Environment variables:']
+
+    def add(self, option):
+        assert isinstance(option, Option)
+
+        if option.possible_origins == ('implied',):
+            # Don't display help if our option can only be implied.
+            return
+
+        # TODO: improve formatting
+        target = self.options if option.name else self.env
+        opt = option.option
+        if option.choices:
+            opt += '={%s}' % ','.join(option.choices)
+        help = option.help or ''
+        if len(option.default):
+            if help:
+                help += ' '
+            help += '[%s]' % ','.join(option.default)
+
+        if len(opt) > 24 or not help:
+            target.append('  %s' % opt)
+            if help:
+                target.append('%s%s' % (' ' * 28, help))
+        else:
+            target.append('  %-24s  %s' % (opt, help))
+
+    def usage(self, out):
+        print('\n\n'.join('\n'.join(t)
+                          for t in (self.intro, self.options, self.env)),
+              file=out)

python/mozbuild/mozbuild/configure/libstdcxx.py

@@ -0,0 +1,81 @@
+#!/usr/bin/python
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+# This script find the version of libstdc++ and prints it as single number
+# with 8 bits per element. For example, GLIBCXX_3.4.10 becomes
+# 3 << 16 | 4 << 8 | 10 = 197642. This format is easy to use
+# in the C preprocessor.
+
+# We find out both the host and target versions. Since the output
+# will be used from shell, we just print the two assignments and evaluate
+# them from shell.
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+import re
+
+re_for_ld = re.compile('.*\((.*)\).*')
+
+def parse_readelf_line(x):
+    """Return the version from a readelf line that looks like:
+    0x00ec: Rev: 1  Flags: none  Index: 8  Cnt: 2  Name: GLIBCXX_3.4.6
+    """
+    return x.split(':')[-1].split('_')[-1].strip()
+
+def parse_ld_line(x):
+    """Parse a line from the output of ld -t. The output of gold is just
+    the full path, gnu ld prints "-lstdc++ (path)".
+    """
+    t = re_for_ld.match(x)
+    if t:
+        return t.groups()[0].strip()
+    return x.strip()
+
+def split_ver(v):
+    """Covert the string '1.2.3' into the list [1,2,3]
+    """
+    return [int(x) for x in v.split('.')]
+
+def cmp_ver(a, b):
+    """Compare versions in the form 'a.b.c'
+    """
+    for (i, j) in zip(split_ver(a), split_ver(b)):
+        if i != j:
+            return i - j
+    return 0
+
+def encode_ver(v):
+    """Encode the version as a single number.
+    """
+    t = split_ver(v)
+    return t[0] << 16 | t[1] << 8 | t[2]
+
+def find_version(e):
+    """Given the value of environment variable CXX or HOST_CXX, find the
+    version of the libstdc++ it uses.
+    """
+    args = e.split()
+    args +=  ['-shared', '-Wl,-t']
+    p = subprocess.Popen(args, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
+    candidates = [x for x in p.stdout if 'libstdc++.so' in x]
+    if not candidates:
+        return ''
+    assert len(candidates) == 1
+    libstdcxx = parse_ld_line(candidates[-1])
+
+    p = subprocess.Popen(['readelf', '-V', libstdcxx], stdout=subprocess.PIPE)
+    versions = [parse_readelf_line(x)
+                for x in p.stdout.readlines() if 'Name: GLIBCXX' in x]
+    last_version = sorted(versions, cmp = cmp_ver)[-1]
+    return encode_ver(last_version)
+
+if __name__ == '__main__':
+    cxx_env = os.environ['CXX']
+    print 'MOZ_LIBSTDCXX_TARGET_VERSION=%s' % find_version(cxx_env)
+    host_cxx_env = os.environ.get('HOST_CXX', cxx_env)
+    print 'MOZ_LIBSTDCXX_HOST_VERSION=%s' % find_version(host_cxx_env)

python/mozbuild/mozbuild/configure/lint.py

@@ -0,0 +1,78 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from StringIO import StringIO
+from . import (
+    CombinedDependsFunction,
+    ConfigureError,
+    ConfigureSandbox,
+    DependsFunction,
+)
+from .lint_util import disassemble_as_iter
+from mozbuild.util import memoize
+
+
+class LintSandbox(ConfigureSandbox):
+    def __init__(self, environ=None, argv=None, stdout=None, stderr=None):
+        out = StringIO()
+        stdout = stdout or out
+        stderr = stderr or out
+        environ = environ or {}
+        argv = argv or []
+        self._wrapped = {}
+        super(LintSandbox, self).__init__({}, environ=environ, argv=argv,
+                                          stdout=stdout, stderr=stderr)
+
+    def run(self, path=None):
+        if path:
+            self.include_file(path)
+
+    def _missing_help_dependency(self, obj):
+        if isinstance(obj, CombinedDependsFunction):
+            return False
+        if isinstance(obj, DependsFunction):
+            if (self._help_option in obj.dependencies or
+                obj in (self._always, self._never)):
+                return False
+            func, glob = self._wrapped[obj.func]
+            # We allow missing --help dependencies for functions that:
+            # - don't use @imports
+            # - don't have a closure
+            # - don't use global variables
+            if func in self._imports or func.func_closure:
+                return True
+            for op, arg in disassemble_as_iter(func):
+                if op in ('LOAD_GLOBAL', 'STORE_GLOBAL'):
+                    # There is a fake os module when one is not imported,
+                    # and it's allowed for functions without a --help
+                    # dependency.
+                    if arg == 'os' and glob.get('os') is self.OS:
+                        continue
+                    return True
+        return False
+
+    @memoize
+    def _value_for_depends(self, obj, need_help_dependency=False):
+        with_help = self._help_option in obj.dependencies
+        if with_help:
+            for arg in obj.dependencies:
+                if self._missing_help_dependency(arg):
+                    raise ConfigureError(
+                        "`%s` depends on '--help' and `%s`. "
+                        "`%s` must depend on '--help'"
+                        % (obj.name, arg.name, arg.name))
+        elif ((self._help or need_help_dependency) and
+              self._missing_help_dependency(obj)):
+            raise ConfigureError("Missing @depends for `%s`: '--help'" %
+                                 obj.name)
+        return super(LintSandbox, self)._value_for_depends(
+            obj, need_help_dependency)
+
+    def _prepare_function(self, func):
+        wrapped, glob = super(LintSandbox, self)._prepare_function(func)
+        if wrapped not in self._wrapped:
+            self._wrapped[wrapped] = func, glob
+        return wrapped, glob

python/mozbuild/mozbuild/configure/lint_util.py

@@ -0,0 +1,52 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import dis
+import inspect
+
+
+# dis.dis only outputs to stdout. This is a modified version that
+# returns an iterator.
+def disassemble_as_iter(co):
+    if inspect.ismethod(co):
+        co = co.im_func
+    if inspect.isfunction(co):
+        co = co.func_code
+    code = co.co_code
+    n = len(code)
+    i = 0
+    extended_arg = 0
+    free = None
+    while i < n:
+        c = code[i]
+        op = ord(c)
+        opname = dis.opname[op]
+        i += 1;
+        if op >= dis.HAVE_ARGUMENT:
+            arg = ord(code[i]) + ord(code[i + 1]) * 256 + extended_arg
+            extended_arg = 0
+            i += 2
+            if op == dis.EXTENDED_ARG:
+                extended_arg = arg * 65536L
+                continue
+            if op in dis.hasconst:
+                yield opname, co.co_consts[arg]
+            elif op in dis.hasname:
+                yield opname, co.co_names[arg]
+            elif op in dis.hasjrel:
+                yield opname, i + arg
+            elif op in dis.haslocal:
+                yield opname, co.co_varnames[arg]
+            elif op in dis.hascompare:
+                yield opname, dis.cmp_op[arg]
+            elif op in dis.hasfree:
+                if free is None:
+                    free = co.co_cellvars + co.co_freevars
+                yield opname, free[arg]
+            else:
+                yield opname, None
+        else:
+            yield opname, None

python/mozbuild/mozbuild/configure/options.py

@@ -0,0 +1,485 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import os
+import sys
+import types
+from collections import OrderedDict
+
+
+def istupleofstrings(obj):
+    return isinstance(obj, tuple) and len(obj) and all(
+        isinstance(o, types.StringTypes) for o in obj)
+
+
+class OptionValue(tuple):
+    '''Represents the value of a configure option.
+
+    This class is not meant to be used directly. Use its subclasses instead.
+
+    The `origin` attribute holds where the option comes from (e.g. environment,
+    command line, or default)
+    '''
+    def __new__(cls, values=(), origin='unknown'):
+        return super(OptionValue, cls).__new__(cls, values)
+
+    def __init__(self, values=(), origin='unknown'):
+        self.origin = origin
+
+    def format(self, option):
+        if option.startswith('--'):
+            prefix, name, values = Option.split_option(option)
+            assert values == ()
+            for prefix_set in (
+                    ('disable', 'enable'),
+                    ('without', 'with'),
+            ):
+                if prefix in prefix_set:
+                    prefix = prefix_set[int(bool(self))]
+                    break
+            if prefix:
+                option = '--%s-%s' % (prefix, name)
+            elif self:
+                option = '--%s' % name
+            else:
+                return ''
+            if len(self):
+                return '%s=%s' % (option, ','.join(self))
+            return option
+        elif self and not len(self):
+            return '%s=1' % option
+        return '%s=%s' % (option, ','.join(self))
+
+    def __eq__(self, other):
+        if type(other) != type(self):
+            return False
+        return super(OptionValue, self).__eq__(other)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __repr__(self):
+        return '%s%s' % (self.__class__.__name__,
+                         super(OptionValue, self).__repr__())
+
+
+class PositiveOptionValue(OptionValue):
+    '''Represents the value for a positive option (--enable/--with/--foo)
+    in the form of a tuple for when values are given to the option (in the form
+    --option=value[,value2...].
+    '''
+    def __nonzero__(self):
+        return True
+
+
+class NegativeOptionValue(OptionValue):
+    '''Represents the value for a negative option (--disable/--without)
+
+    This is effectively an empty tuple with a `origin` attribute.
+    '''
+    def __new__(cls, origin='unknown'):
+        return super(NegativeOptionValue, cls).__new__(cls, origin=origin)
+
+    def __init__(self, origin='unknown'):
+        return super(NegativeOptionValue, self).__init__(origin=origin)
+
+
+class InvalidOptionError(Exception):
+    pass
+
+
+class ConflictingOptionError(InvalidOptionError):
+    def __init__(self, message, **format_data):
+        if format_data:
+            message = message.format(**format_data)
+        super(ConflictingOptionError, self).__init__(message)
+        for k, v in format_data.iteritems():
+            setattr(self, k, v)
+
+
+class Option(object):
+    '''Represents a configure option
+
+    A configure option can be a command line flag or an environment variable
+    or both.
+
+    - `name` is the full command line flag (e.g. --enable-foo).
+    - `env` is the environment variable name (e.g. ENV)
+    - `nargs` is the number of arguments the option may take. It can be a
+      number or the special values '?' (0 or 1), '*' (0 or more), or '+' (1 or
+      more).
+    - `default` can be used to give a default value to the option. When the
+      `name` of the option starts with '--enable-' or '--with-', the implied
+      default is an empty PositiveOptionValue. When it starts with '--disable-'
+      or '--without-', the implied default is a NegativeOptionValue.
+    - `choices` restricts the set of values that can be given to the option.
+    - `help` is the option description for use in the --help output.
+    - `possible_origins` is a tuple of strings that are origins accepted for
+      this option. Example origins are 'mozconfig', 'implied', and 'environment'.
+    '''
+    __slots__ = (
+        'id', 'prefix', 'name', 'env', 'nargs', 'default', 'choices', 'help',
+        'possible_origins',
+    )
+
+    def __init__(self, name=None, env=None, nargs=None, default=None,
+                 possible_origins=None, choices=None, help=None):
+        if not name and not env:
+            raise InvalidOptionError(
+                'At least an option name or an environment variable name must '
+                'be given')
+        if name:
+            if not isinstance(name, types.StringTypes):
+                raise InvalidOptionError('Option must be a string')
+            if not name.startswith('--'):
+                raise InvalidOptionError('Option must start with `--`')
+            if '=' in name:
+                raise InvalidOptionError('Option must not contain an `=`')
+            if not name.islower():
+                raise InvalidOptionError('Option must be all lowercase')
+        if env:
+            if not isinstance(env, types.StringTypes):
+                raise InvalidOptionError(
+                    'Environment variable name must be a string')
+            if not env.isupper():
+                raise InvalidOptionError(
+                    'Environment variable name must be all uppercase')
+        if nargs not in (None, '?', '*', '+') and not (
+                isinstance(nargs, int) and nargs >= 0):
+            raise InvalidOptionError(
+                "nargs must be a positive integer, '?', '*' or '+'")
+        if (not isinstance(default, types.StringTypes) and
+                not isinstance(default, (bool, types.NoneType)) and
+                not istupleofstrings(default)):
+            raise InvalidOptionError(
+                'default must be a bool, a string or a tuple of strings')
+        if choices and not istupleofstrings(choices):
+            raise InvalidOptionError(
+                'choices must be a tuple of strings')
+        if not help:
+            raise InvalidOptionError('A help string must be provided')
+        if possible_origins and not istupleofstrings(possible_origins):
+            raise InvalidOptionError(
+                'possible_origins must be a tuple of strings')
+        self.possible_origins = possible_origins
+
+        if name:
+            prefix, name, values = self.split_option(name)
+            assert values == ()
+
+            # --disable and --without options mean the default is enabled.
+            # --enable and --with options mean the default is disabled.
+            # However, we allow a default to be given so that the default
+            # can be affected by other factors.
+            if prefix:
+                if default is None:
+                    default = prefix in ('disable', 'without')
+                elif default is False:
+                    prefix = {
+                        'disable': 'enable',
+                        'without': 'with',
+                    }.get(prefix, prefix)
+                elif default is True:
+                    prefix = {
+                        'enable': 'disable',
+                        'with': 'without',
+                    }.get(prefix, prefix)
+        else:
+            prefix = ''
+
+        self.prefix = prefix
+        self.name = name
+        self.env = env
+        if default in (None, False):
+            self.default = NegativeOptionValue(origin='default')
+        elif isinstance(default, tuple):
+            self.default = PositiveOptionValue(default, origin='default')
+        elif default is True:
+            self.default = PositiveOptionValue(origin='default')
+        else:
+            self.default = PositiveOptionValue((default,), origin='default')
+        if nargs is None:
+            nargs = 0
+            if len(self.default) == 1:
+                nargs = '?'
+            elif len(self.default) > 1:
+                nargs = '*'
+            elif choices:
+                nargs = 1
+        self.nargs = nargs
+        has_choices = choices is not None
+        if isinstance(self.default, PositiveOptionValue):
+            if has_choices and len(self.default) == 0:
+                raise InvalidOptionError(
+                    'A `default` must be given along with `choices`')
+            if not self._validate_nargs(len(self.default)):
+                raise InvalidOptionError(
+                    "The given `default` doesn't satisfy `nargs`")
+            if has_choices and not all(d in choices for d in self.default):
+                raise InvalidOptionError(
+                    'The `default` value must be one of %s' %
+                    ', '.join("'%s'" % c for c in choices))
+        elif has_choices:
+            maxargs = self.maxargs
+            if len(choices) < maxargs and maxargs != sys.maxint:
+                raise InvalidOptionError('Not enough `choices` for `nargs`')
+        self.choices = choices
+        self.help = help
+
+    @staticmethod
+    def split_option(option):
+        '''Split a flag or variable into a prefix, a name and values
+
+        Variables come in the form NAME=values (no prefix).
+        Flags come in the form --name=values or --prefix-name=values
+        where prefix is one of 'with', 'without', 'enable' or 'disable'.
+        The '=values' part is optional. Values are separated with commas.
+        '''
+        if not isinstance(option, types.StringTypes):
+            raise InvalidOptionError('Option must be a string')
+
+        elements = option.split('=', 1)
+        name = elements[0]
+        values = tuple(elements[1].split(',')) if len(elements) == 2 else ()
+        if name.startswith('--'):
+            name = name[2:]
+            if not name.islower():
+                raise InvalidOptionError('Option must be all lowercase')
+            elements = name.split('-', 1)
+            prefix = elements[0]
+            if len(elements) == 2 and prefix in ('enable', 'disable',
+                                                 'with', 'without'):
+                return prefix, elements[1], values
+        else:
+            if name.startswith('-'):
+                raise InvalidOptionError(
+                    'Option must start with two dashes instead of one')
+            if name.islower():
+                raise InvalidOptionError(
+                    'Environment variable name must be all uppercase')
+        return '', name, values
+
+    @staticmethod
+    def _join_option(prefix, name):
+        # The constraints around name and env in __init__ make it so that
+        # we can distinguish between flags and environment variables with
+        # islower/isupper.
+        if name.isupper():
+            assert not prefix
+            return name
+        elif prefix:
+            return '--%s-%s' % (prefix, name)
+        return '--%s' % name
+
+    @property
+    def option(self):
+        if self.prefix or self.name:
+            return self._join_option(self.prefix, self.name)
+        else:
+            return self.env
+
+    @property
+    def minargs(self):
+        if isinstance(self.nargs, int):
+            return self.nargs
+        return 1 if self.nargs == '+' else 0
+
+    @property
+    def maxargs(self):
+        if isinstance(self.nargs, int):
+            return self.nargs
+        return 1 if self.nargs == '?' else sys.maxint
+
+    def _validate_nargs(self, num):
+        minargs, maxargs = self.minargs, self.maxargs
+        return num >= minargs and num <= maxargs
+
+    def get_value(self, option=None, origin='unknown'):
+        '''Given a full command line option (e.g. --enable-foo=bar) or a
+        variable assignment (FOO=bar), returns the corresponding OptionValue.
+
+        Note: variable assignments can come from either the environment or
+        from the command line (e.g. `../configure CFLAGS=-O2`)
+        '''
+        if not option:
+            return self.default
+
+        if self.possible_origins and origin not in self.possible_origins:
+            raise InvalidOptionError(
+                '%s can not be set by %s. Values are accepted from: %s' %
+                (option, origin, ', '.join(self.possible_origins)))
+
+        prefix, name, values = self.split_option(option)
+        option = self._join_option(prefix, name)
+
+        assert name in (self.name, self.env)
+
+        if prefix in ('disable', 'without'):
+            if values != ():
+                raise InvalidOptionError('Cannot pass a value to %s' % option)
+            return NegativeOptionValue(origin=origin)
+
+        if name == self.env:
+            if values == ('',):
+                return NegativeOptionValue(origin=origin)
+            if self.nargs in (0, '?', '*') and values == ('1',):
+                return PositiveOptionValue(origin=origin)
+
+        values = PositiveOptionValue(values, origin=origin)
+
+        if not self._validate_nargs(len(values)):
+            raise InvalidOptionError('%s takes %s value%s' % (
+                option,
+                {
+                    '?': '0 or 1',
+                    '*': '0 or more',
+                    '+': '1 or more',
+                }.get(self.nargs, str(self.nargs)),
+                's' if (not isinstance(self.nargs, int) or
+                        self.nargs != 1) else ''
+            ))
+
+        if len(values) and self.choices:
+            relative_result = None
+            for val in values:
+                if self.nargs in ('+', '*'):
+                    if val.startswith(('+', '-')):
+                        if relative_result is None:
+                            relative_result = list(self.default)
+                        sign = val[0]
+                        val = val[1:]
+                        if sign == '+':
+                            if val not in relative_result:
+                                relative_result.append(val)
+                        else:
+                            try:
+                                relative_result.remove(val)
+                            except ValueError:
+                                pass
+
+                if val not in self.choices:
+                    raise InvalidOptionError(
+                        "'%s' is not one of %s"
+                        % (val, ', '.join("'%s'" % c for c in self.choices)))
+
+            if relative_result is not None:
+                values = PositiveOptionValue(relative_result, origin=origin)
+
+        return values
+
+    def __repr__(self):
+        return '<%s.%s [%s]>' % (self.__class__.__module__,
+                                 self.__class__.__name__, self.option)
+
+
+class CommandLineHelper(object):
+    '''Helper class to handle the various ways options can be given either
+    on the command line of through the environment.
+
+    For instance, an Option('--foo', env='FOO') can be passed as --foo on the
+    command line, or as FOO=1 in the environment *or* on the command line.
+
+    If multiple variants are given, command line is prefered over the
+    environment, and if different values are given on the command line, the
+    last one wins. (This mimicks the behavior of autoconf, avoiding to break
+    existing mozconfigs using valid options in weird ways)
+
+    Extra options can be added afterwards through API calls. For those,
+    conflicting values will raise an exception.
+    '''
+    def __init__(self, environ=os.environ, argv=sys.argv):
+        self._environ = dict(environ)
+        self._args = OrderedDict()
+        self._extra_args = OrderedDict()
+        self._origins = {}
+        self._last = 0
+
+        for arg in argv[1:]:
+            self.add(arg, 'command-line', self._args)
+
+    def add(self, arg, origin='command-line', args=None):
+        assert origin != 'default'
+        prefix, name, values = Option.split_option(arg)
+        if args is None:
+            args = self._extra_args
+        if args is self._extra_args and name in self._extra_args:
+            old_arg = self._extra_args[name][0]
+            old_prefix, _, old_values = Option.split_option(old_arg)
+            if prefix != old_prefix or values != old_values:
+                raise ConflictingOptionError(
+                    "Cannot add '{arg}' to the {origin} set because it "
+                    "conflicts with '{old_arg}' that was added earlier",
+                    arg=arg, origin=origin, old_arg=old_arg,
+                    old_origin=self._origins[old_arg])
+        self._last += 1
+        args[name] = arg, self._last
+        self._origins[arg] = origin
+
+    def _prepare(self, option, args):
+        arg = None
+        origin = 'command-line'
+        from_name = args.get(option.name)
+        from_env = args.get(option.env)
+        if from_name and from_env:
+            arg1, pos1 = from_name
+            arg2, pos2 = from_env
+            arg, pos = (arg1, pos1) if abs(pos1) > abs(pos2) else (arg2, pos2)
+            if args is self._extra_args and (option.get_value(arg1) !=
+                                             option.get_value(arg2)):
+                origin = self._origins[arg]
+                old_arg = arg2 if abs(pos1) > abs(pos2) else arg1
+                raise ConflictingOptionError(
+                    "Cannot add '{arg}' to the {origin} set because it "
+                    "conflicts with '{old_arg}' that was added earlier",
+                    arg=arg, origin=origin, old_arg=old_arg,
+                    old_origin=self._origins[old_arg])
+        elif from_name or from_env:
+            arg, pos = from_name if from_name else from_env
+        elif option.env and args is self._args:
+            env = self._environ.get(option.env)
+            if env is not None:
+                arg = '%s=%s' % (option.env, env)
+                origin = 'environment'
+
+        origin = self._origins.get(arg, origin)
+
+        for k in (option.name, option.env):
+            try:
+                del args[k]
+            except KeyError:
+                pass
+
+        return arg, origin
+
+    def handle(self, option):
+        '''Return the OptionValue corresponding to the given Option instance,
+        depending on the command line, environment, and extra arguments, and
+        the actual option or variable that set it.
+        Only works once for a given Option.
+        '''
+        assert isinstance(option, Option)
+
+        arg, origin = self._prepare(option, self._args)
+        ret = option.get_value(arg, origin)
+
+        extra_arg, extra_origin = self._prepare(option, self._extra_args)
+        extra_ret = option.get_value(extra_arg, extra_origin)
+
+        if extra_ret.origin == 'default':
+            return ret, arg
+
+        if ret.origin != 'default' and extra_ret != ret:
+            raise ConflictingOptionError(
+                "Cannot add '{arg}' to the {origin} set because it conflicts "
+                "with {old_arg} from the {old_origin} set", arg=extra_arg,
+                origin=extra_ret.origin, old_arg=arg, old_origin=ret.origin)
+
+        return extra_ret, extra_arg
+
+    def __iter__(self):
+        for d in (self._args, self._extra_args):
+            for arg, pos in d.itervalues():
+                yield arg

python/mozbuild/mozbuild/configure/util.py

@@ -0,0 +1,226 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import codecs
+import itertools
+import locale
+import logging
+import os
+import sys
+from collections import deque
+from contextlib import contextmanager
+from distutils.version import LooseVersion
+
+def getpreferredencoding():
+    # locale._parse_localename makes locale.getpreferredencoding
+    # return None when LC_ALL is C, instead of e.g. 'US-ASCII' or
+    # 'ANSI_X3.4-1968' when it uses nl_langinfo.
+    encoding = None
+    try:
+        encoding = locale.getpreferredencoding()
+    except ValueError:
+        # On english OSX, LC_ALL is UTF-8 (not en-US.UTF-8), and
+        # that throws off locale._parse_localename, which ends up
+        # being used on e.g. homebrew python.
+        if os.environ.get('LC_ALL', '').upper() == 'UTF-8':
+            encoding = 'utf-8'
+    return encoding
+
+class Version(LooseVersion):
+    '''A simple subclass of distutils.version.LooseVersion.
+    Adds attributes for `major`, `minor`, `patch` for the first three
+    version components so users can easily pull out major/minor
+    versions, like:
+
+    v = Version('1.2b')
+    v.major == 1
+    v.minor == 2
+    v.patch == 0
+    '''
+    def __init__(self, version):
+        # Can't use super, LooseVersion's base class is not a new-style class.
+        LooseVersion.__init__(self, version)
+        # Take the first three integer components, stopping at the first
+        # non-integer and padding the rest with zeroes.
+        (self.major, self.minor, self.patch) = list(itertools.chain(
+            itertools.takewhile(lambda x:isinstance(x, int), self.version),
+            (0, 0, 0)))[:3]
+
+
+    def __cmp__(self, other):
+        # LooseVersion checks isinstance(StringType), so work around it.
+        if isinstance(other, unicode):
+            other = other.encode('ascii')
+        return LooseVersion.__cmp__(self, other)
+
+
+class ConfigureOutputHandler(logging.Handler):
+    '''A logging handler class that sends info messages to stdout and other
+    messages to stderr.
+
+    Messages sent to stdout are not formatted with the attached Formatter.
+    Additionally, if they end with '... ', no newline character is printed,
+    making the next message printed follow the '... '.
+
+    Only messages above log level INFO (included) are logged.
+
+    Messages below that level can be kept until an ERROR message is received,
+    at which point the last `maxlen` accumulated messages below INFO are
+    printed out. This feature is only enabled under the `queue_debug` context
+    manager.
+    '''
+    def __init__(self, stdout=sys.stdout, stderr=sys.stderr, maxlen=20):
+        super(ConfigureOutputHandler, self).__init__()
+
+        # Python has this feature where it sets the encoding of pipes to
+        # ascii, which blatantly fails when trying to print out non-ascii.
+        def fix_encoding(fh):
+            try:
+                isatty = fh.isatty()
+            except AttributeError:
+                isatty = True
+
+            if not isatty:
+                encoding = getpreferredencoding()
+                if encoding:
+                    return codecs.getwriter(encoding)(fh)
+            return fh
+
+        self._stdout = fix_encoding(stdout)
+        self._stderr = fix_encoding(stderr) if stdout != stderr else self._stdout
+        try:
+            fd1 = self._stdout.fileno()
+            fd2 = self._stderr.fileno()
+            self._same_output = self._is_same_output(fd1, fd2)
+        except AttributeError:
+            self._same_output = self._stdout == self._stderr
+        self._stdout_waiting = None
+        self._debug = deque(maxlen=maxlen + 1)
+        self._keep_if_debug = self.THROW
+        self._queue_is_active = False
+
+    @staticmethod
+    def _is_same_output(fd1, fd2):
+        if fd1 == fd2:
+            return True
+        stat1 = os.fstat(fd1)
+        stat2 = os.fstat(fd2)
+        return stat1.st_ino == stat2.st_ino and stat1.st_dev == stat2.st_dev
+
+    # possible values for _stdout_waiting
+    WAITING = 1
+    INTERRUPTED = 2
+
+    # possible values for _keep_if_debug
+    THROW = 0
+    KEEP = 1
+    PRINT = 2
+
+    def emit(self, record):
+        try:
+            if record.levelno == logging.INFO:
+                stream = self._stdout
+                msg = record.getMessage()
+                if (self._stdout_waiting == self.INTERRUPTED and
+                        self._same_output):
+                    msg = ' ... %s' % msg
+                self._stdout_waiting = msg.endswith('... ')
+                if msg.endswith('... '):
+                    self._stdout_waiting = self.WAITING
+                else:
+                    self._stdout_waiting = None
+                    msg = '%s\n' % msg
+            elif (record.levelno < logging.INFO and
+                    self._keep_if_debug != self.PRINT):
+                if self._keep_if_debug == self.KEEP:
+                    self._debug.append(record)
+                return
+            else:
+                if record.levelno >= logging.ERROR and len(self._debug):
+                    self._emit_queue()
+
+                if self._stdout_waiting == self.WAITING and self._same_output:
+                    self._stdout_waiting = self.INTERRUPTED
+                    self._stdout.write('\n')
+                    self._stdout.flush()
+                stream = self._stderr
+                msg = '%s\n' % self.format(record)
+            stream.write(msg)
+            stream.flush()
+        except (KeyboardInterrupt, SystemExit):
+            raise
+        except:
+            self.handleError(record)
+
+    @contextmanager
+    def queue_debug(self):
+        if self._queue_is_active:
+            yield
+            return
+        self._queue_is_active = True
+        self._keep_if_debug = self.KEEP
+        try:
+            yield
+        except Exception:
+            self._emit_queue()
+            # The exception will be handled and very probably printed out by
+            # something upper in the stack.
+            raise
+        finally:
+            self._queue_is_active = False
+        self._keep_if_debug = self.THROW
+        self._debug.clear()
+
+    def _emit_queue(self):
+        self._keep_if_debug = self.PRINT
+        if len(self._debug) == self._debug.maxlen:
+            r = self._debug.popleft()
+            self.emit(logging.LogRecord(
+                r.name, r.levelno, r.pathname, r.lineno,
+                '<truncated - see config.log for full output>',
+                (), None))
+        while True:
+            try:
+                self.emit(self._debug.popleft())
+            except IndexError:
+                break
+        self._keep_if_debug = self.KEEP
+
+
+class LineIO(object):
+    '''File-like class that sends each line of the written data to a callback
+    (without carriage returns).
+    '''
+    def __init__(self, callback):
+        self._callback = callback
+        self._buf = ''
+        self._encoding = getpreferredencoding()
+
+    def write(self, buf):
+        if self._encoding and isinstance(buf, str):
+            buf = buf.decode(self._encoding)
+        lines = buf.splitlines()
+        if not lines:
+            return
+        if self._buf:
+            lines[0] = self._buf + lines[0]
+            self._buf = ''
+        if not buf.endswith('\n'):
+            self._buf = lines.pop()
+
+        for line in lines:
+            self._callback(line)
+
+    def close(self):
+        if self._buf:
+            self._callback(self._buf)
+            self._buf = ''
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()

python/mozbuild/mozbuild/controller/building.py

@@ -0,0 +1,680 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import getpass
+import json
+import logging
+import os
+import platform
+import subprocess
+import sys
+import time
+import which
+
+from collections import (
+    namedtuple,
+    OrderedDict,
+)
+
+try:
+    import psutil
+except Exception:
+    psutil = None
+
+from mozsystemmonitor.resourcemonitor import SystemResourceMonitor
+
+import mozpack.path as mozpath
+
+from ..base import MozbuildObject
+
+from ..testing import install_test_files
+
+from ..compilation.warnings import (
+    WarningsCollector,
+    WarningsDatabase,
+)
+
+from textwrap import TextWrapper
+
+INSTALL_TESTS_CLOBBER = ''.join([TextWrapper().fill(line) + '\n' for line in
+'''
+The build system was unable to install tests because the CLOBBER file has \
+been updated. This means if you edited any test files, your changes may not \
+be picked up until a full/clobber build is performed.
+
+The easiest and fastest way to perform a clobber build is to run:
+
+ $ mach clobber
+ $ mach build
+
+If you did not modify any test files, it is safe to ignore this message \
+and proceed with running tests. To do this run:
+
+ $ touch {clobber_file}
+'''.splitlines()])
+
+
+
+BuildOutputResult = namedtuple('BuildOutputResult',
+    ('warning', 'state_changed', 'for_display'))
+
+
+class TierStatus(object):
+    """Represents the state and progress of tier traversal.
+
+    The build system is organized into linear phases called tiers. Each tier
+    executes in the order it was defined, 1 at a time.
+    """
+
+    def __init__(self, resources):
+        """Accepts a SystemResourceMonitor to record results against."""
+        self.tiers = OrderedDict()
+        self.tier_status = OrderedDict()
+        self.resources = resources
+
+    def set_tiers(self, tiers):
+        """Record the set of known tiers."""
+        for tier in tiers:
+            self.tiers[tier] = dict(
+                begin_time=None,
+                finish_time=None,
+                duration=None,
+            )
+            self.tier_status[tier] = None
+
+    def begin_tier(self, tier):
+        """Record that execution of a tier has begun."""
+        self.tier_status[tier] = 'active'
+        t = self.tiers[tier]
+        # We should ideally use a monotonic clock here. Unfortunately, we won't
+        # have one until Python 3.
+        t['begin_time'] = time.time()
+        self.resources.begin_phase(tier)
+
+    def finish_tier(self, tier):
+        """Record that execution of a tier has finished."""
+        self.tier_status[tier] = 'finished'
+        t = self.tiers[tier]
+        t['finish_time'] = time.time()
+        t['duration'] = self.resources.finish_phase(tier)
+
+    def tiered_resource_usage(self):
+        """Obtains an object containing resource usage for tiers.
+
+        The returned object is suitable for serialization.
+        """
+        o = []
+
+        for tier, state in self.tiers.items():
+            t_entry = dict(
+                name=tier,
+                start=state['begin_time'],
+                end=state['finish_time'],
+                duration=state['duration'],
+            )
+
+            self.add_resources_to_dict(t_entry, phase=tier)
+
+            o.append(t_entry)
+
+        return o
+
+    def add_resources_to_dict(self, entry, start=None, end=None, phase=None):
+        """Helper function to append resource information to a dict."""
+        cpu_percent = self.resources.aggregate_cpu_percent(start=start,
+            end=end, phase=phase, per_cpu=False)
+        cpu_times = self.resources.aggregate_cpu_times(start=start, end=end,
+            phase=phase, per_cpu=False)
+        io = self.resources.aggregate_io(start=start, end=end, phase=phase)
+
+        if cpu_percent is None:
+            return entry
+
+        entry['cpu_percent'] = cpu_percent
+        entry['cpu_times'] = list(cpu_times)
+        entry['io'] = list(io)
+
+        return entry
+
+    def add_resource_fields_to_dict(self, d):
+        for usage in self.resources.range_usage():
+            cpu_times = self.resources.aggregate_cpu_times(per_cpu=False)
+
+            d['cpu_times_fields'] = list(cpu_times._fields)
+            d['io_fields'] = list(usage.io._fields)
+            d['virt_fields'] = list(usage.virt._fields)
+            d['swap_fields'] = list(usage.swap._fields)
+
+            return d
+
+
+class BuildMonitor(MozbuildObject):
+    """Monitors the output of the build."""
+
+    def init(self, warnings_path):
+        """Create a new monitor.
+
+        warnings_path is a path of a warnings database to use.
+        """
+        self._warnings_path = warnings_path
+        self.resources = SystemResourceMonitor(poll_interval=1.0)
+        self._resources_started = False
+
+        self.tiers = TierStatus(self.resources)
+
+        self.warnings_database = WarningsDatabase()
+        if os.path.exists(warnings_path):
+            try:
+                self.warnings_database.load_from_file(warnings_path)
+            except ValueError:
+                os.remove(warnings_path)
+
+        self._warnings_collector = WarningsCollector(
+            database=self.warnings_database, objdir=self.topobjdir)
+
+        self.build_objects = []
+
+    def start(self):
+        """Record the start of the build."""
+        self.start_time = time.time()
+        self._finder_start_cpu = self._get_finder_cpu_usage()
+
+    def start_resource_recording(self):
+        # This should be merged into start() once bug 892342 lands.
+        self.resources.start()
+        self._resources_started = True
+
+    def on_line(self, line):
+        """Consume a line of output from the build system.
+
+        This will parse the line for state and determine whether more action is
+        needed.
+
+        Returns a BuildOutputResult instance.
+
+        In this named tuple, warning will be an object describing a new parsed
+        warning. Otherwise it will be None.
+
+        state_changed indicates whether the build system changed state with
+        this line. If the build system changed state, the caller may want to
+        query this instance for the current state in order to update UI, etc.
+
+        for_display is a boolean indicating whether the line is relevant to the
+        user. This is typically used to filter whether the line should be
+        presented to the user.
+        """
+        if line.startswith('BUILDSTATUS'):
+            args = line.split()[1:]
+
+            action = args.pop(0)
+            update_needed = True
+
+            if action == 'TIERS':
+                self.tiers.set_tiers(args)
+                update_needed = False
+            elif action == 'TIER_START':
+                tier = args[0]
+                self.tiers.begin_tier(tier)
+            elif action == 'TIER_FINISH':
+                tier, = args
+                self.tiers.finish_tier(tier)
+            elif action == 'OBJECT_FILE':
+                self.build_objects.append(args[0])
+                update_needed = False
+            else:
+                raise Exception('Unknown build status: %s' % action)
+
+            return BuildOutputResult(None, update_needed, False)
+
+        warning = None
+
+        try:
+            warning = self._warnings_collector.process_line(line)
+        except:
+            pass
+
+        return BuildOutputResult(warning, False, True)
+
+    def stop_resource_recording(self):
+        if self._resources_started:
+            self.resources.stop()
+
+        self._resources_started = False
+
+    def finish(self, record_usage=True):
+        """Record the end of the build."""
+        self.stop_resource_recording()
+        self.end_time = time.time()
+        self._finder_end_cpu = self._get_finder_cpu_usage()
+        self.elapsed = self.end_time - self.start_time
+
+        self.warnings_database.prune()
+        self.warnings_database.save_to_file(self._warnings_path)
+
+        if not record_usage:
+            return
+
+        try:
+            usage = self.get_resource_usage()
+            if not usage:
+                return
+
+            self.log_resource_usage(usage)
+            with open(self._get_state_filename('build_resources.json'), 'w') as fh:
+                json.dump(self.resources.as_dict(), fh, indent=2)
+        except Exception as e:
+            self.log(logging.WARNING, 'build_resources_error',
+                {'msg': str(e)},
+                'Exception when writing resource usage file: {msg}')
+
+    def _get_finder_cpu_usage(self):
+        """Obtain the CPU usage of the Finder app on OS X.
+
+        This is used to detect high CPU usage.
+        """
+        if not sys.platform.startswith('darwin'):
+            return None
+
+        if not psutil:
+            return None
+
+        for proc in psutil.process_iter():
+            if proc.name != 'Finder':
+                continue
+
+            if proc.username != getpass.getuser():
+                continue
+
+            # Try to isolate system finder as opposed to other "Finder"
+            # processes.
+            if not proc.exe.endswith('CoreServices/Finder.app/Contents/MacOS/Finder'):
+                continue
+
+            return proc.get_cpu_times()
+
+        return None
+
+    def have_high_finder_usage(self):
+        """Determine whether there was high Finder CPU usage during the build.
+
+        Returns True if there was high Finder CPU usage, False if there wasn't,
+        or None if there is nothing to report.
+        """
+        if not self._finder_start_cpu:
+            return None, None
+
+        # We only measure if the measured range is sufficiently long.
+        if self.elapsed < 15:
+            return None, None
+
+        if not self._finder_end_cpu:
+            return None, None
+
+        start = self._finder_start_cpu
+        end = self._finder_end_cpu
+
+        start_total = start.user + start.system
+        end_total = end.user + end.system
+
+        cpu_seconds = end_total - start_total
+
+        # If Finder used more than 25% of 1 core during the build, report an
+        # error.
+        finder_percent = cpu_seconds / self.elapsed * 100
+
+        return finder_percent > 25, finder_percent
+
+    def have_excessive_swapping(self):
+        """Determine whether there was excessive swapping during the build.
+
+        Returns a tuple of (excessive, swap_in, swap_out). All values are None
+        if no swap information is available.
+        """
+        if not self.have_resource_usage:
+            return None, None, None
+
+        swap_in = sum(m.swap.sin for m in self.resources.measurements)
+        swap_out = sum(m.swap.sout for m in self.resources.measurements)
+
+        # The threshold of 1024 MB has been arbitrarily chosen.
+        #
+        # Choosing a proper value that is ideal for everyone is hard. We will
+        # likely iterate on the logic until people are generally satisfied.
+        # If a value is too low, the eventual warning produced does not carry
+        # much meaning. If the threshold is too high, people may not see the
+        # warning and the warning will thus be ineffective.
+        excessive = swap_in > 512 * 1048576 or swap_out > 512 * 1048576
+        return excessive, swap_in, swap_out
+
+    @property
+    def have_resource_usage(self):
+        """Whether resource usage is available."""
+        return self.resources.start_time is not None
+
+    def get_resource_usage(self):
+        """ Produce a data structure containing the low-level resource usage information.
+
+        This data structure can e.g. be serialized into JSON and saved for
+        subsequent analysis.
+
+        If no resource usage is available, None is returned.
+        """
+        if not self.have_resource_usage:
+            return None
+
+        cpu_percent = self.resources.aggregate_cpu_percent(phase=None,
+            per_cpu=False)
+        cpu_times = self.resources.aggregate_cpu_times(phase=None,
+            per_cpu=False)
+        io = self.resources.aggregate_io(phase=None)
+
+        o = dict(
+            version=3,
+            argv=sys.argv,
+            start=self.start_time,
+            end=self.end_time,
+            duration=self.end_time - self.start_time,
+            resources=[],
+            cpu_percent=cpu_percent,
+            cpu_times=cpu_times,
+            io=io,
+            objects=self.build_objects
+        )
+
+        o['tiers'] = self.tiers.tiered_resource_usage()
+
+        self.tiers.add_resource_fields_to_dict(o)
+
+        for usage in self.resources.range_usage():
+            cpu_percent = self.resources.aggregate_cpu_percent(usage.start,
+                usage.end, per_cpu=False)
+            cpu_times = self.resources.aggregate_cpu_times(usage.start,
+                usage.end, per_cpu=False)
+
+            entry = dict(
+                start=usage.start,
+                end=usage.end,
+                virt=list(usage.virt),
+                swap=list(usage.swap),
+            )
+
+            self.tiers.add_resources_to_dict(entry, start=usage.start,
+                    end=usage.end)
+
+            o['resources'].append(entry)
+
+
+        # If the imports for this file ran before the in-tree virtualenv
+        # was bootstrapped (for instance, for a clobber build in automation),
+        # psutil might not be available.
+        #
+        # Treat psutil as optional to avoid an outright failure to log resources
+        # TODO: it would be nice to collect data on the storage device as well
+        # in this case.
+        o['system'] = {}
+        if psutil:
+            o['system'].update(dict(
+                logical_cpu_count=psutil.cpu_count(),
+                physical_cpu_count=psutil.cpu_count(logical=False),
+                swap_total=psutil.swap_memory()[0],
+                vmem_total=psutil.virtual_memory()[0],
+            ))
+
+        return o
+
+    def log_resource_usage(self, usage):
+        """Summarize the resource usage of this build in a log message."""
+
+        if not usage:
+            return
+
+        params = dict(
+            duration=self.end_time - self.start_time,
+            cpu_percent=usage['cpu_percent'],
+            io_read_bytes=usage['io'].read_bytes,
+            io_write_bytes=usage['io'].write_bytes,
+            io_read_time=usage['io'].read_time,
+            io_write_time=usage['io'].write_time,
+        )
+
+        message = 'Overall system resources - Wall time: {duration:.0f}s; ' \
+            'CPU: {cpu_percent:.0f}%; ' \
+            'Read bytes: {io_read_bytes}; Write bytes: {io_write_bytes}; ' \
+            'Read time: {io_read_time}; Write time: {io_write_time}'
+
+        self.log(logging.WARNING, 'resource_usage', params, message)
+
+        excessive, sin, sout = self.have_excessive_swapping()
+        if excessive is not None and (sin or sout):
+            sin /= 1048576
+            sout /= 1048576
+            self.log(logging.WARNING, 'swap_activity',
+                {'sin': sin, 'sout': sout},
+                'Swap in/out (MB): {sin}/{sout}')
+
+    def ccache_stats(self):
+        ccache_stats = None
+
+        try:
+            ccache = which.which('ccache')
+            output = subprocess.check_output([ccache, '-s'])
+            ccache_stats = CCacheStats(output)
+        except which.WhichError:
+            pass
+        except ValueError as e:
+            self.log(logging.WARNING, 'ccache', {'msg': str(e)}, '{msg}')
+
+        return ccache_stats
+
+
+class CCacheStats(object):
+    """Holds statistics from ccache.
+
+    Instances can be subtracted from each other to obtain differences.
+    print() or str() the object to show a ``ccache -s`` like output
+    of the captured stats.
+
+    """
+    STATS_KEYS = [
+        # (key, description)
+        # Refer to stats.c in ccache project for all the descriptions.
+        ('cache_hit_direct', 'cache hit (direct)'),
+        ('cache_hit_preprocessed', 'cache hit (preprocessed)'),
+        ('cache_hit_rate', 'cache hit rate'),
+        ('cache_miss', 'cache miss'),
+        ('link', 'called for link'),
+        ('preprocessing', 'called for preprocessing'),
+        ('multiple', 'multiple source files'),
+        ('stdout', 'compiler produced stdout'),
+        ('no_output', 'compiler produced no output'),
+        ('empty_output', 'compiler produced empty output'),
+        ('failed', 'compile failed'),
+        ('error', 'ccache internal error'),
+        ('preprocessor_error', 'preprocessor error'),
+        ('cant_use_pch', "can't use precompiled header"),
+        ('compiler_missing', "couldn't find the compiler"),
+        ('cache_file_missing', 'cache file missing'),
+        ('bad_args', 'bad compiler arguments'),
+        ('unsupported_lang', 'unsupported source language'),
+        ('compiler_check_failed', 'compiler check failed'),
+        ('autoconf', 'autoconf compile/link'),
+        ('unsupported_compiler_option', 'unsupported compiler option'),
+        ('out_stdout', 'output to stdout'),
+        ('out_device', 'output to a non-regular file'),
+        ('no_input', 'no input file'),
+        ('bad_extra_file', 'error hashing extra file'),
+        ('num_cleanups', 'cleanups performed'),
+        ('cache_files', 'files in cache'),
+        ('cache_size', 'cache size'),
+        ('cache_max_size', 'max cache size'),
+    ]
+
+    DIRECTORY_DESCRIPTION = "cache directory"
+    PRIMARY_CONFIG_DESCRIPTION = "primary config"
+    SECONDARY_CONFIG_DESCRIPTION = "secondary config      (readonly)"
+    ABSOLUTE_KEYS = {'cache_files', 'cache_size', 'cache_max_size'}
+    FORMAT_KEYS = {'cache_size', 'cache_max_size'}
+
+    GiB = 1024 ** 3
+    MiB = 1024 ** 2
+    KiB = 1024
+
+    def __init__(self, output=None):
+        """Construct an instance from the output of ccache -s."""
+        self._values = {}
+        self.cache_dir = ""
+        self.primary_config = ""
+        self.secondary_config = ""
+
+        if not output:
+            return
+
+        for line in output.splitlines():
+            line = line.strip()
+            if line:
+                self._parse_line(line)
+
+    def _parse_line(self, line):
+        if line.startswith(self.DIRECTORY_DESCRIPTION):
+            self.cache_dir = self._strip_prefix(line, self.DIRECTORY_DESCRIPTION)
+        elif line.startswith(self.PRIMARY_CONFIG_DESCRIPTION):
+            self.primary_config = self._strip_prefix(
+                line, self.PRIMARY_CONFIG_DESCRIPTION)
+        elif line.startswith(self.SECONDARY_CONFIG_DESCRIPTION):
+            self.secondary_config = self._strip_prefix(
+                line, self.SECONDARY_CONFIG_DESCRIPTION)
+        else:
+            for stat_key, stat_description in self.STATS_KEYS:
+                if line.startswith(stat_description):
+                    raw_value = self._strip_prefix(line, stat_description)
+                    self._values[stat_key] = self._parse_value(raw_value)
+                    break
+            else:
+                raise ValueError('Failed to parse ccache stats output: %s' % line)
+
+    @staticmethod
+    def _strip_prefix(line, prefix):
+        return line[len(prefix):].strip() if line.startswith(prefix) else line
+
+    @staticmethod
+    def _parse_value(raw_value):
+        value = raw_value.split()
+        unit = ''
+        if len(value) == 1:
+            numeric = value[0]
+        elif len(value) == 2:
+            numeric, unit = value
+        else:
+            raise ValueError('Failed to parse ccache stats value: %s' % raw_value)
+
+        if '.' in numeric:
+            numeric = float(numeric)
+        else:
+            numeric = int(numeric)
+
+        if unit in ('GB', 'Gbytes'):
+            unit = CCacheStats.GiB
+        elif unit in ('MB', 'Mbytes'):
+            unit = CCacheStats.MiB
+        elif unit in ('KB', 'Kbytes'):
+            unit = CCacheStats.KiB
+        else:
+            unit = 1
+
+        return int(numeric * unit)
+
+    def hit_rate_message(self):
+        return 'ccache (direct) hit rate: {:.1%}; (preprocessed) hit rate: {:.1%}; miss rate: {:.1%}'.format(*self.hit_rates())
+
+    def hit_rates(self):
+        direct = self._values['cache_hit_direct']
+        preprocessed = self._values['cache_hit_preprocessed']
+        miss = self._values['cache_miss']
+        total = float(direct + preprocessed + miss)
+
+        if total > 0:
+            direct /= total
+            preprocessed /= total
+            miss /= total
+
+        return (direct, preprocessed, miss)
+
+    def __sub__(self, other):
+        result = CCacheStats()
+        result.cache_dir = self.cache_dir
+
+        for k, prefix in self.STATS_KEYS:
+            if k not in self._values and k not in other._values:
+                continue
+
+            our_value = self._values.get(k, 0)
+            other_value = other._values.get(k, 0)
+
+            if k in self.ABSOLUTE_KEYS:
+                result._values[k] = our_value
+            else:
+                result._values[k] = our_value - other_value
+
+        return result
+
+    def __str__(self):
+        LEFT_ALIGN = 34
+        lines = []
+
+        if self.cache_dir:
+            lines.append('%s%s' % (self.DIRECTORY_DESCRIPTION.ljust(LEFT_ALIGN),
+                                   self.cache_dir))
+
+        for stat_key, stat_description in self.STATS_KEYS:
+            if stat_key not in self._values:
+                continue
+
+            value = self._values[stat_key]
+
+            if stat_key in self.FORMAT_KEYS:
+                value = '%15s' % self._format_value(value)
+            else:
+                value = '%8u' % value
+
+            lines.append('%s%s' % (stat_description.ljust(LEFT_ALIGN), value))
+
+        return '\n'.join(lines)
+
+    def __nonzero__(self):
+        relative_values = [v for k, v in self._values.items()
+                           if k not in self.ABSOLUTE_KEYS]
+        return (all(v >= 0 for v in relative_values) and
+                any(v > 0 for v in relative_values))
+
+    @staticmethod
+    def _format_value(v):
+        if v > CCacheStats.GiB:
+            return '%.1f Gbytes' % (float(v) / CCacheStats.GiB)
+        elif v > CCacheStats.MiB:
+            return '%.1f Mbytes' % (float(v) / CCacheStats.MiB)
+        else:
+            return '%.1f Kbytes' % (float(v) / CCacheStats.KiB)
+
+
+class BuildDriver(MozbuildObject):
+    """Provides a high-level API for build actions."""
+
+    def install_tests(self, test_objs):
+        """Install test files."""
+
+        if self.is_clobber_needed():
+            print(INSTALL_TESTS_CLOBBER.format(
+                  clobber_file=os.path.join(self.topobjdir, 'CLOBBER')))
+            sys.exit(1)
+
+        if not test_objs:
+            # If we don't actually have a list of tests to install we install
+            # test and support files wholesale.
+            self._run_make(target='install-test-files', pass_thru=True,
+                           print_directory=False)
+        else:
+            install_test_files(mozpath.normpath(self.topsrcdir), self.topobjdir,
+                               '_tests', test_objs)

python/mozbuild/mozbuild/controller/clobber.py

@@ -0,0 +1,237 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function
+
+r'''This module contains code for managing clobbering of the tree.'''
+
+import errno
+import os
+import subprocess
+import sys
+
+from mozfile.mozfile import remove as mozfileremove
+from textwrap import TextWrapper
+
+
+CLOBBER_MESSAGE = ''.join([TextWrapper().fill(line) + '\n' for line in
+'''
+The CLOBBER file has been updated, indicating that an incremental build since \
+your last build will probably not work. A full/clobber build is required.
+
+The reason for the clobber is:
+
+{clobber_reason}
+
+Clobbering can be performed automatically. However, we didn't automatically \
+clobber this time because:
+
+{no_reason}
+
+The easiest and fastest way to clobber is to run:
+
+ $ mach clobber
+
+If you know this clobber doesn't apply to you or you're feeling lucky -- \
+Well, are ya? -- you can ignore this clobber requirement by running:
+
+ $ touch {clobber_file}
+'''.splitlines()])
+
+class Clobberer(object):
+    def __init__(self, topsrcdir, topobjdir):
+        """Create a new object to manage clobbering the tree.
+
+        It is bound to a top source directory and to a specific object
+        directory.
+        """
+        assert os.path.isabs(topsrcdir)
+        assert os.path.isabs(topobjdir)
+
+        self.topsrcdir = os.path.normpath(topsrcdir)
+        self.topobjdir = os.path.normpath(topobjdir)
+        self.src_clobber = os.path.join(topsrcdir, 'CLOBBER')
+        self.obj_clobber = os.path.join(topobjdir, 'CLOBBER')
+
+        # Try looking for mozilla/CLOBBER, for comm-central
+        if not os.path.isfile(self.src_clobber):
+            self.src_clobber = os.path.join(topsrcdir, 'mozilla', 'CLOBBER')
+
+        assert os.path.isfile(self.src_clobber)
+
+    def clobber_needed(self):
+        """Returns a bool indicating whether a tree clobber is required."""
+
+        # No object directory clobber file means we're good.
+        if not os.path.exists(self.obj_clobber):
+            return False
+
+        # Object directory clobber older than current is fine.
+        if os.path.getmtime(self.src_clobber) <= \
+            os.path.getmtime(self.obj_clobber):
+
+            return False
+
+        return True
+
+    def clobber_cause(self):
+        """Obtain the cause why a clobber is required.
+
+        This reads the cause from the CLOBBER file.
+
+        This returns a list of lines describing why the clobber was required.
+        Each line is stripped of leading and trailing whitespace.
+        """
+        with open(self.src_clobber, 'rt') as fh:
+            lines = [l.strip() for l in fh.readlines()]
+            return [l for l in lines if l and not l.startswith('#')]
+
+    def have_winrm(self):
+        # `winrm -h` should print 'winrm version ...' and exit 1
+        try:
+            p = subprocess.Popen(['winrm.exe', '-h'],
+                                 stdout=subprocess.PIPE,
+                                 stderr=subprocess.STDOUT)
+            return p.wait() == 1 and p.stdout.read().startswith('winrm')
+        except:
+            return False
+
+    def remove_objdir(self, full=True):
+        """Remove the object directory.
+
+        ``full`` controls whether to fully delete the objdir. If False,
+        some directories (e.g. Visual Studio Project Files) will not be
+        deleted.
+        """
+        # Top-level files and directories to not clobber by default.
+        no_clobber = {
+            '.mozbuild',
+            'msvc',
+        }
+
+        if full:
+            # mozfile doesn't like unicode arguments (bug 818783).
+            paths = [self.topobjdir.encode('utf-8')]
+        else:
+            try:
+                paths = []
+                for p in os.listdir(self.topobjdir):
+                    if p not in no_clobber:
+                        paths.append(os.path.join(self.topobjdir, p).encode('utf-8'))
+            except OSError as e:
+                if e.errno != errno.ENOENT:
+                    raise
+                return
+
+        procs = []
+        for p in sorted(paths):
+            path = os.path.join(self.topobjdir, p)
+            if sys.platform.startswith('win') and self.have_winrm() and os.path.isdir(path):
+                procs.append(subprocess.Popen(['winrm', '-rf', path]))
+            else:
+                # We use mozfile because it is faster than shutil.rmtree().
+                mozfileremove(path)
+
+        for p in procs:
+            p.wait()
+
+    def ensure_objdir_state(self):
+        """Ensure the CLOBBER file in the objdir exists.
+
+        This is called as part of the build to ensure the clobber information
+        is configured properly for the objdir.
+        """
+        if not os.path.exists(self.topobjdir):
+            os.makedirs(self.topobjdir)
+
+        if not os.path.exists(self.obj_clobber):
+            # Simply touch the file.
+            with open(self.obj_clobber, 'a'):
+                pass
+
+    def maybe_do_clobber(self, cwd, allow_auto=False, fh=sys.stderr):
+        """Perform a clobber if it is required. Maybe.
+
+        This is the API the build system invokes to determine if a clobber
+        is needed and to automatically perform that clobber if we can.
+
+        This returns a tuple of (bool, bool, str). The elements are:
+
+          - Whether a clobber was/is required.
+          - Whether a clobber was performed.
+          - The reason why the clobber failed or could not be performed. This
+            will be None if no clobber is required or if we clobbered without
+            error.
+        """
+        assert cwd
+        cwd = os.path.normpath(cwd)
+
+        if not self.clobber_needed():
+            print('Clobber not needed.', file=fh)
+            self.ensure_objdir_state()
+            return False, False, None
+
+        # So a clobber is needed. We only perform a clobber if we are
+        # allowed to perform an automatic clobber (off by default) and if the
+        # current directory is not under the object directory. The latter is
+        # because operating systems, filesystems, and shell can throw fits
+        # if the current working directory is deleted from under you. While it
+        # can work in some scenarios, we take the conservative approach and
+        # never try.
+        if not allow_auto:
+            return True, False, \
+               self._message('Automatic clobbering is not enabled\n'
+                              '  (add "mk_add_options AUTOCLOBBER=1" to your '
+                              'mozconfig).')
+
+        if cwd.startswith(self.topobjdir) and cwd != self.topobjdir:
+            return True, False, self._message(
+                'Cannot clobber while the shell is inside the object directory.')
+
+        print('Automatically clobbering %s' % self.topobjdir, file=fh)
+        try:
+            self.remove_objdir(False)
+            self.ensure_objdir_state()
+            print('Successfully completed auto clobber.', file=fh)
+            return True, True, None
+        except (IOError) as error:
+            return True, False, self._message(
+                'Error when automatically clobbering: ' + str(error))
+
+    def _message(self, reason):
+        lines = [' ' + line for line in self.clobber_cause()]
+
+        return CLOBBER_MESSAGE.format(clobber_reason='\n'.join(lines),
+            no_reason='  ' + reason, clobber_file=self.obj_clobber)
+
+
+def main(args, env, cwd, fh=sys.stderr):
+    if len(args) != 2:
+        print('Usage: clobber.py topsrcdir topobjdir', file=fh)
+        return 1
+
+    topsrcdir, topobjdir = args
+
+    if not os.path.isabs(topsrcdir):
+        topsrcdir = os.path.abspath(topsrcdir)
+
+    if not os.path.isabs(topobjdir):
+        topobjdir = os.path.abspath(topobjdir)
+
+    auto = True if env.get('AUTOCLOBBER', False) else False
+    clobber = Clobberer(topsrcdir, topobjdir)
+    required, performed, message = clobber.maybe_do_clobber(cwd, auto, fh)
+
+    if not required or performed:
+        if performed and env.get('TINDERBOX_OUTPUT'):
+            print('TinderboxPrint: auto clobber', file=fh)
+        return 0
+
+    print(message, file=fh)
+    return 1
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:], os.environ, os.getcwd(), sys.stdout))
+

python/mozbuild/mozbuild/doctor.py

@@ -0,0 +1,293 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import os
+import subprocess
+import sys
+
+import psutil
+
+from distutils.util import strtobool
+from distutils.version import LooseVersion
+import mozpack.path as mozpath
+
+# Minimum recommended logical processors in system.
+PROCESSORS_THRESHOLD = 4
+
+# Minimum recommended total system memory, in gigabytes.
+MEMORY_THRESHOLD = 7.4
+
+# Minimum recommended free space on each disk, in gigabytes.
+FREESPACE_THRESHOLD = 10
+
+# Latest MozillaBuild version
+LATEST_MOZILLABUILD_VERSION = '1.11.0'
+
+DISABLE_LASTACCESS_WIN = '''
+Disable the last access time feature?
+This improves the speed of file and
+directory access by deferring Last Access Time modification on disk by up to an
+hour. Backup programs that rely on this feature may be affected.
+https://technet.microsoft.com/en-us/library/cc785435.aspx
+'''
+
+class Doctor(object):
+    def __init__(self, srcdir, objdir, fix):
+        self.srcdir = mozpath.normpath(srcdir)
+        self.objdir = mozpath.normpath(objdir)
+        self.srcdir_mount = self.getmount(self.srcdir)
+        self.objdir_mount = self.getmount(self.objdir)
+        self.path_mounts = [
+            ('srcdir', self.srcdir, self.srcdir_mount),
+            ('objdir', self.objdir, self.objdir_mount)
+        ]
+        self.fix = fix
+        self.results = []
+
+    def check_all(self):
+        checks = [
+            'cpu',
+            'memory',
+            'storage_freespace',
+            'fs_lastaccess',
+            'mozillabuild'
+        ]
+        for check in checks:
+            self.report(getattr(self, check))
+        good = True
+        fixable = False
+        denied = False
+        for result in self.results:
+            if result.get('status') != 'GOOD':
+                good = False
+            if result.get('fixable', False):
+                fixable = True
+            if result.get('denied', False):
+                denied = True
+        if denied:
+            print('run "mach doctor --fix" AS ADMIN to re-attempt fixing your system')
+        elif False: # elif fixable:
+            print('run "mach doctor --fix" as admin to attempt fixing your system')
+        return int(not good)
+
+    def getmount(self, path):
+        while path != '/' and not os.path.ismount(path):
+            path = mozpath.abspath(mozpath.join(path, os.pardir))
+        return path
+
+    def prompt_bool(self, prompt, limit=5):
+        ''' Prompts the user with prompt and requires a boolean value. '''
+        valid = False
+        while not valid and limit > 0:
+            try:
+                choice = strtobool(raw_input(prompt + '[Y/N]\n'))
+                valid = True
+            except ValueError:
+                print("ERROR! Please enter a valid option!")
+                limit -= 1
+
+        if limit > 0:
+            return choice
+        else:
+            raise Exception("Error! Reached max attempts of entering option.")
+
+    def report(self, results):
+        # Handle single dict result or list of results.
+        if isinstance(results, dict):
+            results = [results]
+        for result in results:
+            status = result.get('status', 'UNSURE')
+            if status == 'SKIPPED':
+                continue
+            self.results.append(result)
+            print('%s...\t%s\n' % (
+                   result.get('desc', ''),
+                   status
+                )
+            ).expandtabs(40)
+
+    @property
+    def platform(self):
+        platform = getattr(self, '_platform', None)
+        if not platform:
+            platform = sys.platform
+            while platform[-1].isdigit():
+                platform = platform[:-1]
+            setattr(self, '_platform', platform)
+        return platform
+
+    @property
+    def cpu(self):
+        cpu_count = psutil.cpu_count()
+        if cpu_count < PROCESSORS_THRESHOLD:
+            status = 'BAD'
+            desc = '%d logical processors detected, <%d' % (
+                cpu_count, PROCESSORS_THRESHOLD
+            )
+        else:
+            status = 'GOOD'
+            desc = '%d logical processors detected, >=%d' % (
+                cpu_count, PROCESSORS_THRESHOLD
+            )
+        return {'status': status, 'desc': desc}
+
+    @property
+    def memory(self):
+        memory = psutil.virtual_memory().total
+        # Convert to gigabytes.
+        memory_GB = memory / 1024**3.0
+        if memory_GB < MEMORY_THRESHOLD:
+            status = 'BAD'
+            desc = '%.1fGB of physical memory, <%.1fGB' % (
+                memory_GB, MEMORY_THRESHOLD
+            )
+        else:
+            status = 'GOOD'
+            desc = '%.1fGB of physical memory, >%.1fGB' % (
+                memory_GB, MEMORY_THRESHOLD
+            )
+        return {'status': status, 'desc': desc}
+
+    @property
+    def storage_freespace(self):
+        results = []
+        desc = ''
+        mountpoint_line = self.srcdir_mount != self.objdir_mount
+        for (purpose, path, mount) in self.path_mounts:
+            desc += '%s = %s\n' % (purpose, path)
+            if not mountpoint_line:
+                mountpoint_line = True
+                continue
+            try:
+                usage = psutil.disk_usage(mount)
+                freespace, size = usage.free, usage.total
+                freespace_GB = freespace / 1024**3
+                size_GB = size / 1024**3
+                if freespace_GB < FREESPACE_THRESHOLD:
+                    status = 'BAD'
+                    desc += 'mountpoint = %s\n%dGB of %dGB free, <%dGB' % (
+                        mount, freespace_GB, size_GB, FREESPACE_THRESHOLD
+                    )
+                else:
+                    status = 'GOOD'
+                    desc += 'mountpoint = %s\n%dGB of %dGB free, >=%dGB' % (
+                        mount, freespace_GB, size_GB, FREESPACE_THRESHOLD
+                    )
+            except OSError:
+                status = 'UNSURE'
+                desc += 'path invalid'
+            results.append({'status': status, 'desc': desc})
+        return results
+
+    @property
+    def fs_lastaccess(self):
+        results = []
+        if self.platform == 'win':
+            fixable = False
+            denied = False
+            # See 'fsutil behavior':
+            # https://technet.microsoft.com/en-us/library/cc785435.aspx
+            try:
+                command = 'fsutil behavior query disablelastaccess'.split(' ')
+                fsutil_output = subprocess.check_output(command)
+                disablelastaccess = int(fsutil_output.partition('=')[2][1])
+            except subprocess.CalledProcessError:
+                disablelastaccess = -1
+                status = 'UNSURE'
+                desc = 'unable to check lastaccess behavior'
+            if disablelastaccess == 1:
+                status = 'GOOD'
+                desc = 'lastaccess disabled systemwide'
+            elif disablelastaccess == 0:
+                if False: # if self.fix:
+                    choice = self.prompt_bool(DISABLE_LASTACCESS_WIN)
+                    if not choice:
+                        return {'status': 'BAD, NOT FIXED',
+                                'desc': 'lastaccess enabled systemwide'}
+                    try:
+                        command = 'fsutil behavior set disablelastaccess 1'.split(' ')
+                        fsutil_output = subprocess.check_output(command)
+                        status = 'GOOD, FIXED'
+                        desc = 'lastaccess disabled systemwide'
+                    except subprocess.CalledProcessError, e:
+                        desc = 'lastaccess enabled systemwide'
+                        if e.output.find('denied') != -1:
+                            status = 'BAD, FIX DENIED'
+                            denied = True
+                        else:
+                            status = 'BAD, NOT FIXED'
+                else:
+                    status = 'BAD, FIXABLE'
+                    desc = 'lastaccess enabled'
+                    fixable = True
+            results.append({'status': status, 'desc': desc, 'fixable': fixable,
+                            'denied': denied})
+        elif self.platform in ['darwin', 'freebsd', 'linux', 'openbsd']:
+            common_mountpoint = self.srcdir_mount == self.objdir_mount
+            for (purpose, path, mount) in self.path_mounts:
+                results.append(self.check_mount_lastaccess(mount))
+                if common_mountpoint:
+                    break
+        else:
+            results.append({'status': 'SKIPPED'})
+        return results
+
+    def check_mount_lastaccess(self, mount):
+        partitions = psutil.disk_partitions()
+        atime_opts = {'atime', 'noatime', 'relatime', 'norelatime'}
+        option = ''
+        for partition in partitions:
+            if partition.mountpoint == mount:
+                mount_opts = set(partition.opts.split(','))
+                intersection = list(atime_opts & mount_opts)
+                if len(intersection) == 1:
+                    option = intersection[0]
+                break
+        if not option:
+            status = 'BAD'
+            if self.platform == 'linux':
+                option = 'noatime/relatime'
+            else:
+                option = 'noatime'
+            desc = '%s has no explicit %s mount option' % (
+                mount, option
+            )
+        elif option == 'atime' or option == 'norelatime':
+            status = 'BAD'
+            desc = '%s has %s mount option' % (
+                mount, option
+            )
+        elif option == 'noatime' or option == 'relatime':
+            status = 'GOOD'
+            desc = '%s has %s mount option' % (
+                mount, option
+            )
+        return {'status': status, 'desc': desc}
+
+    @property
+    def mozillabuild(self):
+        if self.platform != 'win':
+            return {'status': 'SKIPPED'}
+        MOZILLABUILD = mozpath.normpath(os.environ.get('MOZILLABUILD', ''))
+        if not MOZILLABUILD or not os.path.exists(MOZILLABUILD):
+            return {'desc': 'not running under MozillaBuild'}
+        try:
+            with open(mozpath.join(MOZILLABUILD, 'VERSION'), 'r') as fh:
+                version = fh.readline()
+            if not version:
+                raise ValueError()
+            if LooseVersion(version) < LooseVersion(LATEST_MOZILLABUILD_VERSION):
+                status = 'BAD'
+                desc = 'MozillaBuild %s in use, <%s' % (
+                    version, LATEST_MOZILLABUILD_VERSION
+                )
+            else:
+                status = 'GOOD'
+                desc = 'MozillaBuild %s in use' % version
+        except (IOError, ValueError):
+            status = 'UNSURE'
+            desc = 'MozillaBuild version not found'
+        return {'status': status, 'desc': desc}

python/mozbuild/mozbuild/dotproperties.py

@@ -0,0 +1,83 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file contains utility functions for reading .properties files, like
+# region.properties.
+
+from __future__ import absolute_import, unicode_literals
+
+import codecs
+import re
+import sys
+
+if sys.version_info[0] == 3:
+    str_type = str
+else:
+    str_type = basestring
+
+class DotProperties:
+    r'''A thin representation of a key=value .properties file.'''
+
+    def __init__(self, file=None):
+        self._properties = {}
+        if file:
+            self.update(file)
+
+    def update(self, file):
+        '''Updates properties from a file name or file-like object.
+
+        Ignores empty lines and comment lines.'''
+
+        if isinstance(file, str_type):
+            f = codecs.open(file, 'r', 'utf-8')
+        else:
+            f = file
+
+        for l in f.readlines():
+            line = l.strip()
+            if not line or line.startswith('#'):
+                continue
+            (k, v) = re.split('\s*=\s*', line, 1)
+            self._properties[k] = v
+
+    def get(self, key, default=None):
+        return self._properties.get(key, default)
+
+    def get_list(self, prefix):
+        '''Turns {'list.0':'foo', 'list.1':'bar'} into ['foo', 'bar'].
+
+        Returns [] to indicate an empty or missing list.'''
+
+        if not prefix.endswith('.'):
+            prefix = prefix + '.'
+        indexes = []
+        for k, v in self._properties.iteritems():
+            if not k.startswith(prefix):
+                continue
+            key = k[len(prefix):]
+            if '.' in key:
+                # We have something like list.sublist.0.
+                continue
+            indexes.append(int(key))
+        return [self._properties[prefix + str(index)] for index in sorted(indexes)]
+
+    def get_dict(self, prefix, required_keys=[]):
+        '''Turns {'foo.title':'title', ...} into {'title':'title', ...}.
+
+        If |required_keys| is present, it must be an iterable of required key
+        names.  If a required key is not present, ValueError is thrown.
+
+        Returns {} to indicate an empty or missing dict.'''
+
+        if not prefix.endswith('.'):
+            prefix = prefix + '.'
+
+        D = dict((k[len(prefix):], v) for k, v in self._properties.iteritems()
+                 if k.startswith(prefix) and '.' not in k[len(prefix):])
+
+        for required_key in required_keys:
+            if not required_key in D:
+                raise ValueError('Required key %s not present' % required_key)
+
+        return D

python/mozbuild/mozbuild/frontend/gyp_reader.py

@@ -0,0 +1,248 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import gyp
+import sys
+import os
+import types
+import mozpack.path as mozpath
+from mozpack.files import FileFinder
+from .sandbox import alphabetical_sorted
+from .context import (
+    SourcePath,
+    TemplateContext,
+    VARIABLES,
+)
+from mozbuild.util import (
+    expand_variables,
+    List,
+    memoize,
+)
+from .reader import SandboxValidationError
+
+# Define this module as gyp.generator.mozbuild so that gyp can use it
+# as a generator under the name "mozbuild".
+sys.modules['gyp.generator.mozbuild'] = sys.modules[__name__]
+
+# build/gyp_chromium does this:
+#   script_dir = os.path.dirname(os.path.realpath(__file__))
+#   chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
+#   sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib'))
+# We're not importing gyp_chromium, but we want both script_dir and
+# chrome_src for the default includes, so go backwards from the pylib
+# directory, which is the parent directory of gyp module.
+chrome_src = mozpath.abspath(mozpath.join(mozpath.dirname(gyp.__file__),
+    '../../../..'))
+script_dir = mozpath.join(chrome_src, 'build')
+
+# Default variables gyp uses when evaluating gyp files.
+generator_default_variables = {
+}
+for dirname in ['INTERMEDIATE_DIR', 'SHARED_INTERMEDIATE_DIR', 'PRODUCT_DIR',
+                'LIB_DIR', 'SHARED_LIB_DIR']:
+  # Some gyp steps fail if these are empty(!).
+  generator_default_variables[dirname] = b'dir'
+
+for unused in ['RULE_INPUT_PATH', 'RULE_INPUT_ROOT', 'RULE_INPUT_NAME',
+               'RULE_INPUT_DIRNAME', 'RULE_INPUT_EXT',
+               'EXECUTABLE_PREFIX', 'EXECUTABLE_SUFFIX',
+               'STATIC_LIB_PREFIX', 'STATIC_LIB_SUFFIX',
+               'SHARED_LIB_PREFIX', 'SHARED_LIB_SUFFIX',
+               'LINKER_SUPPORTS_ICF']:
+  generator_default_variables[unused] = b''
+
+
+class GypContext(TemplateContext):
+    """Specialized Context for use with data extracted from Gyp.
+
+    config is the ConfigEnvironment for this context.
+    relobjdir is the object directory that will be used for this context,
+    relative to the topobjdir defined in the ConfigEnvironment.
+    """
+    def __init__(self, config, relobjdir):
+        self._relobjdir = relobjdir
+        TemplateContext.__init__(self, template='Gyp',
+            allowed_variables=VARIABLES, config=config)
+
+
+def encode(value):
+    if isinstance(value, unicode):
+        return value.encode('utf-8')
+    return value
+
+
+def read_from_gyp(config, path, output, vars, non_unified_sources = set()):
+    """Read a gyp configuration and emits GypContexts for the backend to
+    process.
+
+    config is a ConfigEnvironment, path is the path to a root gyp configuration
+    file, output is the base path under which the objdir for the various gyp
+    dependencies will be, and vars a dict of variables to pass to the gyp
+    processor.
+    """
+
+    # gyp expects plain str instead of unicode. The frontend code gives us
+    # unicode strings, so convert them.
+    path = encode(path)
+    str_vars = dict((name, encode(value)) for name, value in vars.items())
+
+    params = {
+        b'parallel': False,
+        b'generator_flags': {},
+        b'build_files': [path],
+        b'root_targets': None,
+    }
+
+    # Files that gyp_chromium always includes
+    includes = [encode(mozpath.join(script_dir, 'common.gypi'))]
+    finder = FileFinder(chrome_src, find_executables=False)
+    includes.extend(encode(mozpath.join(chrome_src, name))
+        for name, _ in finder.find('*/supplement.gypi'))
+
+    # Read the given gyp file and its dependencies.
+    generator, flat_list, targets, data = \
+        gyp.Load([path], format=b'mozbuild',
+            default_variables=str_vars,
+            includes=includes,
+            depth=encode(chrome_src),
+            params=params)
+
+    # Process all targets from the given gyp files and its dependencies.
+    # The path given to AllTargets needs to use os.sep, while the frontend code
+    # gives us paths normalized with forward slash separator.
+    for target in gyp.common.AllTargets(flat_list, targets, path.replace(b'/', os.sep)):
+        build_file, target_name, toolset = gyp.common.ParseQualifiedTarget(target)
+
+        # Each target is given its own objdir. The base of that objdir
+        # is derived from the relative path from the root gyp file path
+        # to the current build_file, placed under the given output
+        # directory. Since several targets can be in a given build_file,
+        # separate them in subdirectories using the build_file basename
+        # and the target_name.
+        reldir  = mozpath.relpath(mozpath.dirname(build_file),
+                                  mozpath.dirname(path))
+        subdir = '%s_%s' % (
+            mozpath.splitext(mozpath.basename(build_file))[0],
+            target_name,
+        )
+        # Emit a context for each target.
+        context = GypContext(config, mozpath.relpath(
+            mozpath.join(output, reldir, subdir), config.topobjdir))
+        context.add_source(mozpath.abspath(build_file))
+        # The list of included files returned by gyp are relative to build_file
+        for f in data[build_file]['included_files']:
+            context.add_source(mozpath.abspath(mozpath.join(
+                mozpath.dirname(build_file), f)))
+
+        spec = targets[target]
+
+        # Derive which gyp configuration to use based on MOZ_DEBUG.
+        c = 'Debug' if config.substs['MOZ_DEBUG'] else 'Release'
+        if c not in spec['configurations']:
+            raise RuntimeError('Missing %s gyp configuration for target %s '
+                               'in %s' % (c, target_name, build_file))
+        target_conf = spec['configurations'][c]
+
+        if spec['type'] == 'none':
+            continue
+        elif spec['type'] == 'static_library':
+            # Remove leading 'lib' from the target_name if any, and use as
+            # library name.
+            name = spec['target_name']
+            if name.startswith('lib'):
+                name = name[3:]
+            # The context expects an unicode string.
+            context['LIBRARY_NAME'] = name.decode('utf-8')
+            # gyp files contain headers and asm sources in sources lists.
+            sources = []
+            unified_sources = []
+            extensions = set()
+            for f in spec.get('sources', []):
+                ext = mozpath.splitext(f)[-1]
+                extensions.add(ext)
+                s = SourcePath(context, f)
+                if ext == '.h':
+                    continue
+                if ext != '.S' and s not in non_unified_sources:
+                    unified_sources.append(s)
+                else:
+                    sources.append(s)
+
+            # The context expects alphabetical order when adding sources
+            context['SOURCES'] = alphabetical_sorted(sources)
+            context['UNIFIED_SOURCES'] = alphabetical_sorted(unified_sources)
+
+            for define in target_conf.get('defines', []):
+                if '=' in define:
+                    name, value = define.split('=', 1)
+                    context['DEFINES'][name] = value
+                else:
+                    context['DEFINES'][define] = True
+
+            for include in target_conf.get('include_dirs', []):
+                # moz.build expects all LOCAL_INCLUDES to exist, so ensure they do.
+                #
+                # NB: gyp files sometimes have actual absolute paths (e.g.
+                # /usr/include32) and sometimes paths that moz.build considers
+                # absolute, i.e. starting from topsrcdir. There's no good way
+                # to tell them apart here, and the actual absolute paths are
+                # likely bogus. In any event, actual absolute paths will be
+                # filtered out by trying to find them in topsrcdir.
+                if include.startswith('/'):
+                    resolved = mozpath.abspath(mozpath.join(config.topsrcdir, include[1:]))
+                else:
+                    resolved = mozpath.abspath(mozpath.join(mozpath.dirname(build_file), include))
+                if not os.path.exists(resolved):
+                    continue
+                context['LOCAL_INCLUDES'] += [include]
+
+            context['ASFLAGS'] = target_conf.get('asflags_mozilla', [])
+            flags = target_conf.get('cflags_mozilla', [])
+            if flags:
+                suffix_map = {
+                    '.c': 'CFLAGS',
+                    '.cpp': 'CXXFLAGS',
+                    '.cc': 'CXXFLAGS',
+                    '.m': 'CMFLAGS',
+                    '.mm': 'CMMFLAGS',
+                }
+                variables = (
+                    suffix_map[e]
+                    for e in extensions if e in suffix_map
+                )
+                for var in variables:
+                    for f in flags:
+                        # We may be getting make variable references out of the
+                        # gyp data, and we don't want those in emitted data, so
+                        # substitute them with their actual value.
+                        f = expand_variables(f, config.substs).split()
+                        if not f:
+                            continue
+                        # the result may be a string or a list.
+                        if isinstance(f, types.StringTypes):
+                            context[var].append(f)
+                        else:
+                            context[var].extend(f)
+        else:
+            # Ignore other types than static_library because we don't have
+            # anything using them, and we're not testing them. They can be
+            # added when that becomes necessary.
+            raise NotImplementedError('Unsupported gyp target type: %s' % spec['type'])
+
+        # Add some features to all contexts. Put here in case LOCAL_INCLUDES
+        # order matters.
+        context['LOCAL_INCLUDES'] += [
+            '!/ipc/ipdl/_ipdlheaders',
+            '/ipc/chromium/src',
+            '/ipc/glue',
+        ]
+        # These get set via VC project file settings for normal GYP builds.
+        if config.substs['OS_TARGET'] == 'WINNT':
+            context['DEFINES']['UNICODE'] = True
+            context['DEFINES']['_UNICODE'] = True
+        context['DISABLE_STL_WRAPPING'] = True
+
+        yield context

python/mozbuild/mozbuild/frontend/mach_commands.py

@@ -0,0 +1,218 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from collections import defaultdict
+import os
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+    SubCommand,
+)
+
+from mozbuild.base import MachCommandBase
+import mozpack.path as mozpath
+
+
+class InvalidPathException(Exception):
+    """Represents an error due to an invalid path."""
+
+
+@CommandProvider
+class MozbuildFileCommands(MachCommandBase):
+    @Command('mozbuild-reference', category='build-dev',
+        description='View reference documentation on mozbuild files.')
+    @CommandArgument('symbol', default=None, nargs='*',
+        help='Symbol to view help on. If not specified, all will be shown.')
+    @CommandArgument('--name-only', '-n', default=False, action='store_true',
+        help='Print symbol names only.')
+    def reference(self, symbol, name_only=False):
+        # mozbuild.sphinx imports some Sphinx modules, so we need to be sure
+        # the optional Sphinx package is installed.
+        self._activate_virtualenv()
+        self.virtualenv_manager.install_pip_package('Sphinx==1.1.3')
+
+        from mozbuild.sphinx import (
+            format_module,
+            function_reference,
+            special_reference,
+            variable_reference,
+        )
+
+        import mozbuild.frontend.context as m
+
+        if name_only:
+            for s in sorted(m.VARIABLES.keys()):
+                print(s)
+
+            for s in sorted(m.FUNCTIONS.keys()):
+                print(s)
+
+            for s in sorted(m.SPECIAL_VARIABLES.keys()):
+                print(s)
+
+            return 0
+
+        if len(symbol):
+            for s in symbol:
+                if s in m.VARIABLES:
+                    for line in variable_reference(s, *m.VARIABLES[s]):
+                        print(line)
+                    continue
+                elif s in m.FUNCTIONS:
+                    for line in function_reference(s, *m.FUNCTIONS[s]):
+                        print(line)
+                    continue
+                elif s in m.SPECIAL_VARIABLES:
+                    for line in special_reference(s, *m.SPECIAL_VARIABLES[s]):
+                        print(line)
+                    continue
+
+                print('Could not find symbol: %s' % s)
+                return 1
+
+            return 0
+
+        for line in format_module(m):
+            print(line)
+
+        return 0
+
+    @Command('file-info', category='build-dev',
+             description='Query for metadata about files.')
+    def file_info(self):
+        """Show files metadata derived from moz.build files.
+
+        moz.build files contain "Files" sub-contexts for declaring metadata
+        against file patterns. This command suite is used to query that data.
+        """
+
+    @SubCommand('file-info', 'bugzilla-component',
+                'Show Bugzilla component info for files listed.')
+    @CommandArgument('-r', '--rev',
+                     help='Version control revision to look up info from')
+    @CommandArgument('paths', nargs='+',
+                     help='Paths whose data to query')
+    def file_info_bugzilla(self, paths, rev=None):
+        """Show Bugzilla component for a set of files.
+
+        Given a requested set of files (which can be specified using
+        wildcards), print the Bugzilla component for each file.
+        """
+        components = defaultdict(set)
+        try:
+            for p, m in self._get_files_info(paths, rev=rev).items():
+                components[m.get('BUG_COMPONENT')].add(p)
+        except InvalidPathException as e:
+            print(e.message)
+            return 1
+
+        for component, files in sorted(components.items(), key=lambda x: (x is None, x)):
+            print('%s :: %s' % (component.product, component.component) if component else 'UNKNOWN')
+            for f in sorted(files):
+                print('  %s' % f)
+
+    @SubCommand('file-info', 'missing-bugzilla',
+                'Show files missing Bugzilla component info')
+    @CommandArgument('-r', '--rev',
+                     help='Version control revision to look up info from')
+    @CommandArgument('paths', nargs='+',
+                     help='Paths whose data to query')
+    def file_info_missing_bugzilla(self, paths, rev=None):
+        try:
+            for p, m in sorted(self._get_files_info(paths, rev=rev).items()):
+                if 'BUG_COMPONENT' not in m:
+                    print(p)
+        except InvalidPathException as e:
+            print(e.message)
+            return 1
+
+    @SubCommand('file-info', 'dep-tests',
+                'Show test files marked as dependencies of these source files.')
+    @CommandArgument('-r', '--rev',
+                     help='Version control revision to look up info from')
+    @CommandArgument('paths', nargs='+',
+                     help='Paths whose data to query')
+    def file_info_test_deps(self, paths, rev=None):
+        try:
+            for p, m in self._get_files_info(paths, rev=rev).items():
+                print('%s:' % mozpath.relpath(p, self.topsrcdir))
+                if m.test_files:
+                    print('\tTest file patterns:')
+                    for p in m.test_files:
+                        print('\t\t%s' % p)
+                if m.test_tags:
+                    print('\tRelevant tags:')
+                    for p in m.test_tags:
+                        print('\t\t%s' % p)
+                if m.test_flavors:
+                    print('\tRelevant flavors:')
+                    for p in m.test_flavors:
+                        print('\t\t%s' % p)
+
+        except InvalidPathException as e:
+            print(e.message)
+            return 1
+
+
+    def _get_reader(self, finder):
+        from mozbuild.frontend.reader import (
+            BuildReader,
+            EmptyConfig,
+        )
+
+        config = EmptyConfig(self.topsrcdir)
+        return BuildReader(config, finder=finder)
+
+    def _get_files_info(self, paths, rev=None):
+        from mozbuild.frontend.reader import default_finder
+        from mozpack.files import FileFinder, MercurialRevisionFinder
+
+        # Normalize to relative from topsrcdir.
+        relpaths = []
+        for p in paths:
+            a = mozpath.abspath(p)
+            if not mozpath.basedir(a, [self.topsrcdir]):
+                raise InvalidPathException('path is outside topsrcdir: %s' % p)
+
+            relpaths.append(mozpath.relpath(a, self.topsrcdir))
+
+        repo = None
+        if rev:
+            hg_path = os.path.join(self.topsrcdir, '.hg')
+            if not os.path.exists(hg_path):
+                raise InvalidPathException('a Mercurial repo is required '
+                        'when specifying a revision')
+
+            repo = self.topsrcdir
+
+        # We need two finders because the reader's finder operates on
+        # absolute paths.
+        finder = FileFinder(self.topsrcdir, find_executables=False)
+        if repo:
+            reader_finder = MercurialRevisionFinder(repo, rev=rev,
+                                                    recognize_repo_paths=True)
+        else:
+            reader_finder = default_finder
+
+        # Expand wildcards.
+        allpaths = []
+        for p in relpaths:
+            if '*' not in p:
+                if p not in allpaths:
+                    allpaths.append(p)
+                continue
+
+            if repo:
+                raise InvalidPathException('cannot use wildcard in version control mode')
+
+            for path, f in finder.find(p):
+                if path not in allpaths:
+                    allpaths.append(path)
+
+        reader = self._get_reader(finder=reader_finder)
+        return reader.files_info(allpaths)

python/mozbuild/mozbuild/frontend/sandbox.py

@@ -0,0 +1,308 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+r"""Python sandbox implementation for build files.
+
+This module contains classes for Python sandboxes that execute in a
+highly-controlled environment.
+
+The main class is `Sandbox`. This provides an execution environment for Python
+code and is used to fill a Context instance for the takeaway information from
+the execution.
+
+Code in this module takes a different approach to exception handling compared
+to what you'd see elsewhere in Python. Arguments to built-in exceptions like
+KeyError are machine parseable. This machine-friendly data is used to present
+user-friendly error messages in the case of errors.
+"""
+
+from __future__ import absolute_import, unicode_literals
+
+import os
+import sys
+import weakref
+
+from mozbuild.util import (
+    exec_,
+    ReadOnlyDict,
+)
+from .context import Context
+from mozpack.files import FileFinder
+
+
+default_finder = FileFinder('/', find_executables=False)
+
+
+def alphabetical_sorted(iterable, cmp=None, key=lambda x: x.lower(),
+                        reverse=False):
+    """sorted() replacement for the sandbox, ordering alphabetically by
+    default.
+    """
+    return sorted(iterable, cmp, key, reverse)
+
+
+class SandboxError(Exception):
+    def __init__(self, file_stack):
+        self.file_stack = file_stack
+
+
+class SandboxExecutionError(SandboxError):
+    """Represents errors encountered during execution of a Sandbox.
+
+    This is a simple container exception. It's purpose is to capture state
+    so something else can report on it.
+    """
+    def __init__(self, file_stack, exc_type, exc_value, trace):
+        SandboxError.__init__(self, file_stack)
+
+        self.exc_type = exc_type
+        self.exc_value = exc_value
+        self.trace = trace
+
+
+class SandboxLoadError(SandboxError):
+    """Represents errors encountered when loading a file for execution.
+
+    This exception represents errors in a Sandbox that occurred as part of
+    loading a file. The error could have occurred in the course of executing
+    a file. If so, the file_stack will be non-empty and the file that caused
+    the load will be on top of the stack.
+    """
+    def __init__(self, file_stack, trace, illegal_path=None, read_error=None):
+        SandboxError.__init__(self, file_stack)
+
+        self.trace = trace
+        self.illegal_path = illegal_path
+        self.read_error = read_error
+
+
+class Sandbox(dict):
+    """Represents a sandbox for executing Python code.
+
+    This class provides a sandbox for execution of a single mozbuild frontend
+    file. The results of that execution is stored in the Context instance given
+    as the ``context`` argument.
+
+    Sandbox is effectively a glorified wrapper around compile() + exec(). You
+    point it at some Python code and it executes it. The main difference from
+    executing Python code like normal is that the executed code is very limited
+    in what it can do: the sandbox only exposes a very limited set of Python
+    functionality. Only specific types and functions are available. This
+    prevents executed code from doing things like import modules, open files,
+    etc.
+
+    Sandbox instances act as global namespace for the sandboxed execution
+    itself. They shall not be used to access the results of the execution.
+    Those results are available in the given Context instance after execution.
+
+    The Sandbox itself is responsible for enforcing rules such as forbidding
+    reassignment of variables.
+
+    Implementation note: Sandbox derives from dict because exec() insists that
+    what it is given for namespaces is a dict.
+    """
+    # The default set of builtins.
+    BUILTINS = ReadOnlyDict({
+        # Only real Python built-ins should go here.
+        'None': None,
+        'False': False,
+        'True': True,
+        'sorted': alphabetical_sorted,
+        'int': int,
+    })
+
+    def __init__(self, context, builtins=None, finder=default_finder):
+        """Initialize a Sandbox ready for execution.
+        """
+        self._builtins = builtins or self.BUILTINS
+        dict.__setitem__(self, '__builtins__', self._builtins)
+
+        assert isinstance(self._builtins, ReadOnlyDict)
+        assert isinstance(context, Context)
+
+        # Contexts are modeled as a stack because multiple context managers
+        # may be active.
+        self._active_contexts = [context]
+
+        # Seen sub-contexts. Will be populated with other Context instances
+        # that were related to execution of this instance.
+        self.subcontexts = []
+
+        # We need to record this because it gets swallowed as part of
+        # evaluation.
+        self._last_name_error = None
+
+        # Current literal source being executed.
+        self._current_source = None
+
+        self._finder = finder
+
+    @property
+    def _context(self):
+        return self._active_contexts[-1]
+
+    def exec_file(self, path):
+        """Execute code at a path in the sandbox.
+
+        The path must be absolute.
+        """
+        assert os.path.isabs(path)
+
+        try:
+            source = self._finder.get(path).read()
+        except Exception as e:
+            raise SandboxLoadError(self._context.source_stack,
+                sys.exc_info()[2], read_error=path)
+
+        self.exec_source(source, path)
+
+    def exec_source(self, source, path=''):
+        """Execute Python code within a string.
+
+        The passed string should contain Python code to be executed. The string
+        will be compiled and executed.
+
+        You should almost always go through exec_file() because exec_source()
+        does not perform extra path normalization. This can cause relative
+        paths to behave weirdly.
+        """
+        def execute():
+            # compile() inherits the __future__ from the module by default. We
+            # do want Unicode literals.
+            code = compile(source, path, 'exec')
+            # We use ourself as the global namespace for the execution. There
+            # is no need for a separate local namespace as moz.build execution
+            # is flat, namespace-wise.
+            old_source = self._current_source
+            self._current_source = source
+            try:
+                exec_(code, self)
+            finally:
+                self._current_source = old_source
+
+        self.exec_function(execute, path=path)
+
+    def exec_function(self, func, args=(), kwargs={}, path='',
+                      becomes_current_path=True):
+        """Execute function with the given arguments in the sandbox.
+        """
+        if path and becomes_current_path:
+            self._context.push_source(path)
+
+        old_sandbox = self._context._sandbox
+        self._context._sandbox = weakref.ref(self)
+
+        # We don't have to worry about bytecode generation here because we are
+        # too low-level for that. However, we could add bytecode generation via
+        # the marshall module if parsing performance were ever an issue.
+
+        old_source = self._current_source
+        self._current_source = None
+        try:
+            func(*args, **kwargs)
+        except SandboxError as e:
+            raise e
+        except NameError as e:
+            # A NameError is raised when a variable could not be found.
+            # The original KeyError has been dropped by the interpreter.
+            # However, we should have it cached in our instance!
+
+            # Unless a script is doing something wonky like catching NameError
+            # itself (that would be silly), if there is an exception on the
+            # global namespace, that's our error.
+            actual = e
+
+            if self._last_name_error is not None:
+                actual = self._last_name_error
+            source_stack = self._context.source_stack
+            if not becomes_current_path:
+                # Add current file to the stack because it wasn't added before
+                # sandbox execution.
+                source_stack.append(path)
+            raise SandboxExecutionError(source_stack, type(actual), actual,
+                                        sys.exc_info()[2])
+
+        except Exception as e:
+            # Need to copy the stack otherwise we get a reference and that is
+            # mutated during the finally.
+            exc = sys.exc_info()
+            source_stack = self._context.source_stack
+            if not becomes_current_path:
+                # Add current file to the stack because it wasn't added before
+                # sandbox execution.
+                source_stack.append(path)
+            raise SandboxExecutionError(source_stack, exc[0], exc[1], exc[2])
+        finally:
+            self._current_source = old_source
+            self._context._sandbox = old_sandbox
+            if path and becomes_current_path:
+                self._context.pop_source()
+
+    def push_subcontext(self, context):
+        """Push a SubContext onto the execution stack.
+
+        When called, the active context will be set to the specified context,
+        meaning all variable accesses will go through it. We also record this
+        SubContext as having been executed as part of this sandbox.
+        """
+        self._active_contexts.append(context)
+        if context not in self.subcontexts:
+            self.subcontexts.append(context)
+
+    def pop_subcontext(self, context):
+        """Pop a SubContext off the execution stack.
+
+        SubContexts must be pushed and popped in opposite order. This is
+        validated as part of the function call to ensure proper consumer API
+        use.
+        """
+        popped = self._active_contexts.pop()
+        assert popped == context
+
+    def __getitem__(self, key):
+        if key.isupper():
+            try:
+                return self._context[key]
+            except Exception as e:
+                self._last_name_error = e
+                raise
+
+        return dict.__getitem__(self, key)
+
+    def __setitem__(self, key, value):
+        if key in self._builtins or key == '__builtins__':
+            raise KeyError('Cannot reassign builtins')
+
+        if key.isupper():
+            # Forbid assigning over a previously set value. Interestingly, when
+            # doing FOO += ['bar'], python actually does something like:
+            #   foo = namespace.__getitem__('FOO')
+            #   foo.__iadd__(['bar'])
+            #   namespace.__setitem__('FOO', foo)
+            # This means __setitem__ is called with the value that is already
+            # in the dict, when doing +=, which is permitted.
+            if key in self._context and self._context[key] is not value:
+                raise KeyError('global_ns', 'reassign', key)
+
+            if (key not in self._context and isinstance(value, (list, dict))
+               and not value):
+                raise KeyError('Variable %s assigned an empty value.' % key)
+
+            self._context[key] = value
+        else:
+            dict.__setitem__(self, key, value)
+
+    def get(self, key, default=None):
+        raise NotImplementedError('Not supported')
+
+    def __len__(self):
+        raise NotImplementedError('Not supported')
+
+    def __iter__(self):
+        raise NotImplementedError('Not supported')
+
+    def __contains__(self, key):
+        if key.isupper():
+            return key in self._context
+        return dict.__contains__(self, key)

python/mozbuild/mozbuild/html_build_viewer.py

@@ -0,0 +1,120 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module contains code for running an HTTP server to view build info.
+
+from __future__ import absolute_import, unicode_literals
+
+import BaseHTTPServer
+import json
+import os
+
+import requests
+
+
+class HTTPHandler(BaseHTTPServer.BaseHTTPRequestHandler):
+    def do_GET(self):
+        s = self.server.wrapper
+        p = self.path
+
+        if p == '/list':
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json; charset=utf-8')
+            self.end_headers()
+
+            keys = sorted(s.json_files.keys())
+            json.dump({'files': keys}, self.wfile)
+            return
+
+        if p.startswith('/resources/'):
+            key = p[len('/resources/'):]
+
+            if key not in s.json_files:
+                self.send_error(404)
+                return
+
+            self.send_response(200)
+            self.send_header('Content-Type', 'application/json; charset=utf-8')
+            self.end_headers()
+
+            self.wfile.write(s.json_files[key])
+            return
+
+        if p == '/':
+            p = '/index.html'
+
+        self.serve_docroot(s.doc_root, p[1:])
+
+    def do_POST(self):
+        if self.path == '/shutdown':
+            self.server.wrapper.do_shutdown = True
+            self.send_response(200)
+            return
+
+        self.send_error(404)
+
+    def serve_docroot(self, root, path):
+        local_path = os.path.normpath(os.path.join(root, path))
+
+        # Cheap security. This doesn't resolve symlinks, etc. But, it should be
+        # acceptable since this server only runs locally.
+        if not local_path.startswith(root):
+            self.send_error(404)
+
+        if not os.path.exists(local_path):
+            self.send_error(404)
+            return
+
+        if os.path.isdir(local_path):
+            self.send_error(500)
+            return
+
+        self.send_response(200)
+        ct = 'text/plain'
+        if path.endswith('.html'):
+            ct = 'text/html'
+
+        self.send_header('Content-Type', ct)
+        self.end_headers()
+
+        with open(local_path, 'rb') as fh:
+            self.wfile.write(fh.read())
+
+
+class BuildViewerServer(object):
+    def __init__(self, address='localhost', port=0):
+        # TODO use pkg_resources to obtain HTML resources.
+        pkg_dir = os.path.dirname(os.path.abspath(__file__))
+        doc_root = os.path.join(pkg_dir, 'resources', 'html-build-viewer')
+        assert os.path.isdir(doc_root)
+
+        self.doc_root = doc_root
+        self.json_files = {}
+
+        self.server = BaseHTTPServer.HTTPServer((address, port), HTTPHandler)
+        self.server.wrapper = self
+        self.do_shutdown = False
+
+    @property
+    def url(self):
+        hostname, port = self.server.server_address
+        return 'http://%s:%d/' % (hostname, port)
+
+    def add_resource_json_file(self, key, path):
+        """Register a resource JSON file with the server.
+
+        The file will be made available under the name/key specified."""
+        with open(path, 'rb') as fh:
+            self.json_files[key] = fh.read()
+
+    def add_resource_json_url(self, key, url):
+        """Register a resource JSON file at a URL."""
+        r = requests.get(url)
+        if r.status_code != 200:
+            raise Exception('Non-200 HTTP response code')
+        self.json_files[key] = r.text
+
+    def run(self):
+        while not self.do_shutdown:
+            self.server.handle_request()

python/mozbuild/mozbuild/jar.py

@@ -0,0 +1,597 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+'''jarmaker.py provides a python class to package up chrome content by
+processing jar.mn files.
+
+See the documentation for jar.mn on MDC for further details on the format.
+'''
+
+from __future__ import absolute_import
+
+import sys
+import os
+import errno
+import re
+import logging
+from time import localtime
+from MozZipFile import ZipFile
+from cStringIO import StringIO
+from collections import defaultdict
+
+from mozbuild.preprocessor import Preprocessor
+from mozbuild.action.buildlist import addEntriesToListFile
+from mozpack.files import FileFinder
+import mozpack.path as mozpath
+if sys.platform == 'win32':
+    from ctypes import windll, WinError
+    CreateHardLink = windll.kernel32.CreateHardLinkA
+
+__all__ = ['JarMaker']
+
+
+class ZipEntry(object):
+    '''Helper class for jar output.
+
+      This class defines a simple file-like object for a zipfile.ZipEntry
+      so that we can consecutively write to it and then close it.
+      This methods hooks into ZipFile.writestr on close().
+      '''
+
+    def __init__(self, name, zipfile):
+        self._zipfile = zipfile
+        self._name = name
+        self._inner = StringIO()
+
+    def write(self, content):
+        '''Append the given content to this zip entry'''
+
+        self._inner.write(content)
+        return
+
+    def close(self):
+        '''The close method writes the content back to the zip file.'''
+
+        self._zipfile.writestr(self._name, self._inner.getvalue())
+
+
+def getModTime(aPath):
+    if not os.path.isfile(aPath):
+        return 0
+    mtime = os.stat(aPath).st_mtime
+    return localtime(mtime)
+
+
+class JarManifestEntry(object):
+    def __init__(self, output, source, is_locale=False, preprocess=False):
+        self.output = output
+        self.source = source
+        self.is_locale = is_locale
+        self.preprocess = preprocess
+
+
+class JarInfo(object):
+    def __init__(self, base_or_jarinfo, name=None):
+        if name is None:
+            assert isinstance(base_or_jarinfo, JarInfo)
+            self.base = base_or_jarinfo.base
+            self.name = base_or_jarinfo.name
+        else:
+            assert not isinstance(base_or_jarinfo, JarInfo)
+            self.base = base_or_jarinfo or ''
+            self.name = name
+            # For compatibility with existing jar.mn files, if there is no
+            # base, the jar name is under chrome/
+            if not self.base:
+                self.name = mozpath.join('chrome', self.name)
+        self.relativesrcdir = None
+        self.chrome_manifests = []
+        self.entries = []
+
+
+class DeprecatedJarManifest(Exception): pass
+
+
+class JarManifestParser(object):
+
+    ignore = re.compile('\s*(\#.*)?$')
+    jarline = re.compile('''
+        (?:
+            (?:\[(?P<base>[\w\d.\-\_\\\/{}@]+)\]\s*)? # optional [base/path]
+            (?P<jarfile>[\w\d.\-\_\\\/{}]+).jar\:    # filename.jar:
+        |
+            (?:\s*(\#.*)?)                           # comment
+        )\s*$                                        # whitespaces
+        ''', re.VERBOSE)
+    relsrcline = re.compile('relativesrcdir\s+(?P<relativesrcdir>.+?):')
+    regline = re.compile('\%\s+(.*)$')
+    entryre = '(?P<optPreprocess>\*)?(?P<optOverwrite>\+?)\s+'
+    entryline = re.compile(entryre
+                           + '(?P<output>[\w\d.\-\_\\\/\+\@]+)\s*(\((?P<locale>\%?)(?P<source>[\w\d.\-\_\\\/\@\*]+)\))?\s*$'
+                           )
+
+    def __init__(self):
+        self._current_jar = None
+        self._jars = []
+
+    def write(self, line):
+        # A Preprocessor instance feeds the parser through calls to this method.
+
+        # Ignore comments and empty lines
+        if self.ignore.match(line):
+            return
+
+        # A jar manifest file can declare several different sections, each of
+        # which applies to a given "jar file". Each of those sections starts
+        # with "<name>.jar:", in which case the path is assumed relative to
+        # a "chrome" directory, or "[<base/path>] <subpath/name>.jar:", where
+        # a base directory is given (usually pointing at the root of the
+        # application or addon) and the jar path is given relative to the base
+        # directory.
+        if self._current_jar is None:
+            m = self.jarline.match(line)
+            if not m:
+                raise RuntimeError(line)
+            if m.group('jarfile'):
+                self._current_jar = JarInfo(m.group('base'),
+                                            m.group('jarfile'))
+                self._jars.append(self._current_jar)
+            return
+
+        # Within each section, there can be three different types of entries:
+
+        # - indications of the relative source directory we pretend to be in
+        # when considering localization files, in the following form;
+        # "relativesrcdir <path>:"
+        m = self.relsrcline.match(line)
+        if m:
+            if self._current_jar.chrome_manifests or self._current_jar.entries:
+                self._current_jar = JarInfo(self._current_jar)
+                self._jars.append(self._current_jar)
+            self._current_jar.relativesrcdir = m.group('relativesrcdir')
+            return
+
+        # - chrome manifest entries, prefixed with "%".
+        m = self.regline.match(line)
+        if m:
+            rline = ' '.join(m.group(1).split())
+            if rline not in self._current_jar.chrome_manifests:
+                self._current_jar.chrome_manifests.append(rline)
+            return
+
+        # - entries indicating files to be part of the given jar. They are
+        # formed thusly:
+        #   "<dest_path>"
+        # or
+        #   "<dest_path> (<source_path>)"
+        # The <dest_path> is where the file(s) will be put in the chrome jar.
+        # The <source_path> is where the file(s) can be found in the source
+        # directory. The <source_path> may start with a "%" for files part
+        # of a localization directory, in which case the "%" counts as the
+        # locale.
+        # Each entry can be prefixed with "*" for preprocessing.
+        m = self.entryline.match(line)
+        if m:
+            if m.group('optOverwrite'):
+                raise DeprecatedJarManifest(
+                    'The "+" prefix is not supported anymore')
+            self._current_jar.entries.append(JarManifestEntry(
+                m.group('output'),
+                m.group('source') or mozpath.basename(m.group('output')),
+                is_locale=bool(m.group('locale')),
+                preprocess=bool(m.group('optPreprocess')),
+            ))
+            return
+
+        self._current_jar = None
+        self.write(line)
+
+    def __iter__(self):
+        return iter(self._jars)
+
+
+class JarMaker(object):
+    '''JarMaker reads jar.mn files and process those into jar files or
+      flat directories, along with chrome.manifest files.
+      '''
+
+    def __init__(self, outputFormat='flat', useJarfileManifest=True,
+        useChromeManifest=False):
+
+        self.outputFormat = outputFormat
+        self.useJarfileManifest = useJarfileManifest
+        self.useChromeManifest = useChromeManifest
+        self.pp = Preprocessor()
+        self.topsourcedir = None
+        self.sourcedirs = []
+        self.localedirs = None
+        self.l10nbase = None
+        self.l10nmerge = None
+        self.relativesrcdir = None
+        self.rootManifestAppId = None
+        self._seen_output = set()
+
+    def getCommandLineParser(self):
+        '''Get a optparse.OptionParser for jarmaker.
+
+        This OptionParser has the options for jarmaker as well as
+        the options for the inner PreProcessor.
+        '''
+
+        # HACK, we need to unescape the string variables we get,
+        # the perl versions didn't grok strings right
+
+        p = self.pp.getCommandLineParser(unescapeDefines=True)
+        p.add_option('-f', type='choice', default='jar',
+            choices=('jar', 'flat', 'symlink'),
+            help='fileformat used for output',
+            metavar='[jar, flat, symlink]',
+            )
+        p.add_option('-v', action='store_true', dest='verbose',
+                     help='verbose output')
+        p.add_option('-q', action='store_false', dest='verbose',
+                     help='verbose output')
+        p.add_option('-e', action='store_true',
+                     help='create chrome.manifest instead of jarfile.manifest'
+                     )
+        p.add_option('-s', type='string', action='append', default=[],
+                     help='source directory')
+        p.add_option('-t', type='string', help='top source directory')
+        p.add_option('-c', '--l10n-src', type='string', action='append'
+                     , help='localization directory')
+        p.add_option('--l10n-base', type='string', action='store',
+                     help='base directory to be used for localization (requires relativesrcdir)'
+                     )
+        p.add_option('--locale-mergedir', type='string', action='store'
+                     ,
+                     help='base directory to be used for l10n-merge (requires l10n-base and relativesrcdir)'
+                     )
+        p.add_option('--relativesrcdir', type='string',
+                     help='relativesrcdir to be used for localization')
+        p.add_option('-d', type='string', help='base directory')
+        p.add_option('--root-manifest-entry-appid', type='string',
+                     help='add an app id specific root chrome manifest entry.'
+                     )
+        return p
+
+    def finalizeJar(self, jardir, jarbase, jarname, chromebasepath, register, doZip=True):
+        '''Helper method to write out the chrome registration entries to
+         jarfile.manifest or chrome.manifest, or both.
+
+        The actual file processing is done in updateManifest.
+        '''
+
+        # rewrite the manifest, if entries given
+        if not register:
+            return
+
+        chromeManifest = os.path.join(jardir, jarbase, 'chrome.manifest')
+
+        if self.useJarfileManifest:
+            self.updateManifest(os.path.join(jardir, jarbase,
+                                             jarname + '.manifest'),
+                                chromebasepath.format(''), register)
+            if jarname != 'chrome':
+                addEntriesToListFile(chromeManifest,
+                                     ['manifest {0}.manifest'.format(jarname)])
+        if self.useChromeManifest:
+            chromebase = os.path.dirname(jarname) + '/'
+            self.updateManifest(chromeManifest,
+                                chromebasepath.format(chromebase), register)
+
+        # If requested, add a root chrome manifest entry (assumed to be in the parent directory
+        # of chromeManifest) with the application specific id. In cases where we're building
+        # lang packs, the root manifest must know about application sub directories.
+
+        if self.rootManifestAppId:
+            rootChromeManifest = \
+                os.path.join(os.path.normpath(os.path.dirname(chromeManifest)),
+                             '..', 'chrome.manifest')
+            rootChromeManifest = os.path.normpath(rootChromeManifest)
+            chromeDir = \
+                os.path.basename(os.path.dirname(os.path.normpath(chromeManifest)))
+            logging.info("adding '%s' entry to root chrome manifest appid=%s"
+                          % (chromeDir, self.rootManifestAppId))
+            addEntriesToListFile(rootChromeManifest,
+                                 ['manifest %s/chrome.manifest application=%s'
+                                  % (chromeDir,
+                                 self.rootManifestAppId)])
+
+    def updateManifest(self, manifestPath, chromebasepath, register):
+        '''updateManifest replaces the % in the chrome registration entries
+        with the given chrome base path, and updates the given manifest file.
+        '''
+        myregister = dict.fromkeys(map(lambda s: s.replace('%',
+            chromebasepath), register))
+        addEntriesToListFile(manifestPath, myregister.iterkeys())
+
+    def makeJar(self, infile, jardir):
+        '''makeJar is the main entry point to JarMaker.
+
+        It takes the input file, the output directory, the source dirs and the
+        top source dir as argument, and optionally the l10n dirs.
+        '''
+
+        # making paths absolute, guess srcdir if file and add to sourcedirs
+        _normpath = lambda p: os.path.normpath(os.path.abspath(p))
+        self.topsourcedir = _normpath(self.topsourcedir)
+        self.sourcedirs = [_normpath(p) for p in self.sourcedirs]
+        if self.localedirs:
+            self.localedirs = [_normpath(p) for p in self.localedirs]
+        elif self.relativesrcdir:
+            self.localedirs = \
+                self.generateLocaleDirs(self.relativesrcdir)
+        if isinstance(infile, basestring):
+            logging.info('processing ' + infile)
+            self.sourcedirs.append(_normpath(os.path.dirname(infile)))
+        pp = self.pp.clone()
+        pp.out = JarManifestParser()
+        pp.do_include(infile)
+
+        for info in pp.out:
+            self.processJarSection(info, jardir)
+
+    def generateLocaleDirs(self, relativesrcdir):
+        if os.path.basename(relativesrcdir) == 'locales':
+            # strip locales
+            l10nrelsrcdir = os.path.dirname(relativesrcdir)
+        else:
+            l10nrelsrcdir = relativesrcdir
+        locdirs = []
+
+        # generate locales dirs, merge, l10nbase, en-US
+        if self.l10nmerge:
+            locdirs.append(os.path.join(self.l10nmerge, l10nrelsrcdir))
+        if self.l10nbase:
+            locdirs.append(os.path.join(self.l10nbase, l10nrelsrcdir))
+        if self.l10nmerge or not self.l10nbase:
+            # add en-US if we merge, or if it's not l10n
+            locdirs.append(os.path.join(self.topsourcedir,
+                           relativesrcdir, 'en-US'))
+        return locdirs
+
+    def processJarSection(self, jarinfo, jardir):
+        '''Internal method called by makeJar to actually process a section
+        of a jar.mn file.
+        '''
+
+        # chromebasepath is used for chrome registration manifests
+        # {0} is getting replaced with chrome/ for chrome.manifest, and with
+        # an empty string for jarfile.manifest
+
+        chromebasepath = '{0}' + os.path.basename(jarinfo.name)
+        if self.outputFormat == 'jar':
+            chromebasepath = 'jar:' + chromebasepath + '.jar!'
+        chromebasepath += '/'
+
+        jarfile = os.path.join(jardir, jarinfo.base, jarinfo.name)
+        jf = None
+        if self.outputFormat == 'jar':
+            # jar
+            jarfilepath = jarfile + '.jar'
+            try:
+                os.makedirs(os.path.dirname(jarfilepath))
+            except OSError, error:
+                if error.errno != errno.EEXIST:
+                    raise
+            jf = ZipFile(jarfilepath, 'a', lock=True)
+            outHelper = self.OutputHelper_jar(jf)
+        else:
+            outHelper = getattr(self, 'OutputHelper_'
+                                + self.outputFormat)(jarfile)
+
+        if jarinfo.relativesrcdir:
+            self.localedirs = self.generateLocaleDirs(jarinfo.relativesrcdir)
+
+        for e in jarinfo.entries:
+            self._processEntryLine(e, outHelper, jf)
+
+        self.finalizeJar(jardir, jarinfo.base, jarinfo.name, chromebasepath,
+                         jarinfo.chrome_manifests)
+        if jf is not None:
+            jf.close()
+
+    def _processEntryLine(self, e, outHelper, jf):
+        out = e.output
+        src = e.source
+
+        # pick the right sourcedir -- l10n, topsrc or src
+
+        if e.is_locale:
+            src_base = self.localedirs
+        elif src.startswith('/'):
+            # path/in/jar/file_name.xul     (/path/in/sourcetree/file_name.xul)
+            # refers to a path relative to topsourcedir, use that as base
+            # and strip the leading '/'
+            src_base = [self.topsourcedir]
+            src = src[1:]
+        else:
+            # use srcdirs and the objdir (current working dir) for relative paths
+            src_base = self.sourcedirs + [os.getcwd()]
+
+        if '*' in src:
+            def _prefix(s):
+                for p in s.split('/'):
+                    if '*' not in p:
+                        yield p + '/'
+            prefix = ''.join(_prefix(src))
+            emitted = set()
+            for _srcdir in src_base:
+                finder = FileFinder(_srcdir, find_executables=False)
+                for path, _ in finder.find(src):
+                    # If the path was already seen in one of the other source
+                    # directories, skip it. That matches the non-wildcard case
+                    # below, where we pick the first existing file.
+                    reduced_path = path[len(prefix):]
+                    if reduced_path in emitted:
+                        continue
+                    emitted.add(reduced_path)
+                    e = JarManifestEntry(
+                        mozpath.join(out, reduced_path),
+                        path,
+                        is_locale=e.is_locale,
+                        preprocess=e.preprocess,
+                    )
+                    self._processEntryLine(e, outHelper, jf)
+            return
+
+        # check if the source file exists
+        realsrc = None
+        for _srcdir in src_base:
+            if os.path.isfile(os.path.join(_srcdir, src)):
+                realsrc = os.path.join(_srcdir, src)
+                break
+        if realsrc is None:
+            if jf is not None:
+                jf.close()
+            raise RuntimeError('File "{0}" not found in {1}'.format(src,
+                               ', '.join(src_base)))
+
+        if out in self._seen_output:
+            raise RuntimeError('%s already added' % out)
+        self._seen_output.add(out)
+
+        if e.preprocess:
+            outf = outHelper.getOutput(out)
+            inf = open(realsrc)
+            pp = self.pp.clone()
+            if src[-4:] == '.css':
+                pp.setMarker('%')
+            pp.out = outf
+            pp.do_include(inf)
+            pp.failUnused(realsrc)
+            outf.close()
+            inf.close()
+            return
+
+        # copy or symlink if newer
+
+        if getModTime(realsrc) > outHelper.getDestModTime(e.output):
+            if self.outputFormat == 'symlink':
+                outHelper.symlink(realsrc, out)
+                return
+            outf = outHelper.getOutput(out)
+
+            # open in binary mode, this can be images etc
+
+            inf = open(realsrc, 'rb')
+            outf.write(inf.read())
+            outf.close()
+            inf.close()
+
+    class OutputHelper_jar(object):
+        '''Provide getDestModTime and getOutput for a given jarfile.'''
+
+        def __init__(self, jarfile):
+            self.jarfile = jarfile
+
+        def getDestModTime(self, aPath):
+            try:
+                info = self.jarfile.getinfo(aPath)
+                return info.date_time
+            except:
+                return 0
+
+        def getOutput(self, name):
+            return ZipEntry(name, self.jarfile)
+
+    class OutputHelper_flat(object):
+        '''Provide getDestModTime and getOutput for a given flat
+        output directory. The helper method ensureDirFor is used by
+        the symlink subclass.
+        '''
+
+        def __init__(self, basepath):
+            self.basepath = basepath
+
+        def getDestModTime(self, aPath):
+            return getModTime(os.path.join(self.basepath, aPath))
+
+        def getOutput(self, name):
+            out = self.ensureDirFor(name)
+
+            # remove previous link or file
+            try:
+                os.remove(out)
+            except OSError, e:
+                if e.errno != errno.ENOENT:
+                    raise
+            return open(out, 'wb')
+
+        def ensureDirFor(self, name):
+            out = os.path.join(self.basepath, name)
+            outdir = os.path.dirname(out)
+            if not os.path.isdir(outdir):
+                try:
+                    os.makedirs(outdir)
+                except OSError, error:
+                    if error.errno != errno.EEXIST:
+                        raise
+            return out
+
+    class OutputHelper_symlink(OutputHelper_flat):
+        '''Subclass of OutputHelper_flat that provides a helper for
+        creating a symlink including creating the parent directories.
+        '''
+
+        def symlink(self, src, dest):
+            out = self.ensureDirFor(dest)
+
+            # remove previous link or file
+            try:
+                os.remove(out)
+            except OSError, e:
+                if e.errno != errno.ENOENT:
+                    raise
+            if sys.platform != 'win32':
+                os.symlink(src, out)
+            else:
+                # On Win32, use ctypes to create a hardlink
+                rv = CreateHardLink(out, src, None)
+                if rv == 0:
+                    raise WinError()
+
+
+def main(args=None):
+    args = args or sys.argv
+    jm = JarMaker()
+    p = jm.getCommandLineParser()
+    (options, args) = p.parse_args(args)
+    jm.outputFormat = options.f
+    jm.sourcedirs = options.s
+    jm.topsourcedir = options.t
+    if options.e:
+        jm.useChromeManifest = True
+        jm.useJarfileManifest = False
+    if options.l10n_base:
+        if not options.relativesrcdir:
+            p.error('relativesrcdir required when using l10n-base')
+        if options.l10n_src:
+            p.error('both l10n-src and l10n-base are not supported')
+        jm.l10nbase = options.l10n_base
+        jm.relativesrcdir = options.relativesrcdir
+        jm.l10nmerge = options.locale_mergedir
+        if jm.l10nmerge and not os.path.isdir(jm.l10nmerge):
+            logging.warning("WARNING: --locale-mergedir passed, but '%s' does not exist. "
+                "Ignore this message if the locale is complete." % jm.l10nmerge)
+    elif options.locale_mergedir:
+        p.error('l10n-base required when using locale-mergedir')
+    jm.localedirs = options.l10n_src
+    if options.root_manifest_entry_appid:
+        jm.rootManifestAppId = options.root_manifest_entry_appid
+    noise = logging.INFO
+    if options.verbose is not None:
+        noise = options.verbose and logging.DEBUG or logging.WARN
+    if sys.version_info[:2] > (2, 3):
+        logging.basicConfig(format='%(message)s')
+    else:
+        logging.basicConfig()
+    logging.getLogger().setLevel(noise)
+    topsrc = options.t
+    topsrc = os.path.normpath(os.path.abspath(topsrc))
+    if not args:
+        infile = sys.stdin
+    else:
+        (infile, ) = args
+    jm.makeJar(infile, options.d)

python/mozbuild/mozbuild/locale/en-US/LC_MESSAGES/mozbuild.po

@@ -0,0 +1,8 @@
+msgid "build.threads.short"
+msgstr "Thread Count"
+
+msgid "build.threads.full"
+msgstr "The number of threads to use when performing CPU intensive tasks. "
+       "This constrols the level of parallelization. The default value is "
+       "the number of cores in your machine."
+

python/mozbuild/mozbuild/makeutil.py

@@ -0,0 +1,186 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import os
+import re
+from types import StringTypes
+from collections import Iterable
+
+
+class Makefile(object):
+    '''Provides an interface for writing simple makefiles
+
+    Instances of this class are created, populated with rules, then
+    written.
+    '''
+
+    def __init__(self):
+        self._statements = []
+
+    def create_rule(self, targets=[]):
+        '''
+        Create a new rule in the makefile for the given targets.
+        Returns the corresponding Rule instance.
+        '''
+        rule = Rule(targets)
+        self._statements.append(rule)
+        return rule
+
+    def add_statement(self, statement):
+        '''
+        Add a raw statement in the makefile. Meant to be used for
+        simple variable assignments.
+        '''
+        self._statements.append(statement)
+
+    def dump(self, fh, removal_guard=True):
+        '''
+        Dump all the rules to the given file handle. Optionally (and by
+        default), add guard rules for file removals (empty rules for other
+        rules' dependencies)
+        '''
+        all_deps = set()
+        all_targets = set()
+        for statement in self._statements:
+            if isinstance(statement, Rule):
+                statement.dump(fh)
+                all_deps.update(statement.dependencies())
+                all_targets.update(statement.targets())
+            else:
+                fh.write('%s\n' % statement)
+        if removal_guard:
+            guard = Rule(sorted(all_deps - all_targets))
+            guard.dump(fh)
+
+
+class _SimpleOrderedSet(object):
+    '''
+    Simple ordered set, specialized for used in Rule below only.
+    It doesn't expose a complete API, and normalizes path separators
+    at insertion.
+    '''
+    def __init__(self):
+        self._list = []
+        self._set = set()
+
+    def __nonzero__(self):
+        return bool(self._set)
+
+    def __iter__(self):
+        return iter(self._list)
+
+    def __contains__(self, key):
+        return key in self._set
+
+    def update(self, iterable):
+        def _add(iterable):
+            emitted = set()
+            for i in iterable:
+                i = i.replace(os.sep, '/')
+                if i not in self._set and i not in emitted:
+                    yield i
+                    emitted.add(i)
+        added = list(_add(iterable))
+        self._set.update(added)
+        self._list.extend(added)
+
+
+class Rule(object):
+    '''Class handling simple rules in the form:
+           target1 target2 ... : dep1 dep2 ...
+                   command1
+                   command2
+                   ...
+    '''
+    def __init__(self, targets=[]):
+        self._targets = _SimpleOrderedSet()
+        self._dependencies = _SimpleOrderedSet()
+        self._commands = []
+        self.add_targets(targets)
+
+    def add_targets(self, targets):
+        '''Add additional targets to the rule.'''
+        assert isinstance(targets, Iterable) and not isinstance(targets, StringTypes)
+        self._targets.update(targets)
+        return self
+
+    def add_dependencies(self, deps):
+        '''Add dependencies to the rule.'''
+        assert isinstance(deps, Iterable) and not isinstance(deps, StringTypes)
+        self._dependencies.update(deps)
+        return self
+
+    def add_commands(self, commands):
+        '''Add commands to the rule.'''
+        assert isinstance(commands, Iterable) and not isinstance(commands, StringTypes)
+        self._commands.extend(commands)
+        return self
+
+    def targets(self):
+        '''Return an iterator on the rule targets.'''
+        # Ensure the returned iterator is actually just that, an iterator.
+        # Avoids caller fiddling with the set itself.
+        return iter(self._targets)
+
+    def dependencies(self):
+        '''Return an iterator on the rule dependencies.'''
+        return iter(d for d in self._dependencies if not d in self._targets)
+
+    def commands(self):
+        '''Return an iterator on the rule commands.'''
+        return iter(self._commands)
+
+    def dump(self, fh):
+        '''
+        Dump the rule to the given file handle.
+        '''
+        if not self._targets:
+            return
+        fh.write('%s:' % ' '.join(self._targets))
+        if self._dependencies:
+            fh.write(' %s' % ' '.join(self.dependencies()))
+        fh.write('\n')
+        for cmd in self._commands:
+            fh.write('\t%s\n' % cmd)
+
+
+# colon followed by anything except a slash (Windows path detection)
+_depfilesplitter = re.compile(r':(?![\\/])')
+
+
+def read_dep_makefile(fh):
+    """
+    Read the file handler containing a dep makefile (simple makefile only
+    containing dependencies) and returns an iterator of the corresponding Rules
+    it contains. Ignores removal guard rules.
+    """
+
+    rule = ''
+    for line in fh.readlines():
+        assert not line.startswith('\t')
+        line = line.strip()
+        if line.endswith('\\'):
+            rule += line[:-1]
+        else:
+            rule += line
+            split_rule = _depfilesplitter.split(rule, 1)
+            if len(split_rule) > 1 and split_rule[1].strip():
+                yield Rule(split_rule[0].strip().split()) \
+                      .add_dependencies(split_rule[1].strip().split())
+            rule = ''
+
+    if rule:
+        raise Exception('Makefile finishes with a backslash. Expected more input.')
+
+def write_dep_makefile(fh, target, deps):
+    '''
+    Write a Makefile containing only target's dependencies to the file handle
+    specified.
+    '''
+    mk = Makefile()
+    rule = mk.create_rule(targets=[target])
+    rule.add_dependencies(deps)
+    mk.dump(fh, removal_guard=True)

python/mozbuild/mozbuild/milestone.py

@@ -0,0 +1,75 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import argparse
+import os
+import re
+import sys
+
+
+def get_milestone_ab_with_num(milestone):
+    """
+    Returns the alpha and beta tag with its number (a1, a2, b3, ...).
+    """
+
+    match = re.search(r"([ab]\d+)", milestone)
+    if match:
+        return match.group(1)
+
+    return ""
+
+
+def get_official_milestone(path):
+    """
+    Returns the contents of the first line in `path` that starts with a digit.
+    """
+
+    with open(path) as fp:
+        for line in fp:
+            line = line.strip()
+            if line[:1].isdigit():
+                return line
+
+    raise Exception("Didn't find a line that starts with a digit.")
+
+
+def get_milestone_major(milestone):
+    """
+    Returns the major (first) part of the milestone.
+    """
+
+    return milestone.split('.')[0]
+
+
+def main(args):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--uaversion', default=False, action='store_true')
+    parser.add_argument('--symbolversion', default=False, action='store_true')
+    parser.add_argument('--topsrcdir', metavar='TOPSRCDIR', required=True)
+    options = parser.parse_args(args)
+
+    milestone_file = os.path.join(options.topsrcdir, 'config', 'milestone.txt')
+
+    milestone = get_official_milestone(milestone_file)
+
+    if options.uaversion:
+        # Only expose the major milestone in the UA string, hide the patch
+        # level (bugs 572659 and 870868).
+        uaversion = "%s.0" % (get_milestone_major(milestone),)
+        print(uaversion)
+
+    elif options.symbolversion:
+        # Only expose major milestone and alpha version. Used for symbol
+        # versioning on Linux.
+        symbolversion = "%s%s" % (get_milestone_major(milestone),
+                                  get_milestone_ab_with_num(milestone))
+        print(symbolversion)
+    else:
+        print(milestone)
+
+
+if __name__ == '__main__':
+    main(sys.argv[1:])

python/mozbuild/mozbuild/mozconfig.py

@@ -0,0 +1,485 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import filecmp
+import os
+import re
+import sys
+import subprocess
+import traceback
+
+from collections import defaultdict
+from mozpack import path as mozpath
+
+
+MOZ_MYCONFIG_ERROR = '''
+The MOZ_MYCONFIG environment variable to define the location of mozconfigs
+is deprecated. If you wish to define the mozconfig path via an environment
+variable, use MOZCONFIG instead.
+'''.strip()
+
+MOZCONFIG_LEGACY_PATH = '''
+You currently have a mozconfig at %s. This implicit location is no longer
+supported. Please move it to %s/.mozconfig or set an explicit path
+via the $MOZCONFIG environment variable.
+'''.strip()
+
+MOZCONFIG_BAD_EXIT_CODE = '''
+Evaluation of your mozconfig exited with an error. This could be triggered
+by a command inside your mozconfig failing. Please change your mozconfig
+to not error and/or to catch errors in executed commands.
+'''.strip()
+
+MOZCONFIG_BAD_OUTPUT = '''
+Evaluation of your mozconfig produced unexpected output.  This could be
+triggered by a command inside your mozconfig failing or producing some warnings
+or error messages. Please change your mozconfig to not error and/or to catch
+errors in executed commands.
+'''.strip()
+
+
+class MozconfigFindException(Exception):
+    """Raised when a mozconfig location is not defined properly."""
+
+
+class MozconfigLoadException(Exception):
+    """Raised when a mozconfig could not be loaded properly.
+
+    This typically indicates a malformed or misbehaving mozconfig file.
+    """
+
+    def __init__(self, path, message, output=None):
+        self.path = path
+        self.output = output
+        Exception.__init__(self, message)
+
+
+class MozconfigLoader(object):
+    """Handles loading and parsing of mozconfig files."""
+
+    RE_MAKE_VARIABLE = re.compile('''
+        ^\s*                    # Leading whitespace
+        (?P<var>[a-zA-Z_0-9]+)  # Variable name
+        \s* [?:]?= \s*          # Assignment operator surrounded by optional
+                                # spaces
+        (?P<value>.*$)''',      # Everything else (likely the value)
+        re.VERBOSE)
+
+    # Default mozconfig files in the topsrcdir.
+    DEFAULT_TOPSRCDIR_PATHS = ('.mozconfig', 'mozconfig')
+
+    DEPRECATED_TOPSRCDIR_PATHS = ('mozconfig.sh', 'myconfig.sh')
+    DEPRECATED_HOME_PATHS = ('.mozconfig', '.mozconfig.sh', '.mozmyconfig.sh')
+
+    IGNORE_SHELL_VARIABLES = {'_'}
+
+    ENVIRONMENT_VARIABLES = {
+        'CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'MOZ_OBJDIR',
+    }
+
+    AUTODETECT = object()
+
+    def __init__(self, topsrcdir):
+        self.topsrcdir = topsrcdir
+
+    @property
+    def _loader_script(self):
+        our_dir = os.path.abspath(os.path.dirname(__file__))
+
+        return os.path.join(our_dir, 'mozconfig_loader')
+
+    def find_mozconfig(self, env=os.environ):
+        """Find the active mozconfig file for the current environment.
+
+        This emulates the logic in mozconfig-find.
+
+        1) If ENV[MOZCONFIG] is set, use that
+        2) If $TOPSRCDIR/mozconfig or $TOPSRCDIR/.mozconfig exists, use it.
+        3) If both exist or if there are legacy locations detected, error out.
+
+        The absolute path to the found mozconfig will be returned on success.
+        None will be returned if no mozconfig could be found. A
+        MozconfigFindException will be raised if there is a bad state,
+        including conditions from #3 above.
+        """
+        # Check for legacy methods first.
+
+        if 'MOZ_MYCONFIG' in env:
+            raise MozconfigFindException(MOZ_MYCONFIG_ERROR)
+
+        env_path = env.get('MOZCONFIG', None) or None
+        if env_path is not None:
+            if not os.path.isabs(env_path):
+                potential_roots = [self.topsrcdir, os.getcwd()]
+                # Attempt to eliminate duplicates for e.g.
+                # self.topsrcdir == os.curdir.
+                potential_roots = set(os.path.abspath(p) for p in potential_roots)
+                existing = [root for root in potential_roots
+                            if os.path.exists(os.path.join(root, env_path))]
+                if len(existing) > 1:
+                    # There are multiple files, but we might have a setup like:
+                    #
+                    # somedirectory/
+                    #   srcdir/
+                    #   objdir/
+                    #
+                    # MOZCONFIG=../srcdir/some/path/to/mozconfig
+                    #
+                    # and be configuring from the objdir.  So even though we
+                    # have multiple existing files, they are actually the same
+                    # file.
+                    mozconfigs = [os.path.join(root, env_path)
+                                  for root in existing]
+                    if not all(map(lambda p1, p2: filecmp.cmp(p1, p2, shallow=False),
+                                   mozconfigs[:-1], mozconfigs[1:])):
+                        raise MozconfigFindException(
+                            'MOZCONFIG environment variable refers to a path that ' +
+                            'exists in more than one of ' + ', '.join(potential_roots) +
+                            '. Remove all but one.')
+                elif not existing:
+                    raise MozconfigFindException(
+                        'MOZCONFIG environment variable refers to a path that ' +
+                        'does not exist in any of ' + ', '.join(potential_roots))
+
+                env_path = os.path.join(existing[0], env_path)
+            elif not os.path.exists(env_path): # non-relative path
+                raise MozconfigFindException(
+                    'MOZCONFIG environment variable refers to a path that '
+                    'does not exist: ' + env_path)
+
+            if not os.path.isfile(env_path):
+                raise MozconfigFindException(
+                    'MOZCONFIG environment variable refers to a '
+                    'non-file: ' + env_path)
+
+        srcdir_paths = [os.path.join(self.topsrcdir, p) for p in
+            self.DEFAULT_TOPSRCDIR_PATHS]
+        existing = [p for p in srcdir_paths if os.path.isfile(p)]
+
+        if env_path is None and len(existing) > 1:
+            raise MozconfigFindException('Multiple default mozconfig files '
+                'present. Remove all but one. ' + ', '.join(existing))
+
+        path = None
+
+        if env_path is not None:
+            path = env_path
+        elif len(existing):
+            assert len(existing) == 1
+            path = existing[0]
+
+        if path is not None:
+            return os.path.abspath(path)
+
+        deprecated_paths = [os.path.join(self.topsrcdir, s) for s in
+            self.DEPRECATED_TOPSRCDIR_PATHS]
+
+        home = env.get('HOME', None)
+        if home is not None:
+            deprecated_paths.extend([os.path.join(home, s) for s in
+            self.DEPRECATED_HOME_PATHS])
+
+        for path in deprecated_paths:
+            if os.path.exists(path):
+                raise MozconfigFindException(
+                    MOZCONFIG_LEGACY_PATH % (path, self.topsrcdir))
+
+        return None
+
+    def read_mozconfig(self, path=None, moz_build_app=None):
+        """Read the contents of a mozconfig into a data structure.
+
+        This takes the path to a mozconfig to load. If the given path is
+        AUTODETECT, will try to find a mozconfig from the environment using
+        find_mozconfig().
+
+        mozconfig files are shell scripts. So, we can't just parse them.
+        Instead, we run the shell script in a wrapper which allows us to record
+        state from execution. Thus, the output from a mozconfig is a friendly
+        static data structure.
+        """
+        if path is self.AUTODETECT:
+            path = self.find_mozconfig()
+
+        result = {
+            'path': path,
+            'topobjdir': None,
+            'configure_args': None,
+            'make_flags': None,
+            'make_extra': None,
+            'env': None,
+            'vars': None,
+        }
+
+        if path is None:
+            return result
+
+        path = mozpath.normsep(path)
+
+        result['configure_args'] = []
+        result['make_extra'] = []
+        result['make_flags'] = []
+
+        env = dict(os.environ)
+
+        # Since mozconfig_loader is a shell script, running it "normally"
+        # actually leads to two shell executions on Windows. Avoid this by
+        # directly calling sh mozconfig_loader.
+        shell = 'sh'
+        if 'MOZILLABUILD' in os.environ:
+            shell = os.environ['MOZILLABUILD'] + '/msys/bin/sh'
+        if sys.platform == 'win32':
+            shell = shell + '.exe'
+
+        command = [shell, mozpath.normsep(self._loader_script),
+                   mozpath.normsep(self.topsrcdir), path, sys.executable,
+                   mozpath.join(mozpath.dirname(self._loader_script),
+                                'action', 'dump_env.py')]
+
+        try:
+            # We need to capture stderr because that's where the shell sends
+            # errors if execution fails.
+            output = subprocess.check_output(command, stderr=subprocess.STDOUT,
+                cwd=self.topsrcdir, env=env)
+        except subprocess.CalledProcessError as e:
+            lines = e.output.splitlines()
+
+            # Output before actual execution shouldn't be relevant.
+            try:
+                index = lines.index('------END_BEFORE_SOURCE')
+                lines = lines[index + 1:]
+            except ValueError:
+                pass
+
+            raise MozconfigLoadException(path, MOZCONFIG_BAD_EXIT_CODE, lines)
+
+        try:
+            parsed = self._parse_loader_output(output)
+        except AssertionError:
+            # _parse_loader_output uses assertions to verify the
+            # well-formedness of the shell output; when these fail, it
+            # generally means there was a problem with the output, but we
+            # include the assertion traceback just to be sure.
+            print('Assertion failed in _parse_loader_output:')
+            traceback.print_exc()
+            raise MozconfigLoadException(path, MOZCONFIG_BAD_OUTPUT,
+                                         output.splitlines())
+
+        def diff_vars(vars_before, vars_after):
+            set1 = set(vars_before.keys()) - self.IGNORE_SHELL_VARIABLES
+            set2 = set(vars_after.keys()) - self.IGNORE_SHELL_VARIABLES
+            added = set2 - set1
+            removed = set1 - set2
+            maybe_modified = set1 & set2
+            changed = {
+                'added': {},
+                'removed': {},
+                'modified': {},
+                'unmodified': {},
+            }
+
+            for key in added:
+                changed['added'][key] = vars_after[key]
+
+            for key in removed:
+                changed['removed'][key] = vars_before[key]
+
+            for key in maybe_modified:
+                if vars_before[key] != vars_after[key]:
+                    changed['modified'][key] = (
+                        vars_before[key], vars_after[key])
+                elif key in self.ENVIRONMENT_VARIABLES:
+                    # In order for irrelevant environment variable changes not
+                    # to incur in re-running configure, only a set of
+                    # environment variables are stored when they are
+                    # unmodified. Otherwise, changes such as using a different
+                    # terminal window, or even rebooting, would trigger
+                    # reconfigures.
+                    changed['unmodified'][key] = vars_after[key]
+
+            return changed
+
+        result['env'] = diff_vars(parsed['env_before'], parsed['env_after'])
+
+        # Environment variables also appear as shell variables, but that's
+        # uninteresting duplication of information. Filter them out.
+        filt = lambda x, y: {k: v for k, v in x.items() if k not in y}
+        result['vars'] = diff_vars(
+            filt(parsed['vars_before'], parsed['env_before']),
+            filt(parsed['vars_after'], parsed['env_after'])
+        )
+
+        result['configure_args'] = [self._expand(o) for o in parsed['ac']]
+
+        if moz_build_app is not None:
+            result['configure_args'].extend(self._expand(o) for o in
+                parsed['ac_app'][moz_build_app])
+
+        if 'MOZ_OBJDIR' in parsed['env_before']:
+            result['topobjdir'] = parsed['env_before']['MOZ_OBJDIR']
+
+        mk = [self._expand(o) for o in parsed['mk']]
+
+        for o in mk:
+            match = self.RE_MAKE_VARIABLE.match(o)
+
+            if match is None:
+                result['make_extra'].append(o)
+                continue
+
+            name, value = match.group('var'), match.group('value')
+
+            if name == 'MOZ_MAKE_FLAGS':
+                result['make_flags'] = value.split()
+                continue
+
+            if name == 'MOZ_OBJDIR':
+                result['topobjdir'] = value
+                continue
+
+            result['make_extra'].append(o)
+
+        return result
+
+    def _parse_loader_output(self, output):
+        mk_options = []
+        ac_options = []
+        ac_app_options = defaultdict(list)
+        before_source = {}
+        after_source = {}
+        env_before_source = {}
+        env_after_source = {}
+
+        current = None
+        current_type = None
+        in_variable = None
+
+        for line in output.splitlines():
+
+            # XXX This is an ugly hack. Data may be lost from things
+            # like environment variable values.
+            # See https://bugzilla.mozilla.org/show_bug.cgi?id=831381
+            line = line.decode('mbcs' if sys.platform == 'win32' else 'utf-8',
+                               'ignore')
+
+            if not line:
+                continue
+
+            if line.startswith('------BEGIN_'):
+                assert current_type is None
+                assert current is None
+                assert not in_variable
+                current_type = line[len('------BEGIN_'):]
+                current = []
+                continue
+
+            if line.startswith('------END_'):
+                assert not in_variable
+                section = line[len('------END_'):]
+                assert current_type == section
+
+                if current_type == 'AC_OPTION':
+                    ac_options.append('\n'.join(current))
+                elif current_type == 'MK_OPTION':
+                    mk_options.append('\n'.join(current))
+                elif current_type == 'AC_APP_OPTION':
+                    app = current.pop(0)
+                    ac_app_options[app].append('\n'.join(current))
+
+                current = None
+                current_type = None
+                continue
+
+            assert current_type is not None
+
+            vars_mapping = {
+                'BEFORE_SOURCE': before_source,
+                'AFTER_SOURCE': after_source,
+                'ENV_BEFORE_SOURCE': env_before_source,
+                'ENV_AFTER_SOURCE': env_after_source,
+            }
+
+            if current_type in vars_mapping:
+                # mozconfigs are sourced using the Bourne shell (or at least
+                # in Bourne shell mode). This means |set| simply lists
+                # variables from the current shell (not functions). (Note that
+                # if Bash is installed in /bin/sh it acts like regular Bourne
+                # and doesn't print functions.) So, lines should have the
+                # form:
+                #
+                #  key='value'
+                #  key=value
+                #
+                # The only complication is multi-line variables. Those have the
+                # form:
+                #
+                #  key='first
+                #  second'
+
+                # TODO Bug 818377 Properly handle multi-line variables of form:
+                # $ foo="a='b'
+                # c='d'"
+                # $ set
+                # foo='a='"'"'b'"'"'
+                # c='"'"'d'"'"
+
+                name = in_variable
+                value = None
+                if in_variable:
+                    # Reached the end of a multi-line variable.
+                    if line.endswith("'") and not line.endswith("\\'"):
+                        current.append(line[:-1])
+                        value = '\n'.join(current)
+                        in_variable = None
+                    else:
+                        current.append(line)
+                        continue
+                else:
+                    equal_pos = line.find('=')
+
+                    if equal_pos < 1:
+                        # TODO log warning?
+                        continue
+
+                    name = line[0:equal_pos]
+                    value = line[equal_pos + 1:]
+
+                    if len(value):
+                        has_quote = value[0] == "'"
+
+                        if has_quote:
+                            value = value[1:]
+
+                        # Lines with a quote not ending in a quote are multi-line.
+                        if has_quote and not value.endswith("'"):
+                            in_variable = name
+                            current.append(value)
+                            continue
+                        else:
+                            value = value[:-1] if has_quote else value
+
+                assert name is not None
+
+                vars_mapping[current_type][name] = value
+
+                current = []
+
+                continue
+
+            current.append(line)
+
+        return {
+            'mk': mk_options,
+            'ac': ac_options,
+            'ac_app': ac_app_options,
+            'vars_before': before_source,
+            'vars_after': after_source,
+            'env_before': env_before_source,
+            'env_after': env_after_source,
+        }
+
+    def _expand(self, s):
+        return s.replace('@TOPSRCDIR@', self.topsrcdir)

python/mozbuild/mozbuild/mozconfig_loader

@@ -0,0 +1,80 @@
+#!/bin/sh
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This script provides an execution environment for mozconfig scripts.
+# This script is not meant to be called by users. Instead, some
+# higher-level driver invokes it and parses the machine-tailored output.
+
+set -e
+
+ac_add_options() {
+  local opt
+  for opt; do
+    case "$opt" in
+    --target=*)
+      echo "------BEGIN_MK_OPTION"
+      echo $opt | sed s/--target/CONFIG_GUESS/
+      echo "------END_MK_OPTION"
+      ;;
+    esac
+    echo "------BEGIN_AC_OPTION"
+    echo $opt
+    echo "------END_AC_OPTION"
+  done
+}
+
+ac_add_app_options() {
+  local app
+  app=$1
+  shift
+  echo "------BEGIN_AC_APP_OPTION"
+  echo $app
+  echo "$*"
+  echo "------END_AC_APP_OPTION"
+}
+
+mk_add_options() {
+  local opt name op value
+  for opt; do
+    echo "------BEGIN_MK_OPTION"
+    echo $opt
+    # Remove any leading "export"
+    opt=${opt#export}
+    case "$opt" in
+    *\?=*) op="?=" ;;
+    *:=*) op=":=" ;;
+    *+=*) op="+=" ;;
+    *=*) op="=" ;;
+    esac
+    # Remove the operator and the value that follows
+    name=${opt%%${op}*}
+    # Note: $(echo ${name}) strips the variable from any leading and trailing
+    # whitespaces.
+    eval "$(echo ${name})_IS_SET=1"
+    echo "------END_MK_OPTION"
+  done
+}
+
+echo "------BEGIN_ENV_BEFORE_SOURCE"
+$3 $4
+echo "------END_ENV_BEFORE_SOURCE"
+
+echo "------BEGIN_BEFORE_SOURCE"
+set
+echo "------END_BEFORE_SOURCE"
+
+topsrcdir=$1
+
+. $2
+
+unset topsrcdir
+
+echo "------BEGIN_AFTER_SOURCE"
+set
+echo "------END_AFTER_SOURCE"
+
+echo "------BEGIN_ENV_AFTER_SOURCE"
+$3 $4
+echo "------END_ENV_AFTER_SOURCE"

python/mozbuild/mozbuild/mozinfo.py

@@ -0,0 +1,160 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This module produces a JSON file that provides basic build info and
+# configuration metadata.
+
+from __future__ import absolute_import
+
+import os
+import re
+import json
+
+
+def build_dict(config, env=os.environ):
+    """
+    Build a dict containing data about the build configuration from
+    the environment.
+    """
+    substs = config.substs
+
+    # Check that all required variables are present first.
+    required = ["TARGET_CPU", "OS_TARGET"]
+    missing = [r for r in required if r not in substs]
+    if missing:
+        raise Exception("Missing required environment variables: %s" %
+                        ', '.join(missing))
+
+    d = {}
+    d['topsrcdir'] = config.topsrcdir
+
+    if config.mozconfig:
+        d['mozconfig'] = config.mozconfig
+
+    # os
+    o = substs["OS_TARGET"]
+    known_os = {"Linux": "linux",
+                "WINNT": "win",
+                "Darwin": "mac",
+                "Android": "b2g" if substs.get("MOZ_WIDGET_TOOLKIT") == "gonk" else "android"}
+    if o in known_os:
+        d["os"] = known_os[o]
+    else:
+        # Allow unknown values, just lowercase them.
+        d["os"] = o.lower()
+
+    # Widget toolkit, just pass the value directly through.
+    d["toolkit"] = substs.get("MOZ_WIDGET_TOOLKIT")
+
+    # Application name
+    if 'MOZ_APP_NAME' in substs:
+        d["appname"] = substs["MOZ_APP_NAME"]
+
+    # Build app name
+    if 'MOZ_MULET' in substs and substs.get('MOZ_MULET') == "1":
+        d["buildapp"] = "mulet"
+    elif 'MOZ_BUILD_APP' in substs:
+        d["buildapp"] = substs["MOZ_BUILD_APP"]
+
+    # processor
+    p = substs["TARGET_CPU"]
+    # for universal mac builds, put in a special value
+    if d["os"] == "mac" and "UNIVERSAL_BINARY" in substs and substs["UNIVERSAL_BINARY"] == "1":
+        p = "universal-x86-x86_64"
+    else:
+        # do some slight massaging for some values
+        #TODO: retain specific values in case someone wants them?
+        if p.startswith("arm"):
+            p = "arm"
+        elif re.match("i[3-9]86", p):
+            p = "x86"
+    d["processor"] = p
+    # hardcoded list of 64-bit CPUs
+    if p in ["x86_64", "ppc64"]:
+        d["bits"] = 64
+    # hardcoded list of known 32-bit CPUs
+    elif p in ["x86", "arm", "ppc"]:
+        d["bits"] = 32
+    # other CPUs will wind up with unknown bits
+
+    d['debug'] = substs.get('MOZ_DEBUG') == '1'
+    d['nightly_build'] = substs.get('NIGHTLY_BUILD') == '1'
+    d['release_or_beta'] = substs.get('RELEASE_OR_BETA') == '1'
+    d['pgo'] = substs.get('MOZ_PGO') == '1'
+    d['crashreporter'] = bool(substs.get('MOZ_CRASHREPORTER'))
+    d['datareporting'] = bool(substs.get('MOZ_DATA_REPORTING'))
+    d['healthreport'] = substs.get('MOZ_SERVICES_HEALTHREPORT') == '1'
+    d['sync'] = substs.get('MOZ_SERVICES_SYNC') == '1'
+    d['asan'] = substs.get('MOZ_ASAN') == '1'
+    d['tsan'] = substs.get('MOZ_TSAN') == '1'
+    d['telemetry'] = substs.get('MOZ_TELEMETRY_REPORTING') == '1'
+    d['tests_enabled'] = substs.get('ENABLE_TESTS') == "1"
+    d['bin_suffix'] = substs.get('BIN_SUFFIX', '')
+    d['addon_signing'] = substs.get('MOZ_ADDON_SIGNING') == '1'
+    d['require_signing'] = substs.get('MOZ_REQUIRE_SIGNING') == '1'
+    d['official'] = bool(substs.get('MOZILLA_OFFICIAL'))
+    d['sm_promise'] = bool(substs.get('SPIDERMONKEY_PROMISE'))
+
+    def guess_platform():
+        if d['buildapp'] in ('browser', 'mulet'):
+            p = d['os']
+            if p == 'mac':
+                p = 'macosx64'
+            elif d['bits'] == 64:
+                p = '{}64'.format(p)
+            elif p in ('win',):
+                p = '{}32'.format(p)
+
+            if d['buildapp'] == 'mulet':
+                p = '{}-mulet'.format(p)
+
+            if d['asan']:
+                p = '{}-asan'.format(p)
+
+            return p
+
+        if d['buildapp'] == 'b2g':
+            if d['toolkit'] == 'gonk':
+                return 'emulator'
+
+            if d['bits'] == 64:
+                return 'linux64_gecko'
+            return 'linux32_gecko'
+
+        if d['buildapp'] == 'mobile/android':
+            if d['processor'] == 'x86':
+                return 'android-x86'
+            return 'android-arm'
+
+    def guess_buildtype():
+        if d['debug']:
+            return 'debug'
+        if d['pgo']:
+            return 'pgo'
+        return 'opt'
+
+    # if buildapp or bits are unknown, we don't have a configuration similar to
+    # any in automation and the guesses are useless.
+    if 'buildapp' in d and (d['os'] == 'mac' or 'bits' in d):
+        d['platform_guess'] = guess_platform()
+        d['buildtype_guess'] = guess_buildtype()
+
+    if 'buildapp' in d and d['buildapp'] == 'mobile/android' and 'MOZ_ANDROID_MIN_SDK_VERSION' in substs:
+        d['android_min_sdk'] = substs['MOZ_ANDROID_MIN_SDK_VERSION']
+
+    return d
+
+
+def write_mozinfo(file, config, env=os.environ):
+    """Write JSON data about the configuration specified in config and an
+    environment variable dict to |file|, which may be a filename or file-like
+    object.
+    See build_dict for information about what  environment variables are used,
+    and what keys are produced.
+    """
+    build_conf = build_dict(config, env)
+    if isinstance(file, basestring):
+        file = open(file, 'wb')
+
+    json.dump(build_conf, file, sort_keys=True, indent=4)

python/mozbuild/mozbuild/preprocessor.py

@@ -0,0 +1,805 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+This is a very primitive line based preprocessor, for times when using
+a C preprocessor isn't an option.
+
+It currently supports the following grammar for expressions, whitespace is
+ignored:
+
+expression :
+  and_cond ( '||' expression ) ? ;
+and_cond:
+  test ( '&&' and_cond ) ? ;
+test:
+  unary ( ( '==' | '!=' ) unary ) ? ;
+unary :
+  '!'? value ;
+value :
+  [0-9]+ # integer
+  | 'defined(' \w+ ')'
+  | \w+  # string identifier or value;
+"""
+
+import sys
+import os
+import re
+from optparse import OptionParser
+import errno
+from makeutil import Makefile
+
+# hack around win32 mangling our line endings
+# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65443
+if sys.platform == "win32":
+    import msvcrt
+    msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
+    os.linesep = '\n'
+
+
+__all__ = [
+  'Context',
+  'Expression',
+  'Preprocessor',
+  'preprocess'
+]
+
+
+class Expression:
+    def __init__(self, expression_string):
+        """
+        Create a new expression with this string.
+        The expression will already be parsed into an Abstract Syntax Tree.
+        """
+        self.content = expression_string
+        self.offset = 0
+        self.__ignore_whitespace()
+        self.e = self.__get_logical_or()
+        if self.content:
+            raise Expression.ParseError, self
+
+    def __get_logical_or(self):
+        """
+        Production: and_cond ( '||' expression ) ?
+        """
+        if not len(self.content):
+            return None
+        rv = Expression.__AST("logical_op")
+        # test
+        rv.append(self.__get_logical_and())
+        self.__ignore_whitespace()
+        if self.content[:2] != '||':
+            # no logical op needed, short cut to our prime element
+            return rv[0]
+        # append operator
+        rv.append(Expression.__ASTLeaf('op', self.content[:2]))
+        self.__strip(2)
+        self.__ignore_whitespace()
+        rv.append(self.__get_logical_or())
+        self.__ignore_whitespace()
+        return rv
+
+    def __get_logical_and(self):
+        """
+        Production: test ( '&&' and_cond ) ?
+        """
+        if not len(self.content):
+            return None
+        rv = Expression.__AST("logical_op")
+        # test
+        rv.append(self.__get_equality())
+        self.__ignore_whitespace()
+        if self.content[:2] != '&&':
+            # no logical op needed, short cut to our prime element
+            return rv[0]
+        # append operator
+        rv.append(Expression.__ASTLeaf('op', self.content[:2]))
+        self.__strip(2)
+        self.__ignore_whitespace()
+        rv.append(self.__get_logical_and())
+        self.__ignore_whitespace()
+        return rv
+
+    def __get_equality(self):
+        """
+        Production: unary ( ( '==' | '!=' ) unary ) ?
+        """
+        if not len(self.content):
+            return None
+        rv = Expression.__AST("equality")
+        # unary
+        rv.append(self.__get_unary())
+        self.__ignore_whitespace()
+        if not re.match('[=!]=', self.content):
+            # no equality needed, short cut to our prime unary
+            return rv[0]
+        # append operator
+        rv.append(Expression.__ASTLeaf('op', self.content[:2]))
+        self.__strip(2)
+        self.__ignore_whitespace()
+        rv.append(self.__get_unary())
+        self.__ignore_whitespace()
+        return rv
+
+    def __get_unary(self):
+        """
+        Production: '!'? value
+        """
+        # eat whitespace right away, too
+        not_ws = re.match('!\s*', self.content)
+        if not not_ws:
+            return self.__get_value()
+        rv = Expression.__AST('not')
+        self.__strip(not_ws.end())
+        rv.append(self.__get_value())
+        self.__ignore_whitespace()
+        return rv
+
+    def __get_value(self):
+        """
+        Production: ( [0-9]+ | 'defined(' \w+ ')' | \w+ )
+        Note that the order is important, and the expression is kind-of
+        ambiguous as \w includes 0-9. One could make it unambiguous by
+        removing 0-9 from the first char of a string literal.
+        """
+        rv = None
+        m = re.match('defined\s*\(\s*(\w+)\s*\)', self.content)
+        if m:
+            word_len = m.end()
+            rv = Expression.__ASTLeaf('defined', m.group(1))
+        else:
+            word_len = re.match('[0-9]*', self.content).end()
+            if word_len:
+                value = int(self.content[:word_len])
+                rv = Expression.__ASTLeaf('int', value)
+            else:
+                word_len = re.match('\w*', self.content).end()
+                if word_len:
+                    rv = Expression.__ASTLeaf('string', self.content[:word_len])
+                else:
+                    raise Expression.ParseError, self
+        self.__strip(word_len)
+        self.__ignore_whitespace()
+        return rv
+
+    def __ignore_whitespace(self):
+        ws_len = re.match('\s*', self.content).end()
+        self.__strip(ws_len)
+        return
+
+    def __strip(self, length):
+        """
+        Remove a given amount of chars from the input and update
+        the offset.
+        """
+        self.content = self.content[length:]
+        self.offset += length
+
+    def evaluate(self, context):
+        """
+        Evaluate the expression with the given context
+        """
+
+        # Helper function to evaluate __get_equality results
+        def eval_equality(tok):
+            left = opmap[tok[0].type](tok[0])
+            right = opmap[tok[2].type](tok[2])
+            rv = left == right
+            if tok[1].value == '!=':
+                rv = not rv
+            return rv
+        # Helper function to evaluate __get_logical_and and __get_logical_or results
+        def eval_logical_op(tok):
+            left = opmap[tok[0].type](tok[0])
+            right = opmap[tok[2].type](tok[2])
+            if tok[1].value == '&&':
+                return left and right
+            elif tok[1].value == '||':
+                return left or right
+            raise Expression.ParseError, self
+
+        # Mapping from token types to evaluator functions
+        # Apart from (non-)equality, all these can be simple lambda forms.
+        opmap = {
+          'logical_op': eval_logical_op,
+          'equality': eval_equality,
+          'not': lambda tok: not opmap[tok[0].type](tok[0]),
+          'string': lambda tok: context[tok.value],
+          'defined': lambda tok: tok.value in context,
+          'int': lambda tok: tok.value}
+
+        return opmap[self.e.type](self.e);
+
+    class __AST(list):
+        """
+        Internal class implementing Abstract Syntax Tree nodes
+        """
+        def __init__(self, type):
+            self.type = type
+            super(self.__class__, self).__init__(self)
+
+    class __ASTLeaf:
+        """
+        Internal class implementing Abstract Syntax Tree leafs
+        """
+        def __init__(self, type, value):
+            self.value = value
+            self.type = type
+        def __str__(self):
+            return self.value.__str__()
+        def __repr__(self):
+            return self.value.__repr__()
+
+    class ParseError(StandardError):
+        """
+        Error raised when parsing fails.
+        It has two members, offset and content, which give the offset of the
+        error and the offending content.
+        """
+        def __init__(self, expression):
+            self.offset = expression.offset
+            self.content = expression.content[:3]
+        def __str__(self):
+            return 'Unexpected content at offset {0}, "{1}"'.format(self.offset,
+                                                                    self.content)
+
+class Context(dict):
+    """
+    This class holds variable values by subclassing dict, and while it
+    truthfully reports True and False on
+
+    name in context
+
+    it returns the variable name itself on
+
+    context["name"]
+
+    to reflect the ambiguity between string literals and preprocessor
+    variables.
+    """
+    def __getitem__(self, key):
+        if key in self:
+            return super(self.__class__, self).__getitem__(key)
+        return key
+
+
+class Preprocessor:
+    """
+    Class for preprocessing text files.
+    """
+    class Error(RuntimeError):
+        def __init__(self, cpp, MSG, context):
+            self.file = cpp.context['FILE']
+            self.line = cpp.context['LINE']
+            self.key = MSG
+            RuntimeError.__init__(self, (self.file, self.line, self.key, context))
+
+    def __init__(self, defines=None, marker='#'):
+        self.context = Context()
+        for k,v in {'FILE': '',
+                    'LINE': 0,
+                    'DIRECTORY': os.path.abspath('.')}.iteritems():
+            self.context[k] = v
+        self.actionLevel = 0
+        self.disableLevel = 0
+        # ifStates can be
+        #  0: hadTrue
+        #  1: wantsTrue
+        #  2: #else found
+        self.ifStates = []
+        self.checkLineNumbers = False
+        self.filters = []
+        self.cmds = {}
+        for cmd, level in {'define': 0,
+                           'undef': 0,
+                           'if': sys.maxint,
+                           'ifdef': sys.maxint,
+                           'ifndef': sys.maxint,
+                           'else': 1,
+                           'elif': 1,
+                           'elifdef': 1,
+                           'elifndef': 1,
+                           'endif': sys.maxint,
+                           'expand': 0,
+                           'literal': 0,
+                           'filter': 0,
+                           'unfilter': 0,
+                           'include': 0,
+                           'includesubst': 0,
+                           'error': 0}.iteritems():
+            self.cmds[cmd] = (level, getattr(self, 'do_' + cmd))
+        self.out = sys.stdout
+        self.setMarker(marker)
+        self.varsubst = re.compile('@(?P<VAR>\w+)@', re.U)
+        self.includes = set()
+        self.silenceMissingDirectiveWarnings = False
+        if defines:
+            self.context.update(defines)
+
+    def failUnused(self, file):
+        msg = None
+        if self.actionLevel == 0 and not self.silenceMissingDirectiveWarnings:
+            msg = 'no preprocessor directives found'
+        elif self.actionLevel == 1:
+            msg = 'no useful preprocessor directives found'
+        if msg:
+            class Fake(object): pass
+            fake = Fake()
+            fake.context = {
+                'FILE': file,
+                'LINE': None,
+            }
+            raise Preprocessor.Error(fake, msg, None)
+
+    def setMarker(self, aMarker):
+        """
+        Set the marker to be used for processing directives.
+        Used for handling CSS files, with pp.setMarker('%'), for example.
+        The given marker may be None, in which case no markers are processed.
+        """
+        self.marker = aMarker
+        if aMarker:
+            self.instruction = re.compile('{0}(?P<cmd>[a-z]+)(?:\s(?P<args>.*))?$'
+                                          .format(aMarker),
+                                          re.U)
+            self.comment = re.compile(aMarker, re.U)
+        else:
+            class NoMatch(object):
+                def match(self, *args):
+                    return False
+            self.instruction = self.comment = NoMatch()
+
+    def setSilenceDirectiveWarnings(self, value):
+        """
+        Sets whether missing directive warnings are silenced, according to
+        ``value``.  The default behavior of the preprocessor is to emit
+        such warnings.
+        """
+        self.silenceMissingDirectiveWarnings = value
+
+    def addDefines(self, defines):
+        """
+        Adds the specified defines to the preprocessor.
+        ``defines`` may be a dictionary object or an iterable of key/value pairs
+        (as tuples or other iterables of length two)
+        """
+        self.context.update(defines)
+
+    def clone(self):
+        """
+        Create a clone of the current processor, including line ending
+        settings, marker, variable definitions, output stream.
+        """
+        rv = Preprocessor()
+        rv.context.update(self.context)
+        rv.setMarker(self.marker)
+        rv.out = self.out
+        return rv
+
+    def processFile(self, input, output, depfile=None):
+        """
+        Preprocesses the contents of the ``input`` stream and writes the result
+        to the ``output`` stream. If ``depfile`` is set,  the dependencies of
+        ``output`` file are written to ``depfile`` in Makefile format.
+        """
+        self.out = output
+
+        self.do_include(input, False)
+        self.failUnused(input.name)
+
+        if depfile:
+            mk = Makefile()
+            mk.create_rule([output.name]).add_dependencies(self.includes)
+            mk.dump(depfile)
+
+    def computeDependencies(self, input):
+        """
+        Reads the ``input`` stream, and computes the dependencies for that input.
+        """
+        try:
+            old_out = self.out
+            self.out = None
+            self.do_include(input, False)
+
+            return self.includes
+        finally:
+            self.out = old_out
+
+    def applyFilters(self, aLine):
+        for f in self.filters:
+            aLine = f[1](aLine)
+        return aLine
+
+    def noteLineInfo(self):
+        # Record the current line and file. Called once before transitioning
+        # into or out of an included file and after writing each line.
+        self.line_info = self.context['FILE'], self.context['LINE']
+
+    def write(self, aLine):
+        """
+        Internal method for handling output.
+        """
+        if not self.out:
+            return
+
+        next_line, next_file = self.context['LINE'], self.context['FILE']
+        if self.checkLineNumbers:
+            expected_file, expected_line = self.line_info
+            expected_line += 1
+            if (expected_line != next_line or
+                expected_file and expected_file != next_file):
+                self.out.write('//@line {line} "{file}"\n'.format(line=next_line,
+                                                                  file=next_file))
+        self.noteLineInfo()
+
+        filteredLine = self.applyFilters(aLine)
+        if filteredLine != aLine:
+            self.actionLevel = 2
+        self.out.write(filteredLine)
+
+    def handleCommandLine(self, args, defaultToStdin = False):
+        """
+        Parse a commandline into this parser.
+        Uses OptionParser internally, no args mean sys.argv[1:].
+        """
+        def get_output_file(path):
+            dir = os.path.dirname(path)
+            if dir:
+                try:
+                    os.makedirs(dir)
+                except OSError as error:
+                    if error.errno != errno.EEXIST:
+                        raise
+            return open(path, 'wb')
+
+        p = self.getCommandLineParser()
+        options, args = p.parse_args(args=args)
+        out = self.out
+        depfile = None
+
+        if options.output:
+            out = get_output_file(options.output)
+        if defaultToStdin and len(args) == 0:
+            args = [sys.stdin]
+            if options.depend:
+                raise Preprocessor.Error(self, "--depend doesn't work with stdin",
+                                         None)
+        if options.depend:
+            if not options.output:
+                raise Preprocessor.Error(self, "--depend doesn't work with stdout",
+                                         None)
+            try:
+                from makeutil import Makefile
+            except:
+                raise Preprocessor.Error(self, "--depend requires the "
+                                               "mozbuild.makeutil module", None)
+            depfile = get_output_file(options.depend)
+
+        if args:
+            for f in args:
+                with open(f, 'rU') as input:
+                    self.processFile(input=input, output=out)
+            if depfile:
+                mk = Makefile()
+                mk.create_rule([options.output]).add_dependencies(self.includes)
+                mk.dump(depfile)
+                depfile.close()
+
+        if options.output:
+            out.close()
+
+    def getCommandLineParser(self, unescapeDefines = False):
+        escapedValue = re.compile('".*"$')
+        numberValue = re.compile('\d+$')
+        def handleD(option, opt, value, parser):
+            vals = value.split('=', 1)
+            if len(vals) == 1:
+                vals.append(1)
+            elif unescapeDefines and escapedValue.match(vals[1]):
+                # strip escaped string values
+                vals[1] = vals[1][1:-1]
+            elif numberValue.match(vals[1]):
+                vals[1] = int(vals[1])
+            self.context[vals[0]] = vals[1]
+        def handleU(option, opt, value, parser):
+            del self.context[value]
+        def handleF(option, opt, value, parser):
+            self.do_filter(value)
+        def handleMarker(option, opt, value, parser):
+            self.setMarker(value)
+        def handleSilenceDirectiveWarnings(option, opt, value, parse):
+            self.setSilenceDirectiveWarnings(True)
+        p = OptionParser()
+        p.add_option('-D', action='callback', callback=handleD, type="string",
+                     metavar="VAR[=VAL]", help='Define a variable')
+        p.add_option('-U', action='callback', callback=handleU, type="string",
+                     metavar="VAR", help='Undefine a variable')
+        p.add_option('-F', action='callback', callback=handleF, type="string",
+                     metavar="FILTER", help='Enable the specified filter')
+        p.add_option('-o', '--output', type="string", default=None,
+                     metavar="FILENAME", help='Output to the specified file '+
+                     'instead of stdout')
+        p.add_option('--depend', type="string", default=None, metavar="FILENAME",
+                     help='Generate dependencies in the given file')
+        p.add_option('--marker', action='callback', callback=handleMarker,
+                     type="string",
+                     help='Use the specified marker instead of #')
+        p.add_option('--silence-missing-directive-warnings', action='callback',
+                     callback=handleSilenceDirectiveWarnings,
+                     help='Don\'t emit warnings about missing directives')
+        return p
+
+    def handleLine(self, aLine):
+        """
+        Handle a single line of input (internal).
+        """
+        if self.actionLevel == 0 and self.comment.match(aLine):
+            self.actionLevel = 1
+        m = self.instruction.match(aLine)
+        if m:
+            args = None
+            cmd = m.group('cmd')
+            try:
+                args = m.group('args')
+            except IndexError:
+                pass
+            if cmd not in self.cmds:
+                raise Preprocessor.Error(self, 'INVALID_CMD', aLine)
+            level, cmd = self.cmds[cmd]
+            if (level >= self.disableLevel):
+                cmd(args)
+            if cmd != 'literal':
+                self.actionLevel = 2
+        elif self.disableLevel == 0 and not self.comment.match(aLine):
+            self.write(aLine)
+
+    # Instruction handlers
+    # These are named do_'instruction name' and take one argument
+
+    # Variables
+    def do_define(self, args):
+        m = re.match('(?P<name>\w+)(?:\s(?P<value>.*))?', args, re.U)
+        if not m:
+            raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
+        val = ''
+        if m.group('value'):
+            val = self.applyFilters(m.group('value'))
+            try:
+                val = int(val)
+            except:
+                pass
+        self.context[m.group('name')] = val
+    def do_undef(self, args):
+        m = re.match('(?P<name>\w+)$', args, re.U)
+        if not m:
+            raise Preprocessor.Error(self, 'SYNTAX_DEF', args)
+        if args in self.context:
+            del self.context[args]
+    # Logic
+    def ensure_not_else(self):
+        if len(self.ifStates) == 0 or self.ifStates[-1] == 2:
+            sys.stderr.write('WARNING: bad nesting of #else\n')
+    def do_if(self, args, replace=False):
+        if self.disableLevel and not replace:
+            self.disableLevel += 1
+            return
+        val = None
+        try:
+            e = Expression(args)
+            val = e.evaluate(self.context)
+        except Exception:
+            # XXX do real error reporting
+            raise Preprocessor.Error(self, 'SYNTAX_ERR', args)
+        if type(val) == str:
+            # we're looking for a number value, strings are false
+            val = False
+        if not val:
+            self.disableLevel = 1
+        if replace:
+            if val:
+                self.disableLevel = 0
+            self.ifStates[-1] = self.disableLevel
+        else:
+            self.ifStates.append(self.disableLevel)
+        pass
+    def do_ifdef(self, args, replace=False):
+        if self.disableLevel and not replace:
+            self.disableLevel += 1
+            return
+        if re.match('\W', args, re.U):
+            raise Preprocessor.Error(self, 'INVALID_VAR', args)
+        if args not in self.context:
+            self.disableLevel = 1
+        if replace:
+            if args in self.context:
+                self.disableLevel = 0
+            self.ifStates[-1] = self.disableLevel
+        else:
+            self.ifStates.append(self.disableLevel)
+        pass
+    def do_ifndef(self, args, replace=False):
+        if self.disableLevel and not replace:
+            self.disableLevel += 1
+            return
+        if re.match('\W', args, re.U):
+            raise Preprocessor.Error(self, 'INVALID_VAR', args)
+        if args in self.context:
+            self.disableLevel = 1
+        if replace:
+            if args not in self.context:
+                self.disableLevel = 0
+            self.ifStates[-1] = self.disableLevel
+        else:
+            self.ifStates.append(self.disableLevel)
+        pass
+    def do_else(self, args, ifState = 2):
+        self.ensure_not_else()
+        hadTrue = self.ifStates[-1] == 0
+        self.ifStates[-1] = ifState # in-else
+        if hadTrue:
+            self.disableLevel = 1
+            return
+        self.disableLevel = 0
+    def do_elif(self, args):
+        if self.disableLevel == 1:
+            if self.ifStates[-1] == 1:
+                self.do_if(args, replace=True)
+        else:
+            self.do_else(None, self.ifStates[-1])
+    def do_elifdef(self, args):
+        if self.disableLevel == 1:
+            if self.ifStates[-1] == 1:
+                self.do_ifdef(args, replace=True)
+        else:
+            self.do_else(None, self.ifStates[-1])
+    def do_elifndef(self, args):
+        if self.disableLevel == 1:
+            if self.ifStates[-1] == 1:
+                self.do_ifndef(args, replace=True)
+        else:
+            self.do_else(None, self.ifStates[-1])
+    def do_endif(self, args):
+        if self.disableLevel > 0:
+            self.disableLevel -= 1
+        if self.disableLevel == 0:
+            self.ifStates.pop()
+    # output processing
+    def do_expand(self, args):
+        lst = re.split('__(\w+)__', args, re.U)
+        do_replace = False
+        def vsubst(v):
+            if v in self.context:
+                return str(self.context[v])
+            return ''
+        for i in range(1, len(lst), 2):
+            lst[i] = vsubst(lst[i])
+        lst.append('\n') # add back the newline
+        self.write(reduce(lambda x, y: x+y, lst, ''))
+    def do_literal(self, args):
+        self.write(args + '\n')
+    def do_filter(self, args):
+        filters = [f for f in args.split(' ') if hasattr(self, 'filter_' + f)]
+        if len(filters) == 0:
+            return
+        current = dict(self.filters)
+        for f in filters:
+            current[f] = getattr(self, 'filter_' + f)
+        filterNames = current.keys()
+        filterNames.sort()
+        self.filters = [(fn, current[fn]) for fn in filterNames]
+        return
+    def do_unfilter(self, args):
+        filters = args.split(' ')
+        current = dict(self.filters)
+        for f in filters:
+            if f in current:
+                del current[f]
+        filterNames = current.keys()
+        filterNames.sort()
+        self.filters = [(fn, current[fn]) for fn in filterNames]
+        return
+    # Filters
+    #
+    # emptyLines
+    #   Strips blank lines from the output.
+    def filter_emptyLines(self, aLine):
+        if aLine == '\n':
+            return ''
+        return aLine
+    # slashslash
+    #   Strips everything after //
+    def filter_slashslash(self, aLine):
+        if (aLine.find('//') == -1):
+            return aLine
+        [aLine, rest] = aLine.split('//', 1)
+        if rest:
+            aLine += '\n'
+        return aLine
+    # spaces
+    #   Collapses sequences of spaces into a single space
+    def filter_spaces(self, aLine):
+        return re.sub(' +', ' ', aLine).strip(' ')
+    # substition
+    #   helper to be used by both substition and attemptSubstitution
+    def filter_substitution(self, aLine, fatal=True):
+        def repl(matchobj):
+            varname = matchobj.group('VAR')
+            if varname in self.context:
+                return str(self.context[varname])
+            if fatal:
+                raise Preprocessor.Error(self, 'UNDEFINED_VAR', varname)
+            return matchobj.group(0)
+        return self.varsubst.sub(repl, aLine)
+    def filter_attemptSubstitution(self, aLine):
+        return self.filter_substitution(aLine, fatal=False)
+    # File ops
+    def do_include(self, args, filters=True):
+        """
+        Preprocess a given file.
+        args can either be a file name, or a file-like object.
+        Files should be opened, and will be closed after processing.
+        """
+        isName = type(args) == str or type(args) == unicode
+        oldCheckLineNumbers = self.checkLineNumbers
+        self.checkLineNumbers = False
+        if isName:
+            try:
+                args = str(args)
+                if filters:
+                    args = self.applyFilters(args)
+                if not os.path.isabs(args):
+                    args = os.path.join(self.context['DIRECTORY'], args)
+                args = open(args, 'rU')
+            except Preprocessor.Error:
+                raise
+            except:
+                raise Preprocessor.Error(self, 'FILE_NOT_FOUND', str(args))
+        self.checkLineNumbers = bool(re.search('\.(js|jsm|java)(?:\.in)?$', args.name))
+        oldFile = self.context['FILE']
+        oldLine = self.context['LINE']
+        oldDir = self.context['DIRECTORY']
+        self.noteLineInfo()
+
+        if args.isatty():
+            # we're stdin, use '-' and '' for file and dir
+            self.context['FILE'] = '-'
+            self.context['DIRECTORY'] = ''
+        else:
+            abspath = os.path.abspath(args.name)
+            self.includes.add(abspath)
+            self.context['FILE'] = abspath
+            self.context['DIRECTORY'] = os.path.dirname(abspath)
+        self.context['LINE'] = 0
+
+        for l in args:
+            self.context['LINE'] += 1
+            self.handleLine(l)
+        if isName:
+            args.close()
+
+        self.context['FILE'] = oldFile
+        self.checkLineNumbers = oldCheckLineNumbers
+        self.context['LINE'] = oldLine
+        self.context['DIRECTORY'] = oldDir
+    def do_includesubst(self, args):
+        args = self.filter_substitution(args)
+        self.do_include(args)
+    def do_error(self, args):
+        raise Preprocessor.Error(self, 'Error: ', str(args))
+
+
+def preprocess(includes=[sys.stdin], defines={},
+               output = sys.stdout,
+               marker='#'):
+    pp = Preprocessor(defines=defines,
+                      marker=marker)
+    for f in includes:
+        with open(f, 'rU') as input:
+            pp.processFile(input=input, output=output)
+    return pp.includes
+
+
+# Keep this module independently executable.
+if __name__ == "__main__":
+    pp = Preprocessor()
+    pp.handleCommandLine(None, True)

python/mozbuild/mozbuild/pythonutil.py

@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import os
+import sys
+
+
+def iter_modules_in_path(*paths):
+    paths = [os.path.abspath(os.path.normcase(p)) + os.sep
+             for p in paths]
+    for name, module in sys.modules.items():
+        if not hasattr(module, '__file__'):
+            continue
+
+        path = module.__file__
+
+        if path.endswith('.pyc'):
+            path = path[:-1]
+        path = os.path.abspath(os.path.normcase(path))
+
+        if any(path.startswith(p) for p in paths):
+            yield path

python/mozbuild/mozbuild/resources/html-build-viewer/index.html

@@ -0,0 +1,475 @@
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Build System Resource Usage</title>
+
+    <meta charset='utf-8'>
+    <script src="http://d3js.org/d3.v3.min.js" charset="utf-8"></script>
+    <style>
+
+svg {
+  overflow: visible;
+}
+
+.axis path,
+.axis line {
+  fill: none;
+  stroke: #000;
+  shape-rendering: crispEdges;
+}
+
+.area {
+  fill: steelblue;
+}
+
+.graphs {
+  text-anchor: end;
+}
+
+.timeline {
+  fill: steelblue;
+  stroke: gray;
+  stroke-width: 3;
+}
+
+.short {
+  fill: gray;
+  stroke: gray;
+  stroke-width: 3;
+}
+
+#tooltip {
+  z-index: 10;
+  position: fixed;
+  background: #efefef;
+}
+    </style>
+  </head>
+  <body>
+    <script>
+var currentResources;
+
+/**
+ * Interface for a build resources JSON file.
+ */
+function BuildResources(data) {
+  if (data.version < 1 || data.version > 3) {
+    throw new Error("Unsupported version of the JSON format: " + data.version);
+  }
+
+  this.resources = [];
+
+  var cpu_fields = data.cpu_times_fields;
+  var io_fields = data.io_fields;
+  var virt_fields = data.virt_fields;
+  var swap_fields = data.swap_fields;
+
+  function convert(dest, source, sourceKey, destKey, fields) {
+    var i = 0;
+    fields.forEach(function (field) {
+      dest[destKey][field] = source[sourceKey][i];
+      i++;
+    });
+  }
+
+  var offset = data.start;
+  var cpu_times_totals = {};
+
+  cpu_fields.forEach(function (field) {
+    cpu_times_totals[field] = 0;
+  });
+
+  this.ioTotal = {};
+  var i = 0;
+  io_fields.forEach(function (field) {
+    this.ioTotal[field] = data.overall.io[i];
+    i++;
+  }.bind(this));
+
+  data.samples.forEach(function (sample) {
+    var entry = {
+      start: sample.start - offset,
+      end: sample.end - offset,
+      duration: sample.duration,
+      cpu_percent: sample.cpu_percent_mean,
+      cpu_times: {},
+      cpu_times_percents: {},
+      io: {},
+      virt: {},
+      swap: {},
+    };
+
+    convert(entry, sample, "cpu_times_sum", "cpu_times", cpu_fields);
+    convert(entry, sample, "io", "io", io_fields);
+    convert(entry, sample, "virt", "virt", virt_fields);
+    convert(entry, sample, "swap", "swap", swap_fields);
+
+    var total = 0;
+    for (var k in entry.cpu_times) {
+      cpu_times_totals[k] += entry.cpu_times[k];
+      total += entry.cpu_times[k];
+    }
+
+    for (var k in entry.cpu_times) {
+      if (total == 0) {
+        if (k == "idle") {
+          entry.cpu_times_percents[k] = 100;
+        } else {
+          entry.cpu_times_percents[k] = 0;
+        }
+      } else {
+        entry.cpu_times_percents[k] = entry.cpu_times[k] / total * 100;
+      }
+    }
+
+    this.resources.push(entry);
+  }.bind(this));
+
+  this.cpu_times_fields = [];
+
+  // Filter out CPU fields that have no values.
+  for (var k in cpu_times_totals) {
+    var v = cpu_times_totals[k];
+    if (v) {
+      this.cpu_times_fields.push(k);
+      continue;
+    }
+
+    this.resources.forEach(function (entry) {
+      delete entry.cpu_times[k];
+      delete entry.cpu_times_percents[k];
+    });
+  }
+
+  this.offset = offset;
+  this.data = data;
+}
+
+BuildResources.prototype = Object.freeze({
+  get start() {
+    return this.data.start;
+  },
+
+  get startDate() {
+    return new Date(this.start * 1000);
+  },
+
+  get end() {
+    return this.data.end;
+  },
+
+  get endDate() {
+    return new Date(this.end * 1000);
+  },
+
+  get duration() {
+    return this.data.duration;
+  },
+
+  get sample_times() {
+    var times = [];
+    this.resources.forEach(function (sample) {
+      times.push(sample.start);
+    });
+
+    return times;
+  },
+
+  get cpuPercent() {
+    return this.data.overall.cpu_percent_mean;
+  },
+
+  get tiers() {
+    var t = [];
+
+    this.data.phases.forEach(function (e) {
+      t.push(e.name);
+    });
+
+    return t;
+  },
+
+  getTier: function (tier) {
+    for (var i = 0; i < this.data.phases.length; i++) {
+      var t = this.data.phases[i];
+
+      if (t.name == tier) {
+        return t;
+      }
+    }
+  },
+});
+
+function updateResourcesGraph() {
+  //var selected = document.getElementById("resourceType");
+  //var what = selected[selected.selectedIndex].value;
+  var what = "cpu";
+
+  renderResources("resource_graph", currentResources, what);
+  document.getElementById("wall_time").innerHTML = Math.round(currentResources.duration * 100) / 100;
+  document.getElementById("start_date").innerHTML = currentResources.startDate.toISOString();
+  document.getElementById("end_date").innerHTML = currentResources.endDate.toISOString();
+  document.getElementById("cpu_percent").innerHTML = Math.round(currentResources.cpuPercent * 100) / 100;
+  document.getElementById("write_bytes").innerHTML = currentResources.ioTotal["write_bytes"];
+  document.getElementById("read_bytes").innerHTML = currentResources.ioTotal["read_bytes"];
+  document.getElementById("write_time").innerHTML = currentResources.ioTotal["write_time"];
+  document.getElementById("read_time").innerHTML = currentResources.ioTotal["read_time"];
+}
+
+function renderKey(key) {
+  d3.json("/resources/" + key, function onResource(error, response) {
+    if (error) {
+      alert("Data not available. Is the server still running?");
+      return;
+    }
+
+    currentResources = new BuildResources(response);
+    updateResourcesGraph();
+  });
+}
+
+function renderResources(id, resources, what) {
+  document.getElementById(id).innerHTML = "";
+
+  var margin = {top: 20, right: 20, bottom: 20, left: 50};
+  var width = window.innerWidth - 50 - margin.left - margin.right;
+  var height = 400 - margin.top - margin.bottom;
+
+  var x = d3.scale.linear()
+    .range([0, width])
+    .domain(d3.extent(resources.resources, function (d) { return d.start; }))
+    ;
+  var y = d3.scale.linear()
+    .range([height, 0])
+    .domain([0, 1])
+    ;
+
+  var xAxis = d3.svg.axis()
+    .scale(x)
+    .orient("bottom")
+    ;
+  var yAxis = d3.svg.axis()
+    .scale(y)
+    .orient("left")
+    .tickFormat(d3.format(".0%"))
+    ;
+
+  var area = d3.svg.area()
+    .x(function (d) { return x(d.start); })
+    .y0(function(d) { return y(d.y0); })
+    .y1(function(d) { return y(d.y0 + d.y); })
+    ;
+
+  var stack = d3.layout.stack()
+    .values(function (d) { return d.values; })
+    ;
+
+  // Manually control the layer order because we want it consistent and want
+  // to inject some sanity.
+  var layers = [
+    ["nice", "#0d9fff"],
+    ["irq", "#ff0d9f"],
+    ["softirq", "#ff0d9f"],
+    ["steal", "#000000"],
+    ["guest", "#000000"],
+    ["guest_nice", "#000000"],
+    ["system", "#f69a5c"],
+    ["iowait", "#ff0d25"],
+    ["user", "#5cb9f6"],
+    ["idle", "#e1e1e1"],
+  ].filter(function (l) {
+    return resources.cpu_times_fields.indexOf(l[0]) != -1;
+  });
+
+  // Draw a legend.
+  var legend = d3.select("#" + id)
+    .append("svg")
+      .attr("width", width + margin.left + margin.right)
+      .attr("height", 15)
+      .append("g")
+        .attr("class", "legend")
+    ;
+
+  legend.selectAll("g")
+    .data(layers)
+    .enter()
+      .append("g")
+        .each(function (d, i) {
+          var g = d3.select(this);
+          g.append("rect")
+            .attr("x", i * 100 + 20)
+            .attr("y", 0)
+            .attr("width", 10)
+            .attr("height", 10)
+            .style("fill", d[1])
+            ;
+          g.append("text")
+            .attr("x", i * 100 + 40)
+            .attr("y", 10)
+            .attr("height", 10)
+            .attr("width", 70)
+            .text(d[0])
+            ;
+          })
+    ;
+
+  var svg = d3.select("#" + id).append("svg")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", height + margin.top + margin.bottom)
+    .append("g")
+      .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
+    ;
+
+  var data = stack(layers.map(function (layer) {
+    return {
+      name: layer[0],
+      color: layer[1],
+      values: resources.resources.map(function (d) {
+        return {
+          start: d.start,
+          y: d.cpu_times_percents[layer[0]] / 100,
+        };
+      }),
+    };
+  }));
+
+  var graphs = svg.selectAll(".graphs")
+    .data(data)
+    .enter().append("g")
+      .attr("class", "graphs")
+    ;
+
+  graphs.append("path")
+    .attr("class", "area")
+    .attr("d", function (d) { return area(d.values); })
+    .style("fill", function (d) { return d.color; })
+    ;
+
+  svg.append("g")
+    .attr("class", "x axis")
+    .attr("transform", "translate(0," + height + ")")
+    .call(xAxis)
+    ;
+
+  svg.append("g")
+    .attr("class", "y axis")
+    .call(yAxis)
+    ;
+
+  // Now we render a timeline of sorts of the tiers
+  // There is a row of rectangles that visualize divisions between the
+  // different items. We use the same x scale as the resource graph so times
+  // line up properly.
+  svg = d3.select("#" + id).append("svg")
+    .attr("width", width + margin.left + margin.right)
+    .attr("height", 100 + margin.top + margin.bottom)
+    .append("g")
+      .attr("transform", "translate(" + margin.left + "," + margin.top + ")")
+    ;
+
+  var y = d3.scale.linear().range([10, 0]).domain([0, 1]);
+
+  resources.tiers.forEach(function (t, i) {
+    var tier = resources.getTier(t);
+
+    var x_start = x(tier.start - resources.offset);
+    var x_end = x(tier.end - resources.offset);
+
+    svg.append("rect")
+      .attr("x", x_start)
+      .attr("y", 20)
+      .attr("height", 30)
+      .attr("width", x_end - x_start)
+      .attr("class", "timeline tier")
+      .attr("tier", t)
+      ;
+  });
+
+  function getEntry(element) {
+    var tier = element.getAttribute("tier");
+
+    var entry = resources.getTier(tier);
+    entry.tier = tier;
+
+    return entry;
+  }
+
+  d3.selectAll(".timeline")
+  .on("mouseenter", function () {
+      var entry = getEntry(this);
+
+      d3.select("#tt_tier").html(entry.tier);
+      d3.select("#tt_duration").html(entry.duration || "n/a");
+      d3.select("#tt_cpu_percent").html(entry.cpu_percent_mean || "n/a");
+
+      d3.select("#tooltip").style("display", "");
+    })
+    .on("mouseleave", function () {
+      var tooltip = d3.select("#tooltip");
+      tooltip.style("display", "none");
+    })
+    .on("mousemove", function () {
+      var e = d3.event;
+      x_offset = 10;
+
+      if (e.pageX > window.innerWidth / 2) {
+        x_offset = -150;
+      }
+
+      d3.select("#tooltip")
+        .style("left", (e.pageX + x_offset) + "px")
+        .style("top", (e.pageY + 10) + "px")
+        ;
+    })
+    ;
+}
+
+document.addEventListener("DOMContentLoaded", function() {
+  d3.json("list", function onList(error, response) {
+    if (!response || !("files" in response)) {
+      return;
+    }
+
+    renderKey(response.files[0]);
+  });
+}, false);
+
+    </script>
+    <h3>Build Resource Usage Report</h3>
+
+    <div id="tooltip" style="display: none;">
+      <table border="0">
+        <tr><td>Tier</td><td id="tt_tier"></td></tr>
+        <tr><td>Duration</td><td id="tt_duration"></td></tr>
+        <tr><td>CPU %</td><td id="tt_cpu_percent"></td></tr>
+      </table>
+    </div>
+
+    <!--
+    <select id="resourceType" onchange="updateResourcesGraph();">
+      <option value="cpu">CPU</option>
+      <option value="io_count">Disk I/O Count</option>
+      <option value="io_bytes">Disk I/O Bytes</option>
+      <option value="io_time">Disk I/O Time</option>
+      <option value="virt">Memory</option>
+    </select>
+    -->
+
+    <div id="resource_graph"></div>
+    <div id="summary" style="padding-top: 20px">
+      <table border="0">
+        <tr><td>Wall Time (s)</td><td id="wall_time"></td></tr>
+        <tr><td>Start Date</td><td id="start_date"></td></tr>
+        <tr><td>End Date</td><td id="end_date"></td></tr>
+        <tr><td>CPU %</td><td id="cpu_percent"></td></tr>
+        <tr><td>Write Bytes</td><td id="write_bytes"></td></tr>
+        <tr><td>Read Bytes</td><td id="read_bytes"></td></tr>
+        <tr><td>Write Time</td><td id="write_time"></td></tr>
+        <tr><td>Read Time</td><td id="read_time"></td></tr>
+      </table>
+    </div>
+  </body>
+</html>

python/mozbuild/mozbuild/shellutil.py

@@ -0,0 +1,209 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import re
+
+
+def _tokens2re(**tokens):
+    # Create a pattern for non-escaped tokens, in the form:
+    #   (?<!\\)(?:a|b|c...)
+    # This is meant to match patterns a, b, or c, or ... if they are not
+    # preceded by a backslash.
+    # where a, b, c... are in the form
+    #   (?P<name>pattern)
+    # which matches the pattern and captures it in a named match group.
+    # The group names and patterns are given as arguments.
+    all_tokens = '|'.join('(?P<%s>%s)' % (name, value)
+                          for name, value in tokens.iteritems())
+    nonescaped = r'(?<!\\)(?:%s)' % all_tokens
+
+    # The final pattern matches either the above pattern, or an escaped
+    # backslash, captured in the "escape" match group.
+    return re.compile('(?:%s|%s)' % (nonescaped, r'(?P<escape>\\\\)'))
+
+UNQUOTED_TOKENS_RE = _tokens2re(
+  whitespace=r'[\t\r\n ]+',
+  quote=r'[\'"]',
+  comment='#',
+  special=r'[<>&|`~(){}$;\*\?]',
+  backslashed=r'\\[^\\]',
+)
+
+DOUBLY_QUOTED_TOKENS_RE = _tokens2re(
+  quote='"',
+  backslashedquote=r'\\"',
+  special='\$',
+  backslashed=r'\\[^\\"]',
+)
+
+ESCAPED_NEWLINES_RE = re.compile(r'\\\n')
+
+# This regexp contains the same characters as all those listed in
+# UNQUOTED_TOKENS_RE. Please keep in sync.
+SHELL_QUOTE_RE = re.compile(r'[\\\t\r\n \'\"#<>&|`~(){}$;\*\?]')
+
+
+class MetaCharacterException(Exception):
+    def __init__(self, char):
+        self.char = char
+
+
+class _ClineSplitter(object):
+    '''
+    Parses a given command line string and creates a list of command
+    and arguments, with wildcard expansion.
+    '''
+    def __init__(self, cline):
+        self.arg = None
+        self.cline = cline
+        self.result = []
+        self._parse_unquoted()
+
+    def _push(self, str):
+        '''
+        Push the given string as part of the current argument
+        '''
+        if self.arg is None:
+            self.arg = ''
+        self.arg += str
+
+    def _next(self):
+        '''
+        Finalize current argument, effectively adding it to the list.
+        '''
+        if self.arg is None:
+            return
+        self.result.append(self.arg)
+        self.arg = None
+
+    def _parse_unquoted(self):
+        '''
+        Parse command line remainder in the context of an unquoted string.
+        '''
+        while self.cline:
+            # Find the next token
+            m = UNQUOTED_TOKENS_RE.search(self.cline)
+            # If we find none, the remainder of the string can be pushed to
+            # the current argument and the argument finalized
+            if not m:
+                self._push(self.cline)
+                break
+            # The beginning of the string, up to the found token, is part of
+            # the current argument
+            if m.start():
+                self._push(self.cline[:m.start()])
+            self.cline = self.cline[m.end():]
+
+            match = {name: value
+                     for name, value in m.groupdict().items() if value}
+            if 'quote' in match:
+                # " or ' start a quoted string
+                if match['quote'] == '"':
+                    self._parse_doubly_quoted()
+                else:
+                    self._parse_quoted()
+            elif 'comment' in match:
+                # Comments are ignored. The current argument can be finalized,
+                # and parsing stopped.
+                break
+            elif 'special' in match:
+                # Unquoted, non-escaped special characters need to be sent to a
+                # shell.
+                raise MetaCharacterException(match['special'])
+            elif 'whitespace' in match:
+                # Whitespaces terminate current argument.
+                self._next()
+            elif 'escape' in match:
+                # Escaped backslashes turn into a single backslash
+                self._push('\\')
+            elif 'backslashed' in match:
+                # Backslashed characters are unbackslashed
+                # e.g. echo \a -> a
+                self._push(match['backslashed'][1])
+            else:
+                raise Exception("Shouldn't reach here")
+        if self.arg:
+            self._next()
+
+    def _parse_quoted(self):
+        # Single quoted strings are preserved, except for the final quote
+        index = self.cline.find("'")
+        if index == -1:
+            raise Exception('Unterminated quoted string in command')
+        self._push(self.cline[:index])
+        self.cline = self.cline[index+1:]
+
+    def _parse_doubly_quoted(self):
+        if not self.cline:
+            raise Exception('Unterminated quoted string in command')
+        while self.cline:
+            m = DOUBLY_QUOTED_TOKENS_RE.search(self.cline)
+            if not m:
+                raise Exception('Unterminated quoted string in command')
+            self._push(self.cline[:m.start()])
+            self.cline = self.cline[m.end():]
+            match = {name: value
+                     for name, value in m.groupdict().items() if value}
+            if 'quote' in match:
+                # a double quote ends the quoted string, so go back to
+                # unquoted parsing
+                return
+            elif 'special' in match:
+                # Unquoted, non-escaped special characters in a doubly quoted
+                # string still have a special meaning and need to be sent to a
+                # shell.
+                raise MetaCharacterException(match['special'])
+            elif 'escape' in match:
+                # Escaped backslashes turn into a single backslash
+                self._push('\\')
+            elif 'backslashedquote' in match:
+                # Backslashed double quotes are un-backslashed
+                self._push('"')
+            elif 'backslashed' in match:
+                # Backslashed characters are kept backslashed
+                self._push(match['backslashed'])
+
+
+def split(cline):
+    '''
+    Split the given command line string.
+    '''
+    s = ESCAPED_NEWLINES_RE.sub('', cline)
+    return _ClineSplitter(s).result
+
+
+def _quote(s):
+    '''Given a string, returns a version that can be used literally on a shell
+    command line, enclosing it with single quotes if necessary.
+
+    As a special case, if given an int, returns a string containing the int,
+    not enclosed in quotes.
+    '''
+    if type(s) == int:
+        return '%d' % s
+
+    # Empty strings need to be quoted to have any significance
+    if s and not SHELL_QUOTE_RE.search(s):
+        return s
+
+    # Single quoted strings can contain any characters unescaped except the
+    # single quote itself, which can't even be escaped, so the string needs to
+    # be closed, an escaped single quote added, and reopened.
+    t = type(s)
+    return t("'%s'") % s.replace(t("'"), t("'\\''"))
+
+
+def quote(*strings):
+    '''Given one or more strings, returns a quoted string that can be used
+    literally on a shell command line.
+
+        >>> quote('a', 'b')
+        "a b"
+        >>> quote('a b', 'c')
+        "'a b' c"
+    '''
+    return ' '.join(_quote(s) for s in strings)
+
+
+__all__ = ['MetaCharacterException', 'split', 'quote']

python/mozbuild/mozbuild/sphinx.py

@@ -0,0 +1,200 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import importlib
+import os
+import sys
+
+from sphinx.util.compat import Directive
+from sphinx.util.docstrings import prepare_docstring
+
+
+def function_reference(f, attr, args, doc):
+    lines = []
+
+    lines.extend([
+        f,
+        '-' * len(f),
+        '',
+    ])
+
+    docstring = prepare_docstring(doc)
+
+    lines.extend([
+        docstring[0],
+        '',
+    ])
+
+    arg_types = []
+
+    for t in args:
+        if isinstance(t, list):
+            inner_types = [t2.__name__ for t2 in t]
+            arg_types.append(' | ' .join(inner_types))
+            continue
+
+        arg_types.append(t.__name__)
+
+    arg_s = '(%s)' % ', '.join(arg_types)
+
+    lines.extend([
+        ':Arguments: %s' % arg_s,
+        '',
+    ])
+
+    lines.extend(docstring[1:])
+    lines.append('')
+
+    return lines
+
+
+def variable_reference(v, st_type, in_type, doc):
+    lines = [
+        v,
+        '-' * len(v),
+        '',
+    ]
+
+    docstring = prepare_docstring(doc)
+
+    lines.extend([
+        docstring[0],
+        '',
+    ])
+
+    lines.extend([
+        ':Storage Type: ``%s``' % st_type.__name__,
+        ':Input Type: ``%s``' % in_type.__name__,
+        '',
+    ])
+
+    lines.extend(docstring[1:])
+    lines.append('')
+
+    return lines
+
+
+def special_reference(v, func, typ, doc):
+    lines = [
+        v,
+        '-' * len(v),
+        '',
+    ]
+
+    docstring = prepare_docstring(doc)
+
+    lines.extend([
+        docstring[0],
+        '',
+        ':Type: ``%s``' % typ.__name__,
+        '',
+    ])
+
+    lines.extend(docstring[1:])
+    lines.append('')
+
+    return lines
+
+
+def format_module(m):
+    lines = []
+
+    for subcontext, cls in sorted(m.SUBCONTEXTS.items()):
+        lines.extend([
+            '.. _mozbuild_subcontext_%s:' % subcontext,
+            '',
+            'Sub-Context: %s' % subcontext,
+            '=============' + '=' * len(subcontext),
+            '',
+        ])
+        lines.extend(prepare_docstring(cls.__doc__))
+        if lines[-1]:
+            lines.append('')
+
+        for k, v in sorted(cls.VARIABLES.items()):
+            lines.extend(variable_reference(k, *v))
+
+    lines.extend([
+        'Variables',
+        '=========',
+        '',
+    ])
+
+    for v in sorted(m.VARIABLES):
+        lines.extend(variable_reference(v, *m.VARIABLES[v]))
+
+    lines.extend([
+        'Functions',
+        '=========',
+        '',
+    ])
+
+    for func in sorted(m.FUNCTIONS):
+        lines.extend(function_reference(func, *m.FUNCTIONS[func]))
+
+    lines.extend([
+        'Special Variables',
+        '=================',
+        '',
+    ])
+
+    for v in sorted(m.SPECIAL_VARIABLES):
+        lines.extend(special_reference(v, *m.SPECIAL_VARIABLES[v]))
+
+    return lines
+
+
+class MozbuildSymbols(Directive):
+    """Directive to insert mozbuild sandbox symbol information."""
+
+    required_arguments = 1
+
+    def run(self):
+        module = importlib.import_module(self.arguments[0])
+        fname = module.__file__
+        if fname.endswith('.pyc'):
+            fname = fname[0:-1]
+
+        self.state.document.settings.record_dependencies.add(fname)
+
+        # We simply format out the documentation as rst then feed it back
+        # into the parser for conversion. We don't even emit ourselves, so
+        # there's no record of us.
+        self.state_machine.insert_input(format_module(module), fname)
+
+        return []
+
+
+def setup(app):
+    app.add_directive('mozbuildsymbols', MozbuildSymbols)
+
+    # Unlike typical Sphinx installs, our documentation is assembled from
+    # many sources and staged in a common location. This arguably isn't a best
+    # practice, but it was the easiest to implement at the time.
+    #
+    # Here, we invoke our custom code for staging/generating all our
+    # documentation.
+    from moztreedocs import SphinxManager
+
+    topsrcdir = app.config._raw_config['topsrcdir']
+    manager = SphinxManager(topsrcdir,
+        os.path.join(topsrcdir, 'tools', 'docs'),
+        app.outdir)
+    manager.generate_docs(app)
+
+    app.srcdir = os.path.join(app.outdir, '_staging')
+
+    # We need to adjust sys.path in order for Python API docs to get generated
+    # properly. We leverage the in-tree virtualenv for this.
+    from mozbuild.virtualenv import VirtualenvManager
+
+    ve = VirtualenvManager(topsrcdir,
+        os.path.join(topsrcdir, 'dummy-objdir'),
+        os.path.join(app.outdir, '_venv'),
+        sys.stderr,
+        os.path.join(topsrcdir, 'build', 'virtualenv_packages.txt'))
+    ve.ensure()
+    ve.activate()

python/mozbuild/mozbuild/test/action/data/invalid/region.properties

@@ -0,0 +1,12 @@
+# A region.properties file with invalid unicode byte sequences.  The
+# sequences were cribbed from Markus Kuhn's "UTF-8 decoder capability
+# and stress test", available at
+# http://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt
+
+# 3.5  Impossible bytes                                                         |
+#                                                                               |
+# The following two bytes cannot appear in a correct UTF-8 string               |
+#                                                                               |
+# 3.5.1  fe = "þ"                                                               |
+# 3.5.2  ff = "ÿ"                                                               |
+# 3.5.3  fe fe ff ff = "þþÿÿ"                                                   |

python/mozbuild/mozbuild/test/action/data/package_fennec_apk/assets/asset.txt

@@ -0,0 +1 @@
+assets/asset.txt

python/mozbuild/mozbuild/test/action/data/package_fennec_apk/classes.dex

@@ -0,0 +1 @@
+classes.dex

python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/res/res.txt

@@ -0,0 +1 @@
+input1/res/res.txt

python/mozbuild/mozbuild/test/action/data/package_fennec_apk/input1/resources.arsc

@@ -0,0 +1 @@
+input1/resources.arsc