import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

testing/mozbase/mozprofile/mozprofile/__init__.py

@@ -0,0 +1,21 @@
+# flake8: noqa
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+To use mozprofile as an API you can import mozprofile.profile_ and/or the AddonManager_.
+
+``mozprofile.profile`` features a generic ``Profile`` class.  In addition,
+subclasses ``FirefoxProfile`` and ``ThundebirdProfile`` are available
+with preset preferences for those applications.
+"""
+
+from addons import *
+from cli import *
+from diff import *
+from permissions import *
+from prefs import *
+from profile import *
+from view import *
+from webapps import *

testing/mozbase/mozprofile/mozprofile/addons.py

@@ -0,0 +1,410 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import sys
+import tempfile
+import urllib2
+import zipfile
+from xml.dom import minidom
+
+import mozfile
+from mozlog.unstructured import getLogger
+
+# Needed for the AMO's rest API -
+# https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API
+AMO_API_VERSION = "1.5"
+
+# Logger for 'mozprofile.addons' module
+module_logger = getLogger(__name__)
+
+
+class AddonFormatError(Exception):
+    """Exception for not well-formed add-on manifest files"""
+
+
+class AddonManager(object):
+    """
+    Handles all operations regarding addons in a profile including:
+    installing and cleaning addons
+    """
+
+    def __init__(self, profile, restore=True):
+        """
+        :param profile: the path to the profile for which we install addons
+        :param restore: whether to reset to the previous state on instance garbage collection
+        """
+        self.profile = profile
+        self.restore = restore
+
+        # Initialize all class members
+        self._internal_init()
+
+    def _internal_init(self):
+        """Internal: Initialize all class members to their default value"""
+
+        # Add-ons installed; needed for cleanup
+        self._addons = []
+
+        # Backup folder for already existing addons
+        self.backup_dir = None
+
+        # Add-ons downloaded and which have to be removed from the file system
+        self.downloaded_addons = []
+
+        # Information needed for profile reset (see http://bit.ly/17JesUf)
+        self.installed_addons = []
+        self.installed_manifests = []
+
+    def __del__(self):
+        # reset to pre-instance state
+        if self.restore:
+            self.clean()
+
+    def clean(self):
+        """Clean up addons in the profile."""
+
+        # Remove all add-ons installed
+        for addon in self._addons:
+            # TODO (bug 934642)
+            # Once we have a proper handling of add-ons we should kill the id
+            # from self._addons once the add-on is removed. For now lets forget
+            # about the exception
+            try:
+                self.remove_addon(addon)
+            except IOError:
+                pass
+
+        # Remove all downloaded add-ons
+        for addon in self.downloaded_addons:
+            mozfile.remove(addon)
+
+        # restore backups
+        if self.backup_dir and os.path.isdir(self.backup_dir):
+            extensions_path = os.path.join(self.profile, 'extensions', 'staged')
+
+            for backup in os.listdir(self.backup_dir):
+                backup_path = os.path.join(self.backup_dir, backup)
+                shutil.move(backup_path, extensions_path)
+
+            if not os.listdir(self.backup_dir):
+                mozfile.remove(self.backup_dir)
+
+        # reset instance variables to defaults
+        self._internal_init()
+
+    @classmethod
+    def download(self, url, target_folder=None):
+        """
+        Downloads an add-on from the specified URL to the target folder
+
+        :param url: URL of the add-on (XPI file)
+        :param target_folder: Folder to store the XPI file in
+
+        """
+        response = urllib2.urlopen(url)
+        fd, path = tempfile.mkstemp(suffix='.xpi')
+        os.write(fd, response.read())
+        os.close(fd)
+
+        if not self.is_addon(path):
+            mozfile.remove(path)
+            raise AddonFormatError('Not a valid add-on: %s' % url)
+
+        # Give the downloaded file a better name by using the add-on id
+        details = self.addon_details(path)
+        new_path = path.replace('.xpi', '_%s.xpi' % details.get('id'))
+
+        # Move the add-on to the target folder if requested
+        if target_folder:
+            new_path = os.path.join(target_folder, os.path.basename(new_path))
+
+        os.rename(path, new_path)
+
+        return new_path
+
+    def get_addon_path(self, addon_id):
+        """Returns the path to the installed add-on
+
+        :param addon_id: id of the add-on to retrieve the path from
+        """
+        # By default we should expect add-ons being located under the
+        # extensions folder. Only if the application hasn't been run and
+        # installed the add-ons yet, it will be located under 'staged'.
+        # Also add-ons could have been unpacked by the application.
+        extensions_path = os.path.join(self.profile, 'extensions')
+        paths = [os.path.join(extensions_path, addon_id),
+                 os.path.join(extensions_path, addon_id + '.xpi'),
+                 os.path.join(extensions_path, 'staged', addon_id),
+                 os.path.join(extensions_path, 'staged', addon_id + '.xpi')]
+        for path in paths:
+            if os.path.exists(path):
+                return path
+
+        raise IOError('Add-on not found: %s' % addon_id)
+
+    @classmethod
+    def is_addon(self, addon_path):
+        """
+        Checks if the given path is a valid addon
+
+        :param addon_path: path to the add-on directory or XPI
+        """
+        try:
+            self.addon_details(addon_path)
+            return True
+        except AddonFormatError:
+            return False
+
+    def install_addons(self, addons=None, manifests=None):
+        """
+        Installs all types of addons
+
+        :param addons: a list of addon paths to install
+        :param manifest: a list of addon manifests to install
+        """
+
+        # install addon paths
+        if addons:
+            if isinstance(addons, basestring):
+                addons = [addons]
+            for addon in set(addons):
+                self.install_from_path(addon)
+
+        # install addon manifests
+        if manifests:
+            if isinstance(manifests, basestring):
+                manifests = [manifests]
+            for manifest in manifests:
+                self.install_from_manifest(manifest)
+
+    def install_from_manifest(self, filepath):
+        """
+        Installs addons from a manifest
+        :param filepath: path to the manifest of addons to install
+        """
+        try:
+            from manifestparser import ManifestParser
+        except ImportError:
+            module_logger.critical(
+                "Installing addons from manifest requires the"
+                " manifestparser package to be installed.")
+            raise
+
+        manifest = ManifestParser()
+        manifest.read(filepath)
+        addons = manifest.get()
+
+        for addon in addons:
+            if '://' in addon['path'] or os.path.exists(addon['path']):
+                self.install_from_path(addon['path'])
+                continue
+
+            # No path specified, try to grab it off AMO
+            locale = addon.get('amo_locale', 'en_US')
+            query = 'https://services.addons.mozilla.org/' + locale + '/firefox/api/' \
+                    + AMO_API_VERSION + '/'
+            if 'amo_id' in addon:
+                # this query grabs information on the addon base on its id
+                query += 'addon/' + addon['amo_id']
+            else:
+                # this query grabs information on the first addon returned from a search
+                query += 'search/' + addon['name'] + '/default/1'
+            install_path = AddonManager.get_amo_install_path(query)
+            self.install_from_path(install_path)
+
+        self.installed_manifests.append(filepath)
+
+    @classmethod
+    def get_amo_install_path(self, query):
+        """
+        Get the addon xpi install path for the specified AMO query.
+
+        :param query: query-documentation_
+
+        .. _query-documentation: https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API # noqa
+        """
+        response = urllib2.urlopen(query)
+        dom = minidom.parseString(response.read())
+        for node in dom.getElementsByTagName('install')[0].childNodes:
+            if node.nodeType == node.TEXT_NODE:
+                return node.data
+
+    @classmethod
+    def addon_details(cls, addon_path):
+        """
+        Returns a dictionary of details about the addon.
+
+        :param addon_path: path to the add-on directory or XPI
+
+        Returns::
+
+            {'id':      u'rainbow@colors.org', # id of the addon
+             'version': u'1.4',                # version of the addon
+             'name':    u'Rainbow',            # name of the addon
+             'unpack':  False }                # whether to unpack the addon
+        """
+
+        details = {
+            'id': None,
+            'unpack': False,
+            'name': None,
+            'version': None
+        }
+
+        def get_namespace_id(doc, url):
+            attributes = doc.documentElement.attributes
+            namespace = ""
+            for i in range(attributes.length):
+                if attributes.item(i).value == url:
+                    if ":" in attributes.item(i).name:
+                        # If the namespace is not the default one remove 'xlmns:'
+                        namespace = attributes.item(i).name.split(':')[1] + ":"
+                        break
+            return namespace
+
+        def get_text(element):
+            """Retrieve the text value of a given node"""
+            rc = []
+            for node in element.childNodes:
+                if node.nodeType == node.TEXT_NODE:
+                    rc.append(node.data)
+            return ''.join(rc).strip()
+
+        if not os.path.exists(addon_path):
+            raise IOError('Add-on path does not exist: %s' % addon_path)
+
+        try:
+            if zipfile.is_zipfile(addon_path):
+                # Bug 944361 - We cannot use 'with' together with zipFile because
+                # it will cause an exception thrown in Python 2.6.
+                try:
+                    compressed_file = zipfile.ZipFile(addon_path, 'r')
+                    manifest = compressed_file.read('install.rdf')
+                finally:
+                    compressed_file.close()
+            elif os.path.isdir(addon_path):
+                with open(os.path.join(addon_path, 'install.rdf'), 'r') as f:
+                    manifest = f.read()
+            else:
+                raise IOError('Add-on path is neither an XPI nor a directory: %s' % addon_path)
+        except (IOError, KeyError) as e:
+            raise AddonFormatError(str(e)), None, sys.exc_info()[2]
+
+        try:
+            doc = minidom.parseString(manifest)
+
+            # Get the namespaces abbreviations
+            em = get_namespace_id(doc, 'http://www.mozilla.org/2004/em-rdf#')
+            rdf = get_namespace_id(doc, 'http://www.w3.org/1999/02/22-rdf-syntax-ns#')
+
+            description = doc.getElementsByTagName(rdf + 'Description').item(0)
+            for entry, value in description.attributes.items():
+                # Remove the namespace prefix from the tag for comparison
+                entry = entry.replace(em, "")
+                if entry in details.keys():
+                    details.update({entry: value})
+            for node in description.childNodes:
+                # Remove the namespace prefix from the tag for comparison
+                entry = node.nodeName.replace(em, "")
+                if entry in details.keys():
+                    details.update({entry: get_text(node)})
+        except Exception as e:
+            raise AddonFormatError(str(e)), None, sys.exc_info()[2]
+
+        # turn unpack into a true/false value
+        if isinstance(details['unpack'], basestring):
+            details['unpack'] = details['unpack'].lower() == 'true'
+
+        # If no ID is set, the add-on is invalid
+        if details.get('id') is None:
+            raise AddonFormatError('Add-on id could not be found.')
+
+        return details
+
+    def install_from_path(self, path, unpack=False):
+        """
+        Installs addon from a filepath, url or directory of addons in the profile.
+
+        :param path: url, path to .xpi, or directory of addons
+        :param unpack: whether to unpack unless specified otherwise in the install.rdf
+        """
+
+        # if the addon is a URL, download it
+        # note that this won't work with protocols urllib2 doesn't support
+        if mozfile.is_url(path):
+            path = self.download(path)
+            self.downloaded_addons.append(path)
+
+        addons = [path]
+
+        # if path is not an add-on, try to install all contained add-ons
+        try:
+            self.addon_details(path)
+        except AddonFormatError as e:
+            module_logger.warning('Could not install %s: %s' % (path, str(e)))
+
+            # If the path doesn't exist, then we don't really care, just return
+            if not os.path.isdir(path):
+                return
+
+            addons = [os.path.join(path, x) for x in os.listdir(path) if
+                      self.is_addon(os.path.join(path, x))]
+            addons.sort()
+
+        # install each addon
+        for addon in addons:
+            # determine the addon id
+            addon_details = self.addon_details(addon)
+            addon_id = addon_details.get('id')
+
+            # if the add-on has to be unpacked force it now
+            # note: we might want to let Firefox do it in case of addon details
+            orig_path = None
+            if os.path.isfile(addon) and (unpack or addon_details['unpack']):
+                orig_path = addon
+                addon = tempfile.mkdtemp()
+                mozfile.extract(orig_path, addon)
+
+            # copy the addon to the profile
+            extensions_path = os.path.join(self.profile, 'extensions', 'staged')
+            addon_path = os.path.join(extensions_path, addon_id)
+
+            if os.path.isfile(addon):
+                addon_path += '.xpi'
+
+                # move existing xpi file to backup location to restore later
+                if os.path.exists(addon_path):
+                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
+                    shutil.move(addon_path, self.backup_dir)
+
+                # copy new add-on to the extension folder
+                if not os.path.exists(extensions_path):
+                    os.makedirs(extensions_path)
+                shutil.copy(addon, addon_path)
+            else:
+                # move existing folder to backup location to restore later
+                if os.path.exists(addon_path):
+                    self.backup_dir = self.backup_dir or tempfile.mkdtemp()
+                    shutil.move(addon_path, self.backup_dir)
+
+                # copy new add-on to the extension folder
+                shutil.copytree(addon, addon_path, symlinks=True)
+
+            # if we had to extract the addon, remove the temporary directory
+            if orig_path:
+                mozfile.remove(addon)
+                addon = orig_path
+
+            self._addons.append(addon_id)
+            self.installed_addons.append(addon)
+
+    def remove_addon(self, addon_id):
+        """Remove the add-on as specified by the id
+
+        :param addon_id: id of the add-on to be removed
+        """
+        path = self.get_addon_path(addon_id)
+        mozfile.remove(path)

testing/mozbase/mozprofile/mozprofile/cli.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Creates and/or modifies a Firefox profile.
+The profile can be modified by passing in addons to install or preferences to set.
+If no profile is specified, a new profile is created and the path of the
+resulting profile is printed.
+"""
+
+import sys
+from optparse import OptionParser
+from prefs import Preferences
+from profile import FirefoxProfile
+from profile import Profile
+
+__all__ = ['MozProfileCLI', 'cli']
+
+
+class MozProfileCLI(object):
+    """The Command Line Interface for ``mozprofile``."""
+
+    module = 'mozprofile'
+    profile_class = Profile
+
+    def __init__(self, args=sys.argv[1:], add_options=None):
+        self.parser = OptionParser(description=__doc__)
+        self.add_options(self.parser)
+        if add_options:
+            add_options(self.parser)
+        (self.options, self.args) = self.parser.parse_args(args)
+
+    def add_options(self, parser):
+
+        parser.add_option("-p", "--profile", dest="profile",
+                          help="The path to the profile to operate on. "
+                          "If none, creates a new profile in temp directory")
+        parser.add_option("-a", "--addon", dest="addons",
+                          action="append", default=[],
+                          help="Addon paths to install. Can be a filepath, "
+                          "a directory containing addons, or a url")
+        parser.add_option("--addon-manifests", dest="addon_manifests",
+                          action="append",
+                          help="An addon manifest to install")
+        parser.add_option("--pref", dest="prefs",
+                          action='append', default=[],
+                          help="A preference to set. "
+                          "Must be a key-value pair separated by a ':'")
+        parser.add_option("--preferences", dest="prefs_files",
+                          action='append', default=[],
+                          metavar="FILE",
+                          help="read preferences from a JSON or INI file. "
+                          "For INI, use 'file.ini:section' to specify a particular section.")
+
+    def profile_args(self):
+        """arguments to instantiate the profile class"""
+        return dict(profile=self.options.profile,
+                    addons=self.options.addons,
+                    addon_manifests=self.options.addon_manifests,
+                    preferences=self.preferences())
+
+    def preferences(self):
+        """profile preferences"""
+
+        # object to hold preferences
+        prefs = Preferences()
+
+        # add preferences files
+        for prefs_file in self.options.prefs_files:
+            prefs.add_file(prefs_file)
+
+        # change CLI preferences into 2-tuples
+        separator = ':'
+        cli_prefs = []
+        for pref in self.options.prefs:
+            if separator not in pref:
+                self.parser.error("Preference must be a key-value pair separated by "
+                                  "a ':' (You gave: %s)" % pref)
+            cli_prefs.append(pref.split(separator, 1))
+
+        # string preferences
+        prefs.add(cli_prefs, cast=True)
+
+        return prefs()
+
+    def profile(self, restore=False):
+        """create the profile"""
+
+        kwargs = self.profile_args()
+        kwargs['restore'] = restore
+        return self.profile_class(**kwargs)
+
+
+def cli(args=sys.argv[1:]):
+    """ Handles the command line arguments for ``mozprofile`` via ``sys.argv``"""
+
+    # add a view method for this cli method only
+    def add_options(parser):
+        parser.add_option('--view', dest='view',
+                          action='store_true', default=False,
+                          help="view summary of profile following invocation")
+        parser.add_option('--firefox', dest='firefox_profile',
+                          action='store_true', default=False,
+                          help="use FirefoxProfile defaults")
+
+    # process the command line
+    cli = MozProfileCLI(args, add_options)
+
+    if cli.args:
+        cli.parser.error("Program doesn't support positional arguments.")
+
+    if cli.options.firefox_profile:
+        cli.profile_class = FirefoxProfile
+
+    # create the profile
+    profile = cli.profile()
+
+    if cli.options.view:
+        # view the profile, if specified
+        print profile.summary()
+        return
+
+    # if no profile was passed in print the newly created profile
+    if not cli.options.profile:
+        print profile.profile
+
+if __name__ == '__main__':
+    cli()

testing/mozbase/mozprofile/mozprofile/diff.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+"""
+diff two profile summaries
+"""
+
+import difflib
+import profile
+import optparse
+import os
+import sys
+
+__all__ = ['diff', 'diff_profiles']
+
+
+def diff(profile1, profile2, diff_function=difflib.unified_diff):
+
+    profiles = (profile1, profile2)
+    parts = {}
+    parts_dict = {}
+    for index in (0, 1):
+        prof = profiles[index]
+
+        # first part, the path, isn't useful for diffing
+        parts[index] = prof.summary(return_parts=True)[1:]
+
+        parts_dict[index] = dict(parts[index])
+
+    # keys the first profile is missing
+    first_missing = [i for i in parts_dict[1]
+                     if i not in parts_dict[0]]
+    parts[0].extend([(i, '') for i in first_missing])
+
+    # diffs
+    retval = []
+    for key, value in parts[0]:
+        other = parts_dict[1].get(key, '')
+        value = value.strip()
+        other = other.strip()
+
+        if key == 'Files':
+            # first line of files is the path; we don't care to diff that
+            value = '\n'.join(value.splitlines()[1:])
+            if other:
+                other = '\n'.join(other.splitlines()[1:])
+
+        value = value.splitlines()
+        other = other.splitlines()
+        section_diff = list(diff_function(value, other, profile1.profile, profile2.profile))
+        if section_diff:
+            retval.append((key, '\n'.join(section_diff)))
+
+    return retval
+
+
+def diff_profiles(args=sys.argv[1:]):
+
+    # parse command line
+    usage = '%prog [options] profile1 profile2'
+    parser = optparse.OptionParser(usage=usage, description=__doc__)
+    options, args = parser.parse_args(args)
+    if len(args) != 2:
+        parser.error("Must give two profile paths")
+    missing = [arg for arg in args if not os.path.exists(arg)]
+    if missing:
+        parser.error("Profile not found: %s" % (', '.join(missing)))
+
+    # get the profile differences
+    diffs = diff(*([profile.Profile(arg)
+                    for arg in args]))
+
+    # display them
+    while diffs:
+        key, value = diffs.pop(0)
+        print '[%s]:\n' % key
+        print value
+        if diffs:
+            print '-' * 4
+
+if __name__ == '__main__':
+    diff_profiles()

testing/mozbase/mozprofile/mozprofile/permissions.py

@@ -0,0 +1,415 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+"""
+add permissions to the profile
+"""
+
+import codecs
+import os
+import sqlite3
+import urlparse
+
+__all__ = ['MissingPrimaryLocationError', 'MultiplePrimaryLocationsError',
+           'DEFAULT_PORTS', 'DuplicateLocationError', 'BadPortLocationError',
+           'LocationsSyntaxError', 'Location', 'ServerLocations',
+           'Permissions']
+
+# http://hg.mozilla.org/mozilla-central/file/b871dfb2186f/build/automation.py.in#l28
+DEFAULT_PORTS = {'http': '8888',
+                 'https': '4443',
+                 'ws': '4443',
+                 'wss': '4443'}
+
+
+class LocationError(Exception):
+    """Signifies an improperly formed location."""
+
+    def __str__(self):
+        s = "Bad location"
+        if self.message:
+            s += ": %s" % self.message
+        return s
+
+
+class MissingPrimaryLocationError(LocationError):
+    """No primary location defined in locations file."""
+
+    def __init__(self):
+        LocationError.__init__(self, "missing primary location")
+
+
+class MultiplePrimaryLocationsError(LocationError):
+    """More than one primary location defined."""
+
+    def __init__(self):
+        LocationError.__init__(self, "multiple primary locations")
+
+
+class DuplicateLocationError(LocationError):
+    """Same location defined twice."""
+
+    def __init__(self, url):
+        LocationError.__init__(self, "duplicate location: %s" % url)
+
+
+class BadPortLocationError(LocationError):
+    """Location has invalid port value."""
+
+    def __init__(self, given_port):
+        LocationError.__init__(self, "bad value for port: %s" % given_port)
+
+
+class LocationsSyntaxError(Exception):
+    """Signifies a syntax error on a particular line in server-locations.txt."""
+
+    def __init__(self, lineno, err=None):
+        self.err = err
+        self.lineno = lineno
+
+    def __str__(self):
+        s = "Syntax error on line %s" % self.lineno
+        if self.err:
+            s += ": %s." % self.err
+        else:
+            s += "."
+        return s
+
+
+class Location(object):
+    """Represents a location line in server-locations.txt."""
+
+    attrs = ('scheme', 'host', 'port')
+
+    def __init__(self, scheme, host, port, options):
+        for attr in self.attrs:
+            setattr(self, attr, locals()[attr])
+        self.options = options
+        try:
+            int(self.port)
+        except ValueError:
+            raise BadPortLocationError(self.port)
+
+    def isEqual(self, location):
+        """compare scheme://host:port, but ignore options"""
+        return len([i for i in self.attrs
+                    if getattr(self, i) == getattr(location, i)]) == len(self.attrs)
+
+    __eq__ = isEqual
+
+    def url(self):
+        return '%s://%s:%s' % (self.scheme, self.host, self.port)
+
+    def __str__(self):
+        return '%s  %s' % (self.url(), ','.join(self.options))
+
+
+class ServerLocations(object):
+    """Iterable collection of locations.
+    Use provided functions to add new locations, rather that manipulating
+    _locations directly, in order to check for errors and to ensure the
+    callback is called, if given.
+    """
+
+    def __init__(self, filename=None, add_callback=None):
+        self.add_callback = add_callback
+        self._locations = []
+        self.hasPrimary = False
+        if filename:
+            self.read(filename)
+
+    def __iter__(self):
+        return self._locations.__iter__()
+
+    def __len__(self):
+        return len(self._locations)
+
+    def add(self, location, suppress_callback=False):
+        if "primary" in location.options:
+            if self.hasPrimary:
+                raise MultiplePrimaryLocationsError()
+            self.hasPrimary = True
+
+        self._locations.append(location)
+        if self.add_callback and not suppress_callback:
+            self.add_callback([location])
+
+    def add_host(self, host, port='80', scheme='http', options='privileged'):
+        if isinstance(options, basestring):
+            options = options.split(',')
+        self.add(Location(scheme, host, port, options))
+
+    def read(self, filename, check_for_primary=True):
+        """
+        Reads the file and adds all valid locations to the ``self._locations`` array.
+
+        :param filename: in the format of server-locations.txt_
+        :param check_for_primary: if True, a ``MissingPrimaryLocationError`` exception is raised
+          if no primary is found
+
+        .. _server-locations.txt: http://dxr.mozilla.org/mozilla-central/source/build/pgo/server-locations.txt # noqa
+
+        The only exception is that the port, if not defined, defaults to 80 or 443.
+
+        FIXME: Shouldn't this default to the protocol-appropriate port?  Is
+        there any reason to have defaults at all?
+        """
+
+        locationFile = codecs.open(filename, "r", "UTF-8")
+        lineno = 0
+        new_locations = []
+
+        for line in locationFile:
+            line = line.strip()
+            lineno += 1
+
+            # check for comments and blank lines
+            if line.startswith("#") or not line:
+                continue
+
+            # split the server from the options
+            try:
+                server, options = line.rsplit(None, 1)
+                options = options.split(',')
+            except ValueError:
+                server = line
+                options = []
+
+            # parse the server url
+            if '://' not in server:
+                server = 'http://' + server
+            scheme, netloc, path, query, fragment = urlparse.urlsplit(server)
+            # get the host and port
+            try:
+                host, port = netloc.rsplit(':', 1)
+            except ValueError:
+                host = netloc
+                port = DEFAULT_PORTS.get(scheme, '80')
+
+            try:
+                location = Location(scheme, host, port, options)
+                self.add(location, suppress_callback=True)
+            except LocationError as e:
+                raise LocationsSyntaxError(lineno, e)
+
+            new_locations.append(location)
+
+        # ensure that a primary is found
+        if check_for_primary and not self.hasPrimary:
+            raise LocationsSyntaxError(lineno + 1,
+                                       MissingPrimaryLocationError())
+
+        if self.add_callback:
+            self.add_callback(new_locations)
+
+
+class Permissions(object):
+    """Allows handling of permissions for ``mozprofile``"""
+
+    def __init__(self, profileDir, locations=None):
+        self._profileDir = profileDir
+        self._locations = ServerLocations(add_callback=self.write_db)
+        if locations:
+            if isinstance(locations, ServerLocations):
+                self._locations = locations
+                self._locations.add_callback = self.write_db
+                self.write_db(self._locations._locations)
+            elif isinstance(locations, list):
+                for l in locations:
+                    self._locations.add_host(**l)
+            elif isinstance(locations, dict):
+                self._locations.add_host(**locations)
+            elif os.path.exists(locations):
+                self._locations.read(locations)
+
+    def write_db(self, locations):
+        """write permissions to the sqlite database"""
+
+        # Open database and create table
+        permDB = sqlite3.connect(os.path.join(self._profileDir, "permissions.sqlite"))
+        cursor = permDB.cursor()
+
+        # SQL copied from
+        # http://dxr.mozilla.org/mozilla-central/source/extensions/cookie/nsPermissionManager.cpp
+        cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
+              id INTEGER PRIMARY KEY
+             ,origin TEXT
+             ,type TEXT
+             ,permission INTEGER
+             ,expireType INTEGER
+             ,expireTime INTEGER
+             ,modificationTime INTEGER
+           )""")
+
+        rows = cursor.execute("PRAGMA table_info(moz_hosts)")
+        count = len(rows.fetchall())
+
+        using_origin = False
+        # if the db contains 7 columns, we're using user_version 5
+        if count == 7:
+            statement = "INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0)"
+            cursor.execute("PRAGMA user_version=5;")
+            using_origin = True
+        # if the db contains 9 columns, we're using user_version 4
+        elif count == 9:
+            statement = "INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0, 0)"
+            cursor.execute("PRAGMA user_version=4;")
+        # if the db contains 8 columns, we're using user_version 3
+        elif count == 8:
+            statement = "INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0, 0, 0)"
+            cursor.execute("PRAGMA user_version=3;")
+        else:
+            statement = "INSERT INTO moz_hosts values(NULL, ?, ?, ?, 0, 0)"
+            cursor.execute("PRAGMA user_version=2;")
+
+        for location in locations:
+            # set the permissions
+            permissions = {'allowXULXBL': 'noxul' not in location.options}
+            for perm, allow in permissions.iteritems():
+                if allow:
+                    permission_type = 1
+                else:
+                    permission_type = 2
+
+                if using_origin:
+                    # This is a crude approximation of the origin generation logic from
+                    # nsPrincipal and nsStandardURL. It should suffice for the permissions
+                    # which the test runners will want to insert into the system.
+                    origin = location.scheme + "://" + location.host
+                    if (location.scheme != 'http' or location.port != '80') and \
+                       (location.scheme != 'https' or location.port != '443'):
+                        origin += ':' + str(location.port)
+
+                    cursor.execute(statement,
+                                   (origin, perm, permission_type))
+                else:
+                    # The database is still using a legacy system based on hosts
+                    # We can insert the permission as a host
+                    #
+                    # XXX This codepath should not be hit, as tests are run with
+                    # fresh profiles. However, if it was hit, permissions would
+                    # not be added to the database correctly (bug 1183185).
+                    cursor.execute(statement,
+                                   (location.host, perm, permission_type))
+
+        # Commit and close
+        permDB.commit()
+        cursor.close()
+
+    def network_prefs(self, proxy=None):
+        """
+        take known locations and generate preferences to handle permissions and proxy
+        returns a tuple of prefs, user_prefs
+        """
+
+        prefs = []
+
+        if proxy:
+            user_prefs = self.pac_prefs(proxy)
+        else:
+            user_prefs = []
+
+        return prefs, user_prefs
+
+    def pac_prefs(self, user_proxy=None):
+        """
+        return preferences for Proxy Auto Config. originally taken from
+        http://dxr.mozilla.org/mozilla-central/source/build/automation.py.in
+        """
+        proxy = DEFAULT_PORTS.copy()
+
+        # We need to proxy every server but the primary one.
+        origins = ["'%s'" % l.url()
+                   for l in self._locations]
+        origins = ", ".join(origins)
+        proxy["origins"] = origins
+
+        for l in self._locations:
+            if "primary" in l.options:
+                proxy["remote"] = l.host
+                proxy[l.scheme] = l.port
+
+        # overwrite defaults with user specified proxy
+        if isinstance(user_proxy, dict):
+            proxy.update(user_proxy)
+
+        # TODO: this should live in a template!
+        # If you must escape things in this string with backslashes, be aware
+        # of the multiple layers of escaping at work:
+        #
+        # - Python will unescape backslashes;
+        # - Writing out the prefs will escape things via JSON serialization;
+        # - The prefs file reader will unescape backslashes;
+        # - The JS engine parser will unescape backslashes.
+        pacURL = """data:text/plain,
+var knownOrigins = (function () {
+  return [%(origins)s].reduce(function(t, h) { t[h] = true; return t; }, {})
+})();
+var uriRegex = new RegExp('^([a-z][-a-z0-9+.]*)' +
+                          '://' +
+                          '(?:[^/@]*@)?' +
+                          '(.*?)' +
+                          '(?::(\\\\d+))?/');
+var defaultPortsForScheme = {
+  'http': 80,
+  'ws': 80,
+  'https': 443,
+  'wss': 443
+};
+var originSchemesRemap = {
+  'ws': 'http',
+  'wss': 'https'
+};
+var proxyForScheme = {
+  'http': 'PROXY %(remote)s:%(http)s',
+  'https': 'PROXY %(remote)s:%(https)s',
+  'ws': 'PROXY %(remote)s:%(ws)s',
+  'wss': 'PROXY %(remote)s:%(wss)s'
+};
+
+function FindProxyForURL(url, host)
+{
+  var matches = uriRegex.exec(url);
+  if (!matches)
+    return 'DIRECT';
+  var originalScheme = matches[1];
+  var host = matches[2];
+  var port = matches[3];
+  if (!port && originalScheme in defaultPortsForScheme) {
+    port = defaultPortsForScheme[originalScheme];
+  }
+  var schemeForOriginChecking = originSchemesRemap[originalScheme] || originalScheme;
+
+  var origin = schemeForOriginChecking + '://' + host + ':' + port;
+  if (!(origin in knownOrigins))
+    return 'DIRECT';
+  return proxyForScheme[originalScheme] || 'DIRECT';
+}""" % proxy
+        pacURL = "".join(pacURL.splitlines())
+
+        prefs = []
+        prefs.append(("network.proxy.type", 2))
+        prefs.append(("network.proxy.autoconfig_url", pacURL))
+
+        return prefs
+
+    def clean_db(self):
+        """Removed permissions added by mozprofile."""
+
+        sqlite_file = os.path.join(self._profileDir, "permissions.sqlite")
+        if not os.path.exists(sqlite_file):
+            return
+
+        # Open database and create table
+        permDB = sqlite3.connect(sqlite_file)
+        cursor = permDB.cursor()
+
+        # TODO: only delete values that we add, this would require sending
+        # in the full permissions object
+        cursor.execute("DROP TABLE IF EXISTS moz_hosts")
+
+        # Commit and close
+        permDB.commit()
+        cursor.close()

testing/mozbase/mozprofile/mozprofile/prefs.py

@@ -0,0 +1,232 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+user preferences
+"""
+
+import json
+import mozfile
+import os
+import tokenize
+from ConfigParser import SafeConfigParser as ConfigParser
+from StringIO import StringIO
+
+__all__ = ('PreferencesReadError', 'Preferences')
+
+
+class PreferencesReadError(Exception):
+    """read error for prefrences files"""
+
+
+class Preferences(object):
+    """assembly of preferences from various sources"""
+
+    def __init__(self, prefs=None):
+        self._prefs = []
+        if prefs:
+            self.add(prefs)
+
+    def add(self, prefs, cast=False):
+        """
+        :param prefs:
+        :param cast: whether to cast strings to value, e.g. '1' -> 1
+        """
+        # wants a list of 2-tuples
+        if isinstance(prefs, dict):
+            prefs = prefs.items()
+        if cast:
+            prefs = [(i, self.cast(j)) for i, j in prefs]
+        self._prefs += prefs
+
+    def add_file(self, path):
+        """a preferences from a file
+
+        :param path:
+        """
+        self.add(self.read(path))
+
+    def __call__(self):
+        return self._prefs
+
+    @classmethod
+    def cast(cls, value):
+        """
+        interpolate a preference from a string
+        from the command line or from e.g. an .ini file, there is no good way to denote
+        what type the preference value is, as natively it is a string
+
+        - integers will get cast to integers
+        - true/false will get cast to True/False
+        - anything enclosed in single quotes will be treated as a string
+          with the ''s removed from both sides
+        """
+
+        if not isinstance(value, basestring):
+            return value  # no op
+        quote = "'"
+        if value == 'true':
+            return True
+        if value == 'false':
+            return False
+        try:
+            return int(value)
+        except ValueError:
+            pass
+        if value.startswith(quote) and value.endswith(quote):
+            value = value[1:-1]
+        return value
+
+    @classmethod
+    def read(cls, path):
+        """read preferences from a file"""
+
+        section = None  # for .ini files
+        basename = os.path.basename(path)
+        if ':' in basename:
+            # section of INI file
+            path, section = path.rsplit(':', 1)
+
+        if not os.path.exists(path) and not mozfile.is_url(path):
+            raise PreferencesReadError("'%s' does not exist" % path)
+
+        if section:
+            try:
+                return cls.read_ini(path, section)
+            except PreferencesReadError:
+                raise
+            except Exception as e:
+                raise PreferencesReadError(str(e))
+
+        # try both JSON and .ini format
+        try:
+            return cls.read_json(path)
+        except Exception as e:
+            try:
+                return cls.read_ini(path)
+            except Exception as f:
+                for exception in e, f:
+                    if isinstance(exception, PreferencesReadError):
+                        raise exception
+                raise PreferencesReadError("Could not recognize format of %s" % path)
+
+    @classmethod
+    def read_ini(cls, path, section=None):
+        """read preferences from an .ini file"""
+
+        parser = ConfigParser()
+        parser.optionxform = str
+        parser.readfp(mozfile.load(path))
+
+        if section:
+            if section not in parser.sections():
+                raise PreferencesReadError("No section '%s' in %s" % (section, path))
+            retval = parser.items(section, raw=True)
+        else:
+            retval = parser.defaults().items()
+
+        # cast the preferences since .ini is just strings
+        return [(i, cls.cast(j)) for i, j in retval]
+
+    @classmethod
+    def read_json(cls, path):
+        """read preferences from a JSON blob"""
+
+        prefs = json.loads(mozfile.load(path).read())
+
+        if type(prefs) not in [list, dict]:
+            raise PreferencesReadError("Malformed preferences: %s" % path)
+        if isinstance(prefs, list):
+            if [i for i in prefs if type(i) != list or len(i) != 2]:
+                raise PreferencesReadError("Malformed preferences: %s" % path)
+            values = [i[1] for i in prefs]
+        elif isinstance(prefs, dict):
+            values = prefs.values()
+        else:
+            raise PreferencesReadError("Malformed preferences: %s" % path)
+        types = (bool, basestring, int)
+        if [i for i in values if not [isinstance(i, j) for j in types]]:
+            raise PreferencesReadError("Only bool, string, and int values allowed")
+        return prefs
+
+    @classmethod
+    def read_prefs(cls, path, pref_setter='user_pref', interpolation=None):
+        """
+        Read preferences from (e.g.) prefs.js
+
+        :param path: The path to the preference file to read.
+        :param pref_setter: The name of the function used to set preferences
+                            in the preference file.
+        :param interpolation: If provided, a dict that will be passed
+                              to str.format to interpolate preference values.
+        """
+
+        marker = '##//'  # magical marker
+        lines = [i.strip() for i in mozfile.load(path).readlines()]
+        _lines = []
+        for line in lines:
+            if not line.startswith(pref_setter):
+                continue
+            if '//' in line:
+                line = line.replace('//', marker)
+            _lines.append(line)
+        string = '\n'.join(_lines)
+
+        # skip trailing comments
+        processed_tokens = []
+        f_obj = StringIO(string)
+        for token in tokenize.generate_tokens(f_obj.readline):
+            if token[0] == tokenize.COMMENT:
+                continue
+            processed_tokens.append(token[:2])  # [:2] gets around http://bugs.python.org/issue9974
+        string = tokenize.untokenize(processed_tokens)
+
+        retval = []
+
+        def pref(a, b):
+            if interpolation and isinstance(b, basestring):
+                b = b.format(**interpolation)
+            retval.append((a, b))
+        lines = [i.strip().rstrip(';') for i in string.split('\n') if i.strip()]
+
+        _globals = {'retval': retval, 'true': True, 'false': False}
+        _globals[pref_setter] = pref
+        for line in lines:
+            try:
+                eval(line, _globals, {})
+            except SyntaxError:
+                print line
+                raise
+
+        # de-magic the marker
+        for index, (key, value) in enumerate(retval):
+            if isinstance(value, basestring) and marker in value:
+                retval[index] = (key, value.replace(marker, '//'))
+
+        return retval
+
+    @classmethod
+    def write(cls, _file, prefs, pref_string='user_pref(%s, %s);'):
+        """write preferences to a file"""
+
+        if isinstance(_file, basestring):
+            f = file(_file, 'a')
+        else:
+            f = _file
+
+        if isinstance(prefs, dict):
+            # order doesn't matter
+            prefs = prefs.items()
+
+        # serialize -> JSON
+        _prefs = [(json.dumps(k), json.dumps(v))
+                  for k, v in prefs]
+
+        # write the preferences
+        for _pref in _prefs:
+            print >> f, pref_string % _pref
+
+        # close the file if opened internally
+        if isinstance(_file, basestring):
+            f.close()

testing/mozbase/mozprofile/mozprofile/profile.py

@@ -0,0 +1,454 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import time
+import tempfile
+import uuid
+
+from addons import AddonManager
+import mozfile
+from permissions import Permissions
+from prefs import Preferences
+from shutil import copytree
+from webapps import WebappCollection
+
+__all__ = ['Profile',
+           'FirefoxProfile',
+           'MetroFirefoxProfile',
+           'ThunderbirdProfile']
+
+
+class Profile(object):
+    """Handles all operations regarding profile.
+
+    Creating new profiles, installing add-ons, setting preferences and
+    handling cleanup.
+
+    The files associated with the profile will be removed automatically after
+    the object is garbage collected: ::
+
+      profile = Profile()
+      print profile.profile  # this is the path to the created profile
+      del profile
+      # the profile path has been removed from disk
+
+    :meth:`cleanup` is called under the hood to remove the profile files. You
+    can ensure this method is called (even in the case of exception) by using
+    the profile as a context manager: ::
+
+      with Profile() as profile:
+          # do things with the profile
+          pass
+      # profile.cleanup() has been called here
+    """
+
+    def __init__(self, profile=None, addons=None, addon_manifests=None, apps=None,
+                 preferences=None, locations=None, proxy=None, restore=True):
+        """
+        :param profile: Path to the profile
+        :param addons: String of one or list of addons to install
+        :param addon_manifests: Manifest for addons (see http://bit.ly/17jQ7i6)
+        :param apps: Dictionary or class of webapps to install
+        :param preferences: Dictionary or class of preferences
+        :param locations: ServerLocations object
+        :param proxy: Setup a proxy
+        :param restore: Flag for removing all custom settings during cleanup
+        """
+        self._addons = addons
+        self._addon_manifests = addon_manifests
+        self._apps = apps
+        self._locations = locations
+        self._proxy = proxy
+
+        # Prepare additional preferences
+        if preferences:
+            if isinstance(preferences, dict):
+                # unordered
+                preferences = preferences.items()
+
+            # sanity check
+            assert not [i for i in preferences if len(i) != 2]
+        else:
+            preferences = []
+        self._preferences = preferences
+
+        # Handle profile creation
+        self.create_new = not profile
+        if profile:
+            # Ensure we have a full path to the profile
+            self.profile = os.path.abspath(os.path.expanduser(profile))
+        else:
+            self.profile = tempfile.mkdtemp(suffix='.mozrunner')
+
+        self.restore = restore
+
+        # Initialize all class members
+        self._internal_init()
+
+    def _internal_init(self):
+        """Internal: Initialize all class members to their default value"""
+
+        if not os.path.exists(self.profile):
+            os.makedirs(self.profile)
+
+        # Preferences files written to
+        self.written_prefs = set()
+
+        # Our magic markers
+        nonce = '%s %s' % (str(time.time()), uuid.uuid4())
+        self.delimeters = ('#MozRunner Prefs Start %s' % nonce,
+                           '#MozRunner Prefs End %s' % nonce)
+
+        # If sub-classes want to set default preferences
+        if hasattr(self.__class__, 'preferences'):
+            self.set_preferences(self.__class__.preferences)
+        # Set additional preferences
+        self.set_preferences(self._preferences)
+
+        self.permissions = Permissions(self.profile, self._locations)
+        prefs_js, user_js = self.permissions.network_prefs(self._proxy)
+        self.set_preferences(prefs_js, 'prefs.js')
+        self.set_preferences(user_js)
+
+        # handle add-on installation
+        self.addon_manager = AddonManager(self.profile, restore=self.restore)
+        self.addon_manager.install_addons(self._addons, self._addon_manifests)
+
+        # handle webapps
+        self.webapps = WebappCollection(profile=self.profile, apps=self._apps)
+        self.webapps.update_manifests()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.cleanup()
+
+    def __del__(self):
+        self.cleanup()
+
+    # cleanup
+
+    def cleanup(self):
+        """Cleanup operations for the profile."""
+
+        if self.restore:
+            # If copies of those class instances exist ensure we correctly
+            # reset them all (see bug 934484)
+            self.clean_preferences()
+            if getattr(self, 'addon_manager', None) is not None:
+                self.addon_manager.clean()
+            if getattr(self, 'permissions', None) is not None:
+                self.permissions.clean_db()
+            if getattr(self, 'webapps', None) is not None:
+                self.webapps.clean()
+
+            # If it's a temporary profile we have to remove it
+            if self.create_new:
+                mozfile.remove(self.profile)
+
+    def reset(self):
+        """
+        reset the profile to the beginning state
+        """
+        self.cleanup()
+
+        self._internal_init()
+
+    def clean_preferences(self):
+        """Removed preferences added by mozrunner."""
+        for filename in self.written_prefs:
+            if not os.path.exists(os.path.join(self.profile, filename)):
+                # file has been deleted
+                break
+            while True:
+                if not self.pop_preferences(filename):
+                    break
+
+    @classmethod
+    def clone(cls, path_from, path_to=None, **kwargs):
+        """Instantiate a temporary profile via cloning
+        - path: path of the basis to clone
+        - kwargs: arguments to the profile constructor
+        """
+        if not path_to:
+            tempdir = tempfile.mkdtemp()  # need an unused temp dir name
+            mozfile.remove(tempdir)  # copytree requires that dest does not exist
+            path_to = tempdir
+        copytree(path_from, path_to)
+
+        c = cls(path_to, **kwargs)
+        c.create_new = True  # deletes a cloned profile when restore is True
+        return c
+
+    def exists(self):
+        """returns whether the profile exists or not"""
+        return os.path.exists(self.profile)
+
+    # methods for preferences
+
+    def set_preferences(self, preferences, filename='user.js'):
+        """Adds preferences dict to profile preferences"""
+
+        # append to the file
+        prefs_file = os.path.join(self.profile, filename)
+        f = open(prefs_file, 'a')
+
+        if preferences:
+
+            # note what files we've touched
+            self.written_prefs.add(filename)
+
+            # opening delimeter
+            f.write('\n%s\n' % self.delimeters[0])
+
+            # write the preferences
+            Preferences.write(f, preferences)
+
+            # closing delimeter
+            f.write('%s\n' % self.delimeters[1])
+
+        f.close()
+
+    def set_persistent_preferences(self, preferences):
+        """
+        Adds preferences dict to profile preferences and save them during a
+        profile reset
+        """
+
+        # this is a dict sometimes, convert
+        if isinstance(preferences, dict):
+            preferences = preferences.items()
+
+        # add new prefs to preserve them during reset
+        for new_pref in preferences:
+            # if dupe remove item from original list
+            self._preferences = [
+                pref for pref in self._preferences if not new_pref[0] == pref[0]]
+            self._preferences.append(new_pref)
+
+        self.set_preferences(preferences, filename='user.js')
+
+    def pop_preferences(self, filename):
+        """
+        pop the last set of preferences added
+        returns True if popped
+        """
+
+        path = os.path.join(self.profile, filename)
+        with file(path) as f:
+            lines = f.read().splitlines()
+
+        def last_index(_list, value):
+            """
+            returns the last index of an item;
+            this should actually be part of python code but it isn't
+            """
+            for index in reversed(range(len(_list))):
+                if _list[index] == value:
+                    return index
+        s = last_index(lines, self.delimeters[0])
+        e = last_index(lines, self.delimeters[1])
+
+        # ensure both markers are found
+        if s is None:
+            assert e is None, '%s found without %s' % (self.delimeters[1], self.delimeters[0])
+            return False  # no preferences found
+        elif e is None:
+            assert s is None, '%s found without %s' % (self.delimeters[0], self.delimeters[1])
+
+        # ensure the markers are in the proper order
+        assert e > s, '%s found at %s, while %s found at %s' % (self.delimeters[1], e,
+                                                                self.delimeters[0], s)
+
+        # write the prefs
+        cleaned_prefs = '\n'.join(lines[:s] + lines[e + 1:])
+        with file(path, 'w') as f:
+            f.write(cleaned_prefs)
+        return True
+
+    # methods for introspection
+
+    def summary(self, return_parts=False):
+        """
+        returns string summarizing profile information.
+        if return_parts is true, return the (Part_name, value) list
+        of tuples instead of the assembled string
+        """
+
+        parts = [('Path', self.profile)]  # profile path
+
+        # directory tree
+        parts.append(('Files', '\n%s' % mozfile.tree(self.profile)))
+
+        # preferences
+        for prefs_file in ('user.js', 'prefs.js'):
+            path = os.path.join(self.profile, prefs_file)
+            if os.path.exists(path):
+
+                # prefs that get their own section
+                # This is currently only 'network.proxy.autoconfig_url'
+                # but could be expanded to include others
+                section_prefs = ['network.proxy.autoconfig_url']
+                line_length = 80
+                # buffer for 80 character display:
+                # length = 80 - len(key) - len(': ') - line_length_buffer
+                line_length_buffer = 10
+                line_length_buffer += len(': ')
+
+                def format_value(key, value):
+                    if key not in section_prefs:
+                        return value
+                    max_length = line_length - len(key) - line_length_buffer
+                    if len(value) > max_length:
+                        value = '%s...' % value[:max_length]
+                    return value
+
+                prefs = Preferences.read_prefs(path)
+                if prefs:
+                    prefs = dict(prefs)
+                    parts.append((prefs_file,
+                                  '\n%s' % ('\n'.join(
+                                      ['%s: %s' % (key, format_value(key, prefs[key]))
+                                       for key in sorted(prefs.keys())]))))
+
+                    # Currently hardcorded to 'network.proxy.autoconfig_url'
+                    # but could be generalized, possibly with a generalized (simple)
+                    # JS-parser
+                    network_proxy_autoconfig = prefs.get('network.proxy.autoconfig_url')
+                    if network_proxy_autoconfig and network_proxy_autoconfig.strip():
+                        network_proxy_autoconfig = network_proxy_autoconfig.strip()
+                        lines = network_proxy_autoconfig.replace(';', ';\n').splitlines()
+                        lines = [line.strip() for line in lines]
+                        origins_string = 'var origins = ['
+                        origins_end = '];'
+                        if origins_string in lines[0]:
+                            start = lines[0].find(origins_string)
+                            end = lines[0].find(origins_end, start)
+                            splitline = [lines[0][:start],
+                                         lines[0][start:start + len(origins_string) - 1],
+                                         ]
+                            splitline.extend(lines[0][start + len(origins_string):end].replace(
+                                ',', ',\n').splitlines())
+                            splitline.append(lines[0][end:])
+                            lines[0:1] = [i.strip() for i in splitline]
+                        parts.append(('Network Proxy Autoconfig, %s' % (prefs_file),
+                                      '\n%s' % '\n'.join(lines)))
+
+        if return_parts:
+            return parts
+
+        retval = '%s\n' % ('\n\n'.join(['[%s]: %s' % (key, value)
+                                        for key, value in parts]))
+        return retval
+
+    __str__ = summary
+
+
+class FirefoxProfile(Profile):
+    """Specialized Profile subclass for Firefox"""
+
+    preferences = {  # Don't automatically update the application
+        'app.update.enabled': False,
+        # Don't restore the last open set of tabs if the browser has crashed
+        'browser.sessionstore.resume_from_crash': False,
+        # Don't check for the default web browser during startup
+        'browser.shell.checkDefaultBrowser': False,
+        # Don't warn on exit when multiple tabs are open
+        'browser.tabs.warnOnClose': False,
+        # Don't warn when exiting the browser
+        'browser.warnOnQuit': False,
+        # Don't send Firefox health reports to the production server
+        'datareporting.healthreport.documentServerURI': 'http://%(server)s/healthreport/',
+        # Only install add-ons from the profile and the application scope
+        # Also ensure that those are not getting disabled.
+        # see: https://developer.mozilla.org/en/Installing_extensions
+        'extensions.enabledScopes': 5,
+        'extensions.autoDisableScopes': 10,
+        # Don't send the list of installed addons to AMO
+        'extensions.getAddons.cache.enabled': False,
+        # Don't install distribution add-ons from the app folder
+        'extensions.installDistroAddons': False,
+        # Dont' run the add-on compatibility check during start-up
+        'extensions.showMismatchUI': False,
+        # Don't automatically update add-ons
+        'extensions.update.enabled': False,
+        # Don't open a dialog to show available add-on updates
+        'extensions.update.notifyUser': False,
+        # Enable test mode to run multiple tests in parallel
+        'focusmanager.testmode': True,
+        # Enable test mode to not raise an OS level dialog for location sharing
+        'geo.provider.testing': True,
+        # Suppress delay for main action in popup notifications
+        'security.notification_enable_delay': 0,
+        # Suppress automatic safe mode after crashes
+        'toolkit.startup.max_resumed_crashes': -1,
+        # Don't report telemetry information
+        'toolkit.telemetry.enabled': False,
+        # Don't send Telemetry reports to the production server. This is
+        # needed as Telemetry sends pings also if FHR upload is enabled.
+        'toolkit.telemetry.server': 'http://%(server)s/telemetry-dummy/',
+    }
+
+
+class MetroFirefoxProfile(Profile):
+    """Specialized Profile subclass for Firefox Metro"""
+
+    preferences = {  # Don't automatically update the application for desktop and metro build
+        'app.update.enabled': False,
+        'app.update.metro.enabled': False,
+        # Dismiss first run content overlay
+        'browser.firstrun-content.dismissed': True,
+        # Don't restore the last open set of tabs if the browser has crashed
+        'browser.sessionstore.resume_from_crash': False,
+        # Don't check for the default web browser during startup
+        'browser.shell.checkDefaultBrowser': False,
+        # Don't send Firefox health reports to the production server
+        'datareporting.healthreport.documentServerURI': 'http://%(server)s/healthreport/',
+        # Enable extensions
+        'extensions.defaultProviders.enabled': True,
+        # Only install add-ons from the profile and the application scope
+        # Also ensure that those are not getting disabled.
+        # see: https://developer.mozilla.org/en/Installing_extensions
+        'extensions.enabledScopes': 5,
+        'extensions.autoDisableScopes': 10,
+        # Don't send the list of installed addons to AMO
+        'extensions.getAddons.cache.enabled': False,
+        # Don't install distribution add-ons from the app folder
+        'extensions.installDistroAddons': False,
+        # Dont' run the add-on compatibility check during start-up
+        'extensions.showMismatchUI': False,
+        # Disable strict compatibility checks to allow add-ons enabled by default
+        'extensions.strictCompatibility': False,
+        # Don't automatically update add-ons
+        'extensions.update.enabled': False,
+        # Don't open a dialog to show available add-on updates
+        'extensions.update.notifyUser': False,
+        # Enable test mode to run multiple tests in parallel
+        'focusmanager.testmode': True,
+        # Suppress delay for main action in popup notifications
+        'security.notification_enable_delay': 0,
+        # Suppress automatic safe mode after crashes
+        'toolkit.startup.max_resumed_crashes': -1,
+        # Don't report telemetry information
+        'toolkit.telemetry.enabled': False,
+        # Don't send Telemetry reports to the production server. This is
+        # needed as Telemetry sends pings also if FHR upload is enabled.
+        'toolkit.telemetry.server': 'http://%(server)s/telemetry-dummy/',
+    }
+
+
+class ThunderbirdProfile(Profile):
+    """Specialized Profile subclass for Thunderbird"""
+
+    preferences = {'extensions.update.enabled': False,
+                   'extensions.update.notifyUser': False,
+                   'browser.shell.checkDefaultBrowser': False,
+                   'browser.tabs.warnOnClose': False,
+                   'browser.warnOnQuit': False,
+                   'browser.sessionstore.resume_from_crash': False,
+                   # prevents the 'new e-mail address' wizard on new profile
+                   'mail.provider.enabled': False,
+                   }

testing/mozbase/mozprofile/mozprofile/view.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+"""
+script to view mozilla profiles
+"""
+
+import mozprofile
+import optparse
+import os
+import sys
+
+__all__ = ['view_profile']
+
+
+def view_profile(args=sys.argv[1:]):
+
+    usage = '%prog [options] profile_path <...>'
+    parser = optparse.OptionParser(usage=usage, description=__doc__)
+    options, args = parser.parse_args(args)
+    if not args:
+        parser.print_usage()
+        parser.exit()
+
+    # check existence
+    missing = [i for i in args
+               if not os.path.exists(i)]
+    if missing:
+        if len(missing) > 1:
+            missing_string = "Profiles do not exist"
+        else:
+            missing_string = "Profile does not exist"
+        parser.error("%s: %s" % (missing_string, ', '.join(missing)))
+
+    # print summary for each profile
+    while args:
+        path = args.pop(0)
+        profile = mozprofile.Profile(path)
+        print profile.summary()
+        if args:
+            print '-' * 4
+
+if __name__ == '__main__':
+    view_profile()

testing/mozbase/mozprofile/mozprofile/webapps.py

@@ -0,0 +1,281 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Handles installing open webapps (https://developer.mozilla.org/en-US/docs/Apps)
+to a profile. A webapp object is a dict that contains some metadata about
+the webapp and must at least include a name, description and manifestURL.
+
+Each webapp has a manifest (https://developer.mozilla.org/en-US/docs/Apps/Manifest).
+Additionally there is a separate json manifest that keeps track of the installed
+webapps, their manifestURLs and their permissions.
+"""
+
+from string import Template
+import json
+import os
+import shutil
+
+import mozfile
+
+__all__ = ["Webapp", "WebappCollection", "WebappFormatException", "APP_STATUS_NOT_INSTALLED",
+           "APP_STATUS_INSTALLED", "APP_STATUS_PRIVILEGED", "APP_STATUS_CERTIFIED"]
+
+
+# from http://hg.mozilla.org/mozilla-central/file/add0b94c2c0b/caps/idl/nsIPrincipal.idl#l163
+APP_STATUS_NOT_INSTALLED = 0
+APP_STATUS_INSTALLED = 1
+APP_STATUS_PRIVILEGED = 2
+APP_STATUS_CERTIFIED = 3
+
+
+class WebappFormatException(Exception):
+    """thrown for invalid webapp objects"""
+
+
+class Webapp(dict):
+    """A webapp definition"""
+
+    required_keys = ('name', 'description', 'manifestURL')
+
+    def __init__(self, *args, **kwargs):
+        try:
+            dict.__init__(self, *args, **kwargs)
+        except (TypeError, ValueError):
+            raise WebappFormatException("Webapp object should be an instance of type 'dict'")
+        self.validate()
+
+    def __eq__(self, other):
+        """Webapps are considered equal if they have the same name"""
+        if not isinstance(other, self.__class__):
+            return False
+        return self['name'] == other['name']
+
+    def __ne__(self, other):
+        """Webapps are considered not equal if they have different names"""
+        return not self.__eq__(other)
+
+    def validate(self):
+        # TODO some keys are required if another key has a certain value
+        for key in self.required_keys:
+            if key not in self:
+                raise WebappFormatException("Webapp object missing required key '%s'" % key)
+
+
+class WebappCollection(object):
+    """A list-like object that collects webapps and updates the webapp manifests"""
+
+    json_template = Template(""""$name": {
+  "name": "$name",
+  "origin": "$origin",
+  "installOrigin": "$origin",
+  "receipt": null,
+  "installTime": 132333986000,
+  "manifestURL": "$manifestURL",
+  "localId": $localId,
+  "id": "$name",
+  "appStatus": $appStatus,
+  "csp": "$csp"
+}""")
+
+    manifest_template = Template("""{
+  "name": "$name",
+  "csp": "$csp",
+  "description": "$description",
+  "launch_path": "/",
+  "developer": {
+    "name": "Mozilla",
+    "url": "https://mozilla.org/"
+  },
+  "permissions": [
+  ],
+  "locales": {
+    "en-US": {
+      "name": "$name",
+      "description": "$description"
+    }
+  },
+  "default_locale": "en-US",
+  "icons": {
+  }
+}
+""")
+
+    def __init__(self, profile, apps=None, json_template=None, manifest_template=None):
+        """
+        :param profile: the file path to a profile
+        :param apps: [optional] a list of webapp objects or file paths to json files describing
+          webapps
+        :param json_template: [optional] string template describing the webapp json format
+        :param manifest_template: [optional] string template describing the webapp manifest format
+        """
+        if not isinstance(profile, basestring):
+            raise TypeError("Must provide path to a profile, received '%s'" % type(profile))
+        self.profile = profile
+        self.webapps_dir = os.path.join(self.profile, 'webapps')
+        self.backup_dir = os.path.join(self.profile, '.mozprofile_backup', 'webapps')
+
+        self._apps = []
+        self._installed_apps = []
+        if apps:
+            if not isinstance(apps, (list, set, tuple)):
+                apps = [apps]
+
+            for app in apps:
+                if isinstance(app, basestring) and os.path.isfile(app):
+                    self.extend(self.read_json(app))
+                else:
+                    self.append(app)
+
+        self.json_template = json_template or self.json_template
+        self.manifest_template = manifest_template or self.manifest_template
+
+    def __getitem__(self, index):
+        return self._apps.__getitem__(index)
+
+    def __setitem__(self, index, value):
+        return self._apps.__setitem__(index, Webapp(value))
+
+    def __delitem__(self, index):
+        return self._apps.__delitem__(index)
+
+    def __len__(self):
+        return self._apps.__len__()
+
+    def __contains__(self, value):
+        return self._apps.__contains__(Webapp(value))
+
+    def append(self, value):
+        return self._apps.append(Webapp(value))
+
+    def insert(self, index, value):
+        return self._apps.insert(index, Webapp(value))
+
+    def extend(self, values):
+        return self._apps.extend([Webapp(v) for v in values])
+
+    def remove(self, value):
+        return self._apps.remove(Webapp(value))
+
+    def _write_webapps_json(self, apps):
+        contents = []
+        for app in apps:
+            contents.append(self.json_template.substitute(app))
+        contents = '{\n' + ',\n'.join(contents) + '\n}\n'
+        webapps_json_path = os.path.join(self.webapps_dir, 'webapps.json')
+        webapps_json_file = open(webapps_json_path, "w")
+        webapps_json_file.write(contents)
+        webapps_json_file.close()
+
+    def _write_webapp_manifests(self, write_apps=[], remove_apps=[]):
+        # Write manifests for installed apps
+        for app in write_apps:
+            manifest_dir = os.path.join(self.webapps_dir, app['name'])
+            manifest_path = os.path.join(manifest_dir, 'manifest.webapp')
+            if not os.path.isfile(manifest_path):
+                if not os.path.isdir(manifest_dir):
+                    os.mkdir(manifest_dir)
+                manifest = self.manifest_template.substitute(app)
+                manifest_file = open(manifest_path, "a")
+                manifest_file.write(manifest)
+                manifest_file.close()
+        # Remove manifests for removed apps
+        for app in remove_apps:
+            self._installed_apps.remove(app)
+            manifest_dir = os.path.join(self.webapps_dir, app['name'])
+            mozfile.remove(manifest_dir)
+
+    def update_manifests(self):
+        """Updates the webapp manifests with the webapps represented in this collection
+
+        If update_manifests is called a subsequent time, there could have been apps added or
+        removed to the collection in the interim. The manifests will be adjusted accordingly
+        """
+        apps_to_install = [app for app in self._apps if app not in self._installed_apps]
+        apps_to_remove = [app for app in self._installed_apps if app not in self._apps]
+        if apps_to_install == apps_to_remove == []:
+            # nothing to do
+            return
+
+        if not os.path.isdir(self.webapps_dir):
+            os.makedirs(self.webapps_dir)
+        elif not self._installed_apps:
+            shutil.copytree(self.webapps_dir, self.backup_dir)
+
+        webapps_json_path = os.path.join(self.webapps_dir, 'webapps.json')
+        webapps_json = []
+        if os.path.isfile(webapps_json_path):
+            webapps_json = self.read_json(webapps_json_path, description="description")
+            webapps_json = [a for a in webapps_json if a not in apps_to_remove]
+
+        # Iterate over apps already in webapps.json to determine the starting local
+        # id and to ensure apps are properly formatted
+        start_id = 1
+        for local_id, app in enumerate(webapps_json):
+            app['localId'] = local_id + 1
+            start_id += 1
+            if not app.get('csp'):
+                app['csp'] = ''
+            if not app.get('appStatus'):
+                app['appStatus'] = 3
+
+        # Append apps_to_install to the pre-existent apps
+        for local_id, app in enumerate(apps_to_install):
+            app['localId'] = local_id + start_id
+            # ignore if it's already installed
+            if app in webapps_json:
+                start_id -= 1
+                continue
+            webapps_json.append(app)
+            self._installed_apps.append(app)
+
+        # Write the full contents to webapps.json
+        self._write_webapps_json(webapps_json)
+
+        # Create/remove manifest file for each app.
+        self._write_webapp_manifests(apps_to_install, apps_to_remove)
+
+    def clean(self):
+        """Remove all webapps that were installed and restore profile to previous state"""
+        if self._installed_apps:
+            mozfile.remove(self.webapps_dir)
+
+        if os.path.isdir(self.backup_dir):
+            shutil.copytree(self.backup_dir, self.webapps_dir)
+            mozfile.remove(self.backup_dir)
+
+        self._apps = []
+        self._installed_apps = []
+
+    @classmethod
+    def read_json(cls, path, **defaults):
+        """Reads a json file which describes a set of webapps. The json format is either a
+        dictionary where each key represents the name of a webapp (e.g B2G format) or a list
+        of webapp objects.
+
+        :param path: Path to a json file defining webapps
+        :param defaults: Default key value pairs added to each webapp object if key doesn't exist
+
+        Returns a list of Webapp objects
+        """
+        f = open(path, 'r')
+        app_json = json.load(f)
+        f.close()
+
+        apps = []
+        if isinstance(app_json, dict):
+            for k, v in app_json.iteritems():
+                v['name'] = k
+                apps.append(v)
+        else:
+            apps = app_json
+            if not isinstance(apps, list):
+                apps = [apps]
+
+        ret = []
+        for app in apps:
+            d = defaults.copy()
+            d.update(app)
+            ret.append(Webapp(**d))
+        return ret

testing/mozbase/mozprofile/setup.py

@@ -0,0 +1,45 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+from setuptools import setup
+
+PACKAGE_NAME = 'mozprofile'
+PACKAGE_VERSION = '0.28'
+
+# we only support python 2 right now
+assert sys.version_info[0] == 2
+
+deps = ['mozfile >= 1.0', 'mozlog >= 3.0']
+
+setup(name=PACKAGE_NAME,
+      version=PACKAGE_VERSION,
+      description="Library to create and modify Mozilla application profiles",
+      long_description="see http://mozbase.readthedocs.org/",
+      classifiers=['Environment :: Console',
+                   'Intended Audience :: Developers',
+                   'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
+                   'Natural Language :: English',
+                   'Operating System :: OS Independent',
+                   'Programming Language :: Python',
+                   'Topic :: Software Development :: Libraries :: Python Modules',
+                   ],
+      keywords='mozilla',
+      author='Mozilla Automation and Tools team',
+      author_email='tools@lists.mozilla.org',
+      url='https://wiki.mozilla.org/Auto-tools/Projects/Mozbase',
+      license='MPL 2.0',
+      packages=['mozprofile'],
+      include_package_data=True,
+      zip_safe=False,
+      install_requires=deps,
+      extras_require={'manifest': ['manifestparser >= 0.6']},
+      tests_require=['mozhttpd'],
+      entry_points="""
+      # -*- Entry points: -*-
+      [console_scripts]
+      mozprofile = mozprofile:cli
+      view-profile = mozprofile:view_profile
+      diff-profiles = mozprofile:diff_profiles
+      """, )

testing/mozbase/mozprofile/tests/addon_stubs.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+import os
+import tempfile
+import zipfile
+
+import mozfile
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+# stubs is a dict of the form {'addon id': 'install manifest content'}
+stubs = {
+    'test-addon-1@mozilla.org': 'test_addon_1.rdf',
+    'test-addon-2@mozilla.org': 'test_addon_2.rdf',
+    'test-addon-3@mozilla.org': 'test_addon_3.rdf',
+    'test-addon-4@mozilla.org': 'test_addon_4.rdf',
+    'test-addon-invalid-no-id@mozilla.org': 'test_addon_invalid_no_id.rdf',
+    'test-addon-invalid-version@mozilla.org': 'test_addon_invalid_version.rdf',
+    'test-addon-invalid-no-manifest@mozilla.org': None,
+    'test-addon-invalid-not-wellformed@mozilla.org': 'test_addon_invalid_not_wellformed.rdf',
+    'test-addon-unpack@mozilla.org': 'test_addon_unpack.rdf'}
+
+
+def generate_addon(addon_id, path=None, name=None, xpi=True):
+    """
+    Method to generate a single addon.
+
+    :param addon_id: id of an addon to generate from the stubs dictionary
+    :param path: path where addon and .xpi should be generated
+    :param name: name for the addon folder or .xpi file
+    :param xpi: Flag if an XPI or folder should be generated
+
+    Returns the file-path of the addon's .xpi file
+    """
+
+    if addon_id not in stubs.keys():
+        raise IOError('Requested addon stub "%s" does not exist' % addon_id)
+
+    # Generate directory structure for addon
+    try:
+        tmpdir = path or tempfile.mkdtemp()
+        addon_dir = os.path.join(tmpdir, name or addon_id)
+        os.mkdir(addon_dir)
+    except IOError:
+        raise IOError('Could not generate directory structure for addon stub.')
+
+    # Write install.rdf for addon
+    if stubs[addon_id]:
+        install_rdf = os.path.join(addon_dir, 'install.rdf')
+        with open(install_rdf, 'w') as f:
+            manifest = os.path.join(here, 'install_manifests', stubs[addon_id])
+            f.write(open(manifest, 'r').read())
+
+    if not xpi:
+        return addon_dir
+
+    # Generate the .xpi for the addon
+    xpi_file = os.path.join(tmpdir, (name or addon_id) + '.xpi')
+    with zipfile.ZipFile(xpi_file, 'w') as x:
+        x.write(install_rdf, install_rdf[len(addon_dir):])
+
+    # Ensure we remove the temporary folder to not install the addon twice
+    mozfile.rmtree(addon_dir)
+
+    return xpi_file
+
+
+def generate_manifest(addon_list, path=None):
+    tmpdir = path or tempfile.mkdtemp()
+    addons = [generate_addon(addon, path=tmpdir) for addon in addon_list]
+
+    manifest = os.path.join(tmpdir, 'manifest.ini')
+    with open(manifest, 'w') as f:
+        for addon in addons:
+            f.write('[' + addon + ']\n')
+
+    return manifest

testing/mozbase/mozprofile/tests/addonid.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+
+import os
+import tempfile
+import unittest
+import shutil
+from mozprofile import addons
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+class AddonIDTest(unittest.TestCase):
+    """ Test finding the addon id in a variety of install.rdf styles """
+
+    def make_install_rdf(self, filecontents):
+        path = tempfile.mkdtemp()
+        f = open(os.path.join(path, "install.rdf"), "w")
+        f.write(filecontents)
+        f.close()
+        return path
+
+    def test_addonID(self):
+        testlist = self.get_test_list()
+        for t in testlist:
+            try:
+                p = self.make_install_rdf(t)
+                a = addons.AddonManager(os.path.join(p, "profile"))
+                addon_id = a.addon_details(p)['id']
+                self.assertEqual(addon_id, "winning", "We got the addon id")
+            finally:
+                shutil.rmtree(p)
+
+    def test_addonID_xpi(self):
+        a = addons.AddonManager("profile")
+        addon = a.addon_details(os.path.join(here, "addons", "empty.xpi"))
+        self.assertEqual(addon['id'], "test-empty@quality.mozilla.org", "We got the addon id")
+
+    def get_test_list(self):
+        """ This just returns a hardcoded list of install.rdf snippets for testing.
+            When adding snippets for testing, remember that the id we're looking for
+            is "winning" (no quotes).  So, make sure you have that id in your snippet
+            if you want it to pass.
+        """
+        tests = [
+            """<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+   <Description about="urn:mozilla:install-manifest">
+     <em:id>winning</em:id>
+     <em:name>MozMill</em:name>
+     <em:version>2.0a</em:version>
+     <em:creator>Adam Christian</em:creator>
+     <em:description>A testing extension based on the
+            Windmill Testing Framework client source</em:description>
+     <em:unpack>true</em:unpack>
+     <em:targetApplication>
+       <!-- Firefox -->
+       <Description>
+         <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+         <em:minVersion>3.5</em:minVersion>
+         <em:maxVersion>8.*</em:maxVersion>
+       </Description>
+     </em:targetApplication>
+     <em:targetApplication>
+       <!-- Thunderbird -->
+       <Description>
+         <em:id>{3550f703-e582-4d05-9a08-453d09bdfdc6}</em:id>
+         <em:minVersion>3.0a1pre</em:minVersion>
+         <em:maxVersion>3.2*</em:maxVersion>
+       </Description>
+     </em:targetApplication>
+     <em:targetApplication>
+       <!-- Sunbird -->
+       <Description>
+         <em:id>{718e30fb-e89b-41dd-9da7-e25a45638b28}</em:id>
+         <em:minVersion>0.6a1</em:minVersion>
+         <em:maxVersion>1.0pre</em:maxVersion>
+       </Description>
+     </em:targetApplication>
+     <em:targetApplication>
+       <!-- SeaMonkey -->
+       <Description>
+         <em:id>{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}</em:id>
+         <em:minVersion>2.0a1</em:minVersion>
+         <em:maxVersion>2.1*</em:maxVersion>
+       </Description>
+     </em:targetApplication>
+    <em:targetApplication>
+       <!-- Songbird -->
+       <Description>
+         <em:id>songbird@songbirdnest.com</em:id>
+         <em:minVersion>0.3pre</em:minVersion>
+         <em:maxVersion>1.3.0a</em:maxVersion>
+       </Description>
+    </em:targetApplication>
+    <em:targetApplication>
+         <Description>
+          <em:id>toolkit@mozilla.org</em:id>
+          <em:minVersion>1.9.1</em:minVersion>
+          <em:maxVersion>2.0*</em:maxVersion>
+         </Description>
+     </em:targetApplication>
+   </Description>
+</RDF>""",
+            """<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+   <Description about="urn:mozilla:install-manifest">
+     <em:targetApplication>
+       <!-- Firefox -->
+       <Description>
+         <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+         <em:minVersion>3.5</em:minVersion>
+         <em:maxVersion>8.*</em:maxVersion>
+       </Description>
+     </em:targetApplication>
+     <em:id>winning</em:id>
+     <em:name>MozMill</em:name>
+     <em:version>2.0a</em:version>
+     <em:creator>Adam Christian</em:creator>
+     <em:description>A testing extension based on the
+            Windmill Testing Framework client source</em:description>
+     <em:unpack>true</em:unpack>
+    </Description>
+ </RDF>""",
+            """<RDF xmlns="http://www.mozilla.org/2004/em-rdf#"
+        xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
+   <rdf:Description about="urn:mozilla:install-manifest">
+     <id>winning</id>
+     <name>foo</name>
+     <version>42</version>
+     <description>A testing extension based on the
+            Windmill Testing Framework client source</description>
+ </rdf:Description>
+</RDF>""",
+            """<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:foobar="http://www.mozilla.org/2004/em-rdf#">
+   <Description about="urn:mozilla:install-manifest">
+     <foobar:targetApplication>
+       <!-- Firefox -->
+       <Description>
+         <foobar:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</foobar:id>
+         <foobar:minVersion>3.5</foobar:minVersion>
+         <foobar:maxVersion>8.*</foobar:maxVersion>
+       </Description>
+     </foobar:targetApplication>
+     <foobar:id>winning</foobar:id>
+     <foobar:name>MozMill</foobar:name>
+     <foobar:version>2.0a</foobar:version>
+     <foobar:creator>Adam Christian</foobar:creator>
+     <foobar:description>A testing extension based on the
+            Windmill Testing Framework client source</foobar:description>
+     <foobar:unpack>true</foobar:unpack>
+    </Description>
+ </RDF>""",
+            """<?xml version="1.0"?>
+<!--
+
+-->
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+  <Description about="urn:mozilla:install-manifest"
+               em:id="winning"
+               em:name="Language Pack"
+               em:version="42.0a2"
+               em:type="8"
+               em:creator="Some Contributor">
+    <em:contributor></em:contributor>
+
+    <em:targetApplication>
+      <Description>
+        <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+        <em:minVersion>42.0a2</em:minVersion>
+        <em:maxVersion>42.0a2</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+  </Description>
+</RDF>
+"""]
+        return tests
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/addons/empty/install.rdf

@@ -0,0 +1,20 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+	<Description about="urn:mozilla:install-manifest">
+		<em:id>test-empty@quality.mozilla.org</em:id>
+		<em:version>0.1</em:version>
+		<em:name>Test Extension (empty)</em:name>
+		<em:creator>Mozilla QA</em:creator>
+		<em:homepageURL>http://quality.mozilla.org</em:homepageURL>
+		<em:type>2</em:type>
+
+		<!-- Firefox -->
+		<em:targetApplication>
+			<Description>
+				<em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+				<em:minVersion>3.5.*</em:minVersion>
+				<em:maxVersion>*</em:maxVersion>
+			</Description>
+		</em:targetApplication>
+	</Description>
+</RDF>

testing/mozbase/mozprofile/tests/bug758250.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import mozprofile
+import os
+import shutil
+import tempfile
+import unittest
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+class Bug758250(unittest.TestCase):
+    """
+    use of --profile in mozrunner just blows away addon sources:
+    https://bugzilla.mozilla.org/show_bug.cgi?id=758250
+    """
+
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp()
+        self.addon = os.path.join(here, 'addons', 'empty')
+
+    def tearDown(self):
+        # remove vestiges
+        shutil.rmtree(self.tmpdir)
+
+    def test_profile_addon_cleanup(self):
+
+        # sanity check: the empty addon should be here
+        self.assertTrue(os.path.exists(self.addon))
+        self.assertTrue(os.path.isdir(self.addon))
+        self.assertTrue(os.path.exists(os.path.join(self.addon, 'install.rdf')))
+
+        # because we are testing data loss, let's make sure we make a copy
+        shutil.rmtree(self.tmpdir)
+        shutil.copytree(self.addon, self.tmpdir)
+        self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'install.rdf')))
+
+        # make a starter profile
+        profile = mozprofile.FirefoxProfile()
+        path = profile.profile
+
+        # make a new profile based on the old
+        newprofile = mozprofile.FirefoxProfile(profile=path, addons=[self.tmpdir])
+        newprofile.cleanup()
+
+        # the source addon *should* still exist
+        self.assertTrue(os.path.exists(self.tmpdir))
+        self.assertTrue(os.path.exists(os.path.join(self.tmpdir, 'install.rdf')))
+
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/bug785146.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozfile
+import os
+import shutil
+import sqlite3
+import tempfile
+import unittest
+from mozprofile.permissions import Permissions
+
+
+class PermissionsTest(unittest.TestCase):
+
+    locations = """http://mochi.test:8888  primary,privileged
+http://127.0.0.1:80             noxul
+http://127.0.0.1:8888           privileged
+"""
+
+    def setUp(self):
+        self.profile_dir = tempfile.mkdtemp()
+        self.locations_file = mozfile.NamedTemporaryFile()
+        self.locations_file.write(self.locations)
+        self.locations_file.flush()
+
+    def tearDown(self):
+        if self.profile_dir:
+            shutil.rmtree(self.profile_dir)
+        if self.locations_file:
+            self.locations_file.close()
+
+    def test_schema_version(self):
+        perms = Permissions(self.profile_dir, self.locations_file.name)
+        perms_db_filename = os.path.join(self.profile_dir, 'permissions.sqlite')
+        perms.write_db(self.locations_file)
+
+        stmt = 'PRAGMA user_version;'
+
+        con = sqlite3.connect(perms_db_filename)
+        cur = con.cursor()
+        cur.execute(stmt)
+        entries = cur.fetchall()
+
+        schema_version = entries[0][0]
+        self.assertEqual(schema_version, 5)
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/files/prefs_with_comments.js

@@ -0,0 +1,6 @@
+# A leading comment
+user_pref("browser.startup.homepage", "http://planet.mozilla.org"); # A trailing comment
+user_pref("zoom.minPercent", 30);
+// Another leading comment
+user_pref("zoom.maxPercent", 300); // Another trailing comment
+user_pref("webgl.verbose", "false");

testing/mozbase/mozprofile/tests/files/prefs_with_interpolation.js

@@ -0,0 +1,4 @@
+user_pref("browser.foo", "http://{server}");
+user_pref("zoom.minPercent", 30);
+user_pref("webgl.verbose", "false");
+user_pref("browser.bar", "{abc}xyz");

testing/mozbase/mozprofile/tests/files/webapps1.json

@@ -0,0 +1,50 @@
+[{ "name": "http_example_org",
+   "csp": "",
+   "origin": "http://example.org",
+   "manifestURL": "http://example.org/manifest.webapp",
+   "description": "http://example.org App",
+   "appStatus": 1
+ },
+ { "name": "https_example_com",
+   "csp": "",
+   "origin": "https://example.com",
+   "manifestURL": "https://example.com/manifest.webapp",
+   "description": "https://example.com App",
+   "appStatus": 1
+ },
+ { "name": "http_test1_example_org",
+   "csp": "",
+   "origin": "http://test1.example.org",
+   "manifestURL": "http://test1.example.org/manifest.webapp",
+   "description": "http://test1.example.org App",
+   "appStatus": 1
+ },
+ { "name": "http_test1_example_org_8000",
+   "csp": "",
+   "origin": "http://test1.example.org:8000",
+   "manifestURL": "http://test1.example.org:8000/manifest.webapp",
+   "description": "http://test1.example.org:8000 App",
+   "appStatus": 1
+ },
+ { "name": "http_sub1_test1_example_org",
+   "csp": "",
+   "origin": "http://sub1.test1.example.org",
+   "manifestURL": "http://sub1.test1.example.org/manifest.webapp",
+   "description": "http://sub1.test1.example.org App",
+   "appStatus": 1
+ },
+ { "name": "https_example_com_privileged",
+   "csp": "",
+   "origin": "https://example.com",
+   "manifestURL": "https://example.com/manifest_priv.webapp",
+   "description": "https://example.com Privileged App",
+   "appStatus": 2
+ },
+ { "name": "https_example_com_certified",
+   "csp": "",
+   "origin": "https://example.com",
+   "manifestURL": "https://example.com/manifest_cert.webapp",
+   "description": "https://example.com Certified App",
+   "appStatus": 3
+ }
+]

testing/mozbase/mozprofile/tests/files/webapps2.json

@@ -0,0 +1,37 @@
+{
+    "https_example_csp_certified": {
+      "csp": "default-src *; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'",
+      "origin": "https://example.com",
+      "manifestURL": "https://example.com/manifest_csp_cert.webapp",
+      "description": "https://example.com certified app with manifest policy",
+      "appStatus": 3
+    },
+    "https_example_csp_installed": {
+      "csp": "default-src *; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'",
+      "origin": "https://example.com",
+      "manifestURL": "https://example.com/manifest_csp_inst.webapp",
+      "description": "https://example.com installed app with manifest policy",
+      "appStatus": 1
+    },
+    "https_example_csp_privileged": {
+      "csp": "default-src *; script-src 'self'; object-src 'none'; style-src 'self' 'unsafe-inline'",
+      "origin": "https://example.com",
+      "manifestURL": "https://example.com/manifest_csp_priv.webapp",
+      "description": "https://example.com privileged app with manifest policy",
+      "appStatus": 2
+    },
+    "https_a_domain_certified": {
+      "csp": "",
+      "origin": "https://acertified.com",
+      "manifestURL": "https://acertified.com/manifest.webapp",
+      "description": "https://acertified.com certified app",
+      "appStatus": 3
+    },
+    "https_a_domain_privileged": {
+      "csp": "",
+      "origin": "https://aprivileged.com",
+      "manifestURL": "https://aprivileged.com/manifest.webapp",
+      "description": "https://aprivileged.com privileged app ",
+      "appStatus": 2
+    }
+}

testing/mozbase/mozprofile/tests/install_manifests/test_addon_1.rdf

@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <em:id>test-addon-1@mozilla.org</em:id>
+        <em:version>0.1</em:version>
+        <em:name>Test Add-on 1</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>

testing/mozbase/mozprofile/tests/install_manifests/test_addon_2.rdf

@@ -0,0 +1,21 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <em:id>test-addon-2@mozilla.org</em:id>
+        <em:version>0.2</em:version>
+        <em:name>Test Add-on 2</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>

testing/mozbase/mozprofile/tests/install_manifests/test_addon_3.rdf

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <em:id>test-addon-3@mozilla.org</em:id>
+        <em:version>0.1</em:version>
+        <em:name>Test Add-on 3</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>
+

testing/mozbase/mozprofile/tests/install_manifests/test_addon_4.rdf

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <em:id>test-addon-4@mozilla.org</em:id>
+        <em:version>0.1</em:version>
+        <em:name>Test Add-on 4</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>
+

testing/mozbase/mozprofile/tests/install_manifests/test_addon_invalid_no_id.rdf

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <!-- Invalid because of a missing add-on id -->
+        <em:version>0.1</em:version>
+        <em:name>Test Invalid Extension (no id)</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <!-- Invalid target application string -->
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>

testing/mozbase/mozprofile/tests/install_manifests/test_addon_invalid_not_wellformed.rdf

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <!-- Invalid because it's not well-formed -->
+        <em:id>test-addon-invalid-not-wellformed@mozilla.org</em:id
+        <em:version>0.1</em:version>
+        <em:name>Test Invalid Extension (no id)</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <!-- Invalid target application string -->
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>

testing/mozbase/mozprofile/tests/install_manifests/test_addon_invalid_version.rdf

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <em:id>test-addon-invalid-version@mozilla.org</em:id>
+        <!-- Invalid addon version -->
+        <em:version>0.NOPE</em:version>
+        <em:name>Test Invalid Extension (invalid version)</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <!-- Invalid target application string -->
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>

testing/mozbase/mozprofile/tests/install_manifests/test_addon_unpack.rdf

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+    xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+    <Description about="urn:mozilla:install-manifest">
+        <em:id>test-addon-unpack@mozilla.org</em:id>
+        <em:version>0.1</em:version>
+        <em:name>Test Add-on (unpack)</em:name>
+        <em:creator>Mozilla</em:creator>
+        <em:homepageURL>http://mozilla.org</em:homepageURL>
+        <em:type>2</em:type>
+        <em:unpack>true</em:unpack>
+
+        <!-- Firefox -->
+        <em:targetApplication>
+            <Description>
+                <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+                <em:minVersion>3.5.*</em:minVersion>
+                <em:maxVersion>*</em:maxVersion>
+            </Description>
+        </em:targetApplication>
+    </Description>
+</RDF>

testing/mozbase/mozprofile/tests/manifest.ini

@@ -0,0 +1,12 @@
+[addonid.py]
+[server_locations.py]
+[test_preferences.py]
+[permissions.py]
+[bug758250.py]
+[test_nonce.py]
+[bug785146.py]
+[test_clone_cleanup.py]
+[test_webapps.py]
+[test_profile.py]
+[test_profile_view.py]
+[test_addons.py]

testing/mozbase/mozprofile/tests/permissions.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozfile
+import os
+import shutil
+import sqlite3
+import tempfile
+import unittest
+from mozprofile.permissions import Permissions
+
+
+class PermissionsTest(unittest.TestCase):
+
+    locations = """http://mochi.test:8888  primary,privileged
+http://127.0.0.1:80             noxul
+http://127.0.0.1:8888           privileged
+"""
+
+    profile_dir = None
+    locations_file = None
+
+    def setUp(self):
+        self.profile_dir = tempfile.mkdtemp()
+        self.locations_file = mozfile.NamedTemporaryFile()
+        self.locations_file.write(self.locations)
+        self.locations_file.flush()
+
+    def tearDown(self):
+        if self.profile_dir:
+            shutil.rmtree(self.profile_dir)
+        if self.locations_file:
+            self.locations_file.close()
+
+    def write_perm_db(self, version=3):
+        permDB = sqlite3.connect(os.path.join(self.profile_dir, "permissions.sqlite"))
+        cursor = permDB.cursor()
+
+        cursor.execute("PRAGMA user_version=%d;" % version)
+
+        if version == 5:
+            cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
+              id INTEGER PRIMARY KEY,
+              origin TEXT,
+              type TEXT,
+              permission INTEGER,
+              expireType INTEGER,
+              expireTime INTEGER,
+              modificationTime INTEGER)""")
+        elif version == 4:
+            cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
+               id INTEGER PRIMARY KEY,
+               host TEXT,
+               type TEXT,
+               permission INTEGER,
+               expireType INTEGER,
+               expireTime INTEGER,
+               modificationTime INTEGER,
+               appId INTEGER,
+               isInBrowserElement INTEGER)""")
+        elif version == 3:
+            cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
+               id INTEGER PRIMARY KEY,
+               host TEXT,
+               type TEXT,
+               permission INTEGER,
+               expireType INTEGER,
+               expireTime INTEGER,
+               appId INTEGER,
+               isInBrowserElement INTEGER)""")
+        elif version == 2:
+            cursor.execute("""CREATE TABLE IF NOT EXISTS moz_hosts (
+               id INTEGER PRIMARY KEY,
+               host TEXT,
+               type TEXT,
+               permission INTEGER,
+               expireType INTEGER,
+               expireTime INTEGER)""")
+        else:
+            raise Exception("version must be 2, 3, 4 or 5")
+
+        permDB.commit()
+        cursor.close()
+
+    def test_create_permissions_db(self):
+        perms = Permissions(self.profile_dir, self.locations_file.name)
+        perms_db_filename = os.path.join(self.profile_dir, 'permissions.sqlite')
+
+        select_stmt = 'select origin, type, permission from moz_hosts'
+
+        con = sqlite3.connect(perms_db_filename)
+        cur = con.cursor()
+        cur.execute(select_stmt)
+        entries = cur.fetchall()
+
+        self.assertEqual(len(entries), 3)
+
+        self.assertEqual(entries[0][0], 'http://mochi.test:8888')
+        self.assertEqual(entries[0][1], 'allowXULXBL')
+        self.assertEqual(entries[0][2], 1)
+
+        self.assertEqual(entries[1][0], 'http://127.0.0.1')
+        self.assertEqual(entries[1][1], 'allowXULXBL')
+        self.assertEqual(entries[1][2], 2)
+
+        self.assertEqual(entries[2][0], 'http://127.0.0.1:8888')
+        self.assertEqual(entries[2][1], 'allowXULXBL')
+        self.assertEqual(entries[2][2], 1)
+
+        perms._locations.add_host('a.b.c', port='8081', scheme='https', options='noxul')
+
+        cur.execute(select_stmt)
+        entries = cur.fetchall()
+
+        self.assertEqual(len(entries), 4)
+        self.assertEqual(entries[3][0], 'https://a.b.c:8081')
+        self.assertEqual(entries[3][1], 'allowXULXBL')
+        self.assertEqual(entries[3][2], 2)
+
+        # when creating a DB we should default to user_version==5
+        cur.execute('PRAGMA user_version')
+        entries = cur.fetchall()
+        self.assertEqual(entries[0][0], 5)
+
+        perms.clean_db()
+        # table should be removed
+        cur.execute("select * from sqlite_master where type='table'")
+        entries = cur.fetchall()
+        self.assertEqual(len(entries), 0)
+
+    def test_nw_prefs(self):
+        perms = Permissions(self.profile_dir, self.locations_file.name)
+
+        prefs, user_prefs = perms.network_prefs(False)
+
+        self.assertEqual(len(user_prefs), 0)
+        self.assertEqual(len(prefs), 0)
+
+        prefs, user_prefs = perms.network_prefs(True)
+        self.assertEqual(len(user_prefs), 2)
+        self.assertEqual(user_prefs[0], ('network.proxy.type', 2))
+        self.assertEqual(user_prefs[1][0], 'network.proxy.autoconfig_url')
+
+        origins_decl = "var knownOrigins = (function () {  return ['http://mochi.test:8888', " \
+                       "'http://127.0.0.1:80', 'http://127.0.0.1:8888'].reduce"
+        self.assertTrue(origins_decl in user_prefs[1][1])
+
+        proxy_check = ("'http': 'PROXY mochi.test:8888'",
+                       "'https': 'PROXY mochi.test:4443'",
+                       "'ws': 'PROXY mochi.test:4443'",
+                       "'wss': 'PROXY mochi.test:4443'")
+        self.assertTrue(all(c in user_prefs[1][1] for c in proxy_check))
+
+    def verify_user_version(self, version):
+        """Verifies that we call INSERT statements using the correct number
+        of columns for existing databases.
+        """
+        self.write_perm_db(version=version)
+        Permissions(self.profile_dir, self.locations_file.name)
+        perms_db_filename = os.path.join(self.profile_dir, 'permissions.sqlite')
+
+        select_stmt = 'select * from moz_hosts'
+
+        con = sqlite3.connect(perms_db_filename)
+        cur = con.cursor()
+        cur.execute(select_stmt)
+        entries = cur.fetchall()
+
+        self.assertEqual(len(entries), 3)
+
+        columns = {
+            1: 6,
+            2: 6,
+            3: 8,
+            4: 9,
+            5: 7,
+        }[version]
+
+        self.assertEqual(len(entries[0]), columns)
+        for x in range(4, columns):
+            self.assertEqual(entries[0][x], 0)
+
+    def test_existing_permissions_db_v2(self):
+        self.verify_user_version(2)
+
+    def test_existing_permissions_db_v3(self):
+        self.verify_user_version(3)
+
+    def test_existing_permissions_db_v4(self):
+        self.verify_user_version(4)
+
+    def test_existing_permissions_db_v5(self):
+        self.verify_user_version(5)
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/server_locations.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozfile
+import unittest
+from mozprofile.permissions import ServerLocations, \
+    MissingPrimaryLocationError, MultiplePrimaryLocationsError, \
+    DuplicateLocationError, BadPortLocationError, LocationsSyntaxError
+
+
+class ServerLocationsTest(unittest.TestCase):
+    """test server locations"""
+
+    locations = """# This is the primary location from which tests run.
+#
+http://mochi.test:8888          primary,privileged
+
+# a few test locations
+http://127.0.0.1:80             privileged
+http://127.0.0.1:8888           privileged
+https://test:80                 privileged
+http://example.org:80           privileged
+http://test1.example.org        privileged
+
+    """
+
+    locations_no_primary = """http://secondary.test:80        privileged
+http://tertiary.test:8888       privileged
+"""
+
+    locations_bad_port = """http://mochi.test:8888  primary,privileged
+http://127.0.0.1:80             privileged
+http://127.0.0.1:8888           privileged
+http://test:badport             privileged
+http://example.org:80           privileged
+"""
+
+    def compare_location(self, location, scheme, host, port, options):
+        self.assertEqual(location.scheme, scheme)
+        self.assertEqual(location.host, host)
+        self.assertEqual(location.port, port)
+        self.assertEqual(location.options, options)
+
+    def create_temp_file(self, contents):
+        f = mozfile.NamedTemporaryFile()
+        f.write(contents)
+        f.flush()
+        return f
+
+    def test_server_locations(self):
+        # write a permissions file
+        f = self.create_temp_file(self.locations)
+
+        # read the locations
+        locations = ServerLocations(f.name)
+
+        # ensure that they're what we expect
+        self.assertEqual(len(locations), 6)
+        i = iter(locations)
+        self.compare_location(i.next(), 'http', 'mochi.test', '8888',
+                              ['primary', 'privileged'])
+        self.compare_location(i.next(), 'http', '127.0.0.1', '80',
+                              ['privileged'])
+        self.compare_location(i.next(), 'http', '127.0.0.1', '8888',
+                              ['privileged'])
+        self.compare_location(i.next(), 'https', 'test', '80', ['privileged'])
+        self.compare_location(i.next(), 'http', 'example.org', '80',
+                              ['privileged'])
+        self.compare_location(i.next(), 'http', 'test1.example.org', '8888',
+                              ['privileged'])
+
+        locations.add_host('mozilla.org')
+        self.assertEqual(len(locations), 7)
+        self.compare_location(i.next(), 'http', 'mozilla.org', '80',
+                              ['privileged'])
+
+        # test some errors
+        self.assertRaises(MultiplePrimaryLocationsError, locations.add_host,
+                          'primary.test', options='primary')
+
+        # We no longer throw these DuplicateLocation Error
+        try:
+            locations.add_host('127.0.0.1')
+        except DuplicateLocationError:
+            self.assertTrue(False, "Should no longer throw DuplicateLocationError")
+
+        self.assertRaises(BadPortLocationError, locations.add_host, '127.0.0.1',
+                          port='abc')
+
+        # test some errors in locations file
+        f = self.create_temp_file(self.locations_no_primary)
+
+        exc = None
+        try:
+            ServerLocations(f.name)
+        except LocationsSyntaxError as e:
+            exc = e
+        self.assertNotEqual(exc, None)
+        self.assertEqual(exc.err.__class__, MissingPrimaryLocationError)
+        self.assertEqual(exc.lineno, 3)
+
+        # test bad port in a locations file to ensure lineno calculated
+        # properly.
+        f = self.create_temp_file(self.locations_bad_port)
+
+        exc = None
+        try:
+            ServerLocations(f.name)
+        except LocationsSyntaxError as e:
+            exc = e
+        self.assertNotEqual(exc, None)
+        self.assertEqual(exc.err.__class__, BadPortLocationError)
+        self.assertEqual(exc.lineno, 4)
+
+    def test_server_locations_callback(self):
+        class CallbackTest(object):
+            last_locations = None
+
+            def callback(self, locations):
+                self.last_locations = locations
+
+        c = CallbackTest()
+        f = self.create_temp_file(self.locations)
+        locations = ServerLocations(f.name, c.callback)
+
+        # callback should be for all locations in file
+        self.assertEqual(len(c.last_locations), 6)
+
+        # validate arbitrary one
+        self.compare_location(c.last_locations[2], 'http', '127.0.0.1', '8888',
+                              ['privileged'])
+
+        locations.add_host('a.b.c')
+
+        # callback should be just for one location
+        self.assertEqual(len(c.last_locations), 1)
+        self.compare_location(c.last_locations[0], 'http', 'a.b.c', '80',
+                              ['privileged'])
+
+        # read a second file, which should generate a callback with both
+        # locations.
+        f = self.create_temp_file(self.locations_no_primary)
+        locations.read(f.name)
+        self.assertEqual(len(c.last_locations), 2)
+
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_addons.py

@@ -0,0 +1,415 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import tempfile
+import unittest
+import urllib2
+
+from manifestparser import ManifestParser
+import mozfile
+import mozhttpd
+import mozlog.unstructured as mozlog
+import mozprofile
+
+from addon_stubs import generate_addon, generate_manifest
+
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+class TestAddonsManager(unittest.TestCase):
+    """ Class to test mozprofile.addons.AddonManager """
+
+    def setUp(self):
+        self.logger = mozlog.getLogger('mozprofile.addons')
+        self.logger.setLevel(mozlog.ERROR)
+
+        self.profile = mozprofile.profile.Profile()
+        self.am = self.profile.addon_manager
+
+        self.profile_path = self.profile.profile
+        self.tmpdir = tempfile.mkdtemp()
+        self.addCleanup(mozfile.remove, self.tmpdir)
+
+    def test_install_addons_multiple_same_source(self):
+        # Generate installer stubs for all possible types of addons
+        addon_xpi = generate_addon('test-addon-1@mozilla.org',
+                                   path=self.tmpdir)
+        addon_folder = generate_addon('test-addon-1@mozilla.org',
+                                      path=self.tmpdir,
+                                      xpi=False)
+
+        # The same folder should not be installed twice
+        self.am.install_addons([addon_folder, addon_folder])
+        self.assertEqual(self.am.installed_addons, [addon_folder])
+        self.am.clean()
+
+        # The same XPI file should not be installed twice
+        self.am.install_addons([addon_xpi, addon_xpi])
+        self.assertEqual(self.am.installed_addons, [addon_xpi])
+        self.am.clean()
+
+        # Even if it is the same id the add-on should be installed twice, if
+        # specified via XPI and folder
+        self.am.install_addons([addon_folder, addon_xpi])
+        self.assertEqual(len(self.am.installed_addons), 2)
+        self.assertIn(addon_folder, self.am.installed_addons)
+        self.assertIn(addon_xpi, self.am.installed_addons)
+        self.am.clean()
+
+    def test_download(self):
+        server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
+        server.start()
+
+        # Download a valid add-on without a class instance to the general
+        # tmp folder and clean-up
+        try:
+            addon = server.get_url() + 'empty.xpi'
+            xpi_file = mozprofile.addons.AddonManager.download(addon)
+            self.assertTrue(os.path.isfile(xpi_file))
+            self.assertIn('test-empty@quality.mozilla.org.xpi',
+                          os.path.basename(xpi_file))
+            self.assertNotIn(self.tmpdir, os.path.dirname(xpi_file))
+        finally:
+            # Given that the file is stored outside of the created tmp dir
+            # we have to ensure to explicitely remove it
+            if os.path.isfile(xpi_file):
+                os.remove(xpi_file)
+
+        # Download an valid add-on to a special folder
+        addon = server.get_url() + 'empty.xpi'
+        xpi_file = self.am.download(addon, self.tmpdir)
+        self.assertTrue(os.path.isfile(xpi_file))
+        self.assertIn('test-empty@quality.mozilla.org.xpi',
+                      os.path.basename(xpi_file))
+        self.assertIn(self.tmpdir, os.path.dirname(xpi_file))
+        self.assertEqual(self.am.downloaded_addons, [])
+        os.remove(xpi_file)
+
+        # Download an invalid add-on to a special folder
+        addon = server.get_url() + 'invalid.xpi'
+        self.assertRaises(mozprofile.addons.AddonFormatError,
+                          self.am.download, addon, self.tmpdir)
+        self.assertEqual(os.listdir(self.tmpdir), [])
+
+        # Download from an invalid URL
+        addon = server.get_url() + 'not_existent.xpi'
+        self.assertRaises(urllib2.HTTPError,
+                          self.am.download, addon, self.tmpdir)
+        self.assertEqual(os.listdir(self.tmpdir), [])
+
+        # Download from an invalid URL
+        addon = 'not_existent.xpi'
+        self.assertRaises(ValueError,
+                          self.am.download, addon, self.tmpdir)
+        self.assertEqual(os.listdir(self.tmpdir), [])
+
+        server.stop()
+
+    def test_install_from_path_xpi(self):
+        addons_to_install = []
+        addons_installed = []
+
+        # Generate installer stubs and install them
+        for ext in ['test-addon-1@mozilla.org', 'test-addon-2@mozilla.org']:
+            temp_addon = generate_addon(ext, path=self.tmpdir)
+            addons_to_install.append(self.am.addon_details(temp_addon)['id'])
+            self.am.install_from_path(temp_addon)
+
+        # Generate a list of addons installed in the profile
+        addons_installed = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join(
+                            self.profile.profile, 'extensions', 'staged'))]
+        self.assertEqual(addons_to_install.sort(), addons_installed.sort())
+
+    def test_install_from_path_folder(self):
+        # Generate installer stubs for all possible types of addons
+        addons = []
+        addons.append(generate_addon('test-addon-1@mozilla.org',
+                                     path=self.tmpdir))
+        addons.append(generate_addon('test-addon-2@mozilla.org',
+                                     path=self.tmpdir,
+                                     xpi=False))
+        addons.append(generate_addon('test-addon-3@mozilla.org',
+                                     path=self.tmpdir,
+                                     name='addon-3'))
+        addons.append(generate_addon('test-addon-4@mozilla.org',
+                                     path=self.tmpdir,
+                                     name='addon-4',
+                                     xpi=False))
+        addons.sort()
+
+        self.am.install_from_path(self.tmpdir)
+
+        self.assertEqual(self.am.installed_addons, addons)
+
+    def test_install_from_path_unpack(self):
+        # Generate installer stubs for all possible types of addons
+        addon_xpi = generate_addon('test-addon-unpack@mozilla.org',
+                                   path=self.tmpdir)
+        addon_folder = generate_addon('test-addon-unpack@mozilla.org',
+                                      path=self.tmpdir,
+                                      xpi=False)
+        addon_no_unpack = generate_addon('test-addon-1@mozilla.org',
+                                         path=self.tmpdir)
+
+        # Test unpack flag for add-on as XPI
+        self.am.install_from_path(addon_xpi)
+        self.assertEqual(self.am.installed_addons, [addon_xpi])
+        self.am.clean()
+
+        # Test unpack flag for add-on as folder
+        self.am.install_from_path(addon_folder)
+        self.assertEqual(self.am.installed_addons, [addon_folder])
+        self.am.clean()
+
+        # Test forcing unpack an add-on
+        self.am.install_from_path(addon_no_unpack, unpack=True)
+        self.assertEqual(self.am.installed_addons, [addon_no_unpack])
+        self.am.clean()
+
+    def test_install_from_path_url(self):
+        server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
+        server.start()
+
+        addon = server.get_url() + 'empty.xpi'
+        self.am.install_from_path(addon)
+
+        server.stop()
+
+        self.assertEqual(len(self.am.downloaded_addons), 1)
+        self.assertTrue(os.path.isfile(self.am.downloaded_addons[0]))
+        self.assertIn('test-empty@quality.mozilla.org.xpi',
+                      os.path.basename(self.am.downloaded_addons[0]))
+
+    def test_install_from_path_after_reset(self):
+        # Installing the same add-on after a reset should not cause a failure
+        addon = generate_addon('test-addon-1@mozilla.org',
+                               path=self.tmpdir, xpi=False)
+
+        # We cannot use self.am because profile.reset() creates a new instance
+        self.profile.addon_manager.install_from_path(addon)
+
+        self.profile.reset()
+
+        self.profile.addon_manager.install_from_path(addon)
+        self.assertEqual(self.profile.addon_manager.installed_addons, [addon])
+
+    def test_install_from_path_backup(self):
+        staged_path = os.path.join(self.profile_path, 'extensions', 'staged')
+
+        # Generate installer stubs for all possible types of addons
+        addon_xpi = generate_addon('test-addon-1@mozilla.org',
+                                   path=self.tmpdir)
+        addon_folder = generate_addon('test-addon-1@mozilla.org',
+                                      path=self.tmpdir,
+                                      xpi=False)
+        addon_name = generate_addon('test-addon-1@mozilla.org',
+                                    path=self.tmpdir,
+                                    name='test-addon-1-dupe@mozilla.org')
+
+        # Test backup of xpi files
+        self.am.install_from_path(addon_xpi)
+        self.assertIsNone(self.am.backup_dir)
+
+        self.am.install_from_path(addon_xpi)
+        self.assertIsNotNone(self.am.backup_dir)
+        self.assertEqual(os.listdir(self.am.backup_dir),
+                         ['test-addon-1@mozilla.org.xpi'])
+
+        self.am.clean()
+        self.assertEqual(os.listdir(staged_path),
+                         ['test-addon-1@mozilla.org.xpi'])
+        self.am.clean()
+
+        # Test backup of folders
+        self.am.install_from_path(addon_folder)
+        self.assertIsNone(self.am.backup_dir)
+
+        self.am.install_from_path(addon_folder)
+        self.assertIsNotNone(self.am.backup_dir)
+        self.assertEqual(os.listdir(self.am.backup_dir),
+                         ['test-addon-1@mozilla.org'])
+
+        self.am.clean()
+        self.assertEqual(os.listdir(staged_path),
+                         ['test-addon-1@mozilla.org'])
+        self.am.clean()
+
+        # Test backup of xpi files with another file name
+        self.am.install_from_path(addon_name)
+        self.assertIsNone(self.am.backup_dir)
+
+        self.am.install_from_path(addon_xpi)
+        self.assertIsNotNone(self.am.backup_dir)
+        self.assertEqual(os.listdir(self.am.backup_dir),
+                         ['test-addon-1@mozilla.org.xpi'])
+
+        self.am.clean()
+        self.assertEqual(os.listdir(staged_path),
+                         ['test-addon-1@mozilla.org.xpi'])
+        self.am.clean()
+
+    def test_install_from_path_invalid_addons(self):
+        # Generate installer stubs for all possible types of addons
+        addons = []
+        addons.append(generate_addon('test-addon-invalid-no-manifest@mozilla.org',
+                                     path=self.tmpdir,
+                                     xpi=False))
+        addons.append(generate_addon('test-addon-invalid-no-id@mozilla.org',
+                                     path=self.tmpdir))
+
+        self.am.install_from_path(self.tmpdir)
+
+        self.assertEqual(self.am.installed_addons, [])
+
+    @unittest.skip("Feature not implemented as part of AddonManger")
+    def test_install_from_path_error(self):
+        """ Check install_from_path raises an error with an invalid addon"""
+
+        temp_addon = generate_addon('test-addon-invalid-version@mozilla.org')
+        # This should raise an error here
+        self.am.install_from_path(temp_addon)
+
+    def test_install_from_manifest(self):
+        temp_manifest = generate_manifest(['test-addon-1@mozilla.org',
+                                           'test-addon-2@mozilla.org'])
+        m = ManifestParser()
+        m.read(temp_manifest)
+        addons = m.get()
+
+        # Obtain details of addons to install from the manifest
+        addons_to_install = [self.am.addon_details(x['path']).get('id') for x in addons]
+
+        self.am.install_from_manifest(temp_manifest)
+        # Generate a list of addons installed in the profile
+        addons_installed = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join(
+                            self.profile.profile, 'extensions', 'staged'))]
+        self.assertEqual(addons_installed.sort(), addons_to_install.sort())
+
+        # Cleanup the temporary addon and manifest directories
+        mozfile.rmtree(os.path.dirname(temp_manifest))
+
+    def test_addon_details(self):
+        # Generate installer stubs for a valid and invalid add-on manifest
+        valid_addon = generate_addon('test-addon-1@mozilla.org',
+                                     path=self.tmpdir)
+        invalid_addon = generate_addon('test-addon-invalid-not-wellformed@mozilla.org',
+                                       path=self.tmpdir)
+
+        # Check valid add-on
+        details = self.am.addon_details(valid_addon)
+        self.assertEqual(details['id'], 'test-addon-1@mozilla.org')
+        self.assertEqual(details['name'], 'Test Add-on 1')
+        self.assertEqual(details['unpack'], False)
+        self.assertEqual(details['version'], '0.1')
+
+        # Check invalid add-on
+        self.assertRaises(mozprofile.addons.AddonFormatError,
+                          self.am.addon_details, invalid_addon)
+
+        # Check invalid path
+        self.assertRaises(IOError,
+                          self.am.addon_details, '')
+
+        # Check invalid add-on format
+        addon_path = os.path.join(os.path.join(here, 'files'), 'not_an_addon.txt')
+        self.assertRaises(mozprofile.addons.AddonFormatError,
+                          self.am.addon_details, addon_path)
+
+    @unittest.skip("Bug 900154")
+    def test_clean_addons(self):
+        addon_one = generate_addon('test-addon-1@mozilla.org')
+        addon_two = generate_addon('test-addon-2@mozilla.org')
+
+        self.am.install_addons(addon_one)
+        installed_addons = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join(
+                            self.profile.profile, 'extensions', 'staged'))]
+
+        # Create a new profile based on an existing profile
+        # Install an extra addon in the new profile
+        # Cleanup addons
+        duplicate_profile = mozprofile.profile.Profile(profile=self.profile.profile,
+                                                       addons=addon_two)
+        duplicate_profile.addon_manager.clean()
+
+        addons_after_cleanup = [unicode(x[:-len('.xpi')]) for x in os.listdir(os.path.join(
+                                duplicate_profile.profile, 'extensions', 'staged'))]
+        # New addons installed should be removed by clean_addons()
+        self.assertEqual(installed_addons, addons_after_cleanup)
+
+    def test_noclean(self):
+        """test `restore=True/False` functionality"""
+
+        server = mozhttpd.MozHttpd(docroot=os.path.join(here, 'addons'))
+        server.start()
+
+        profile = tempfile.mkdtemp()
+        tmpdir = tempfile.mkdtemp()
+
+        try:
+            # empty initially
+            self.assertFalse(bool(os.listdir(profile)))
+
+            # make an addon
+            addons = []
+            addons.append(generate_addon('test-addon-1@mozilla.org',
+                                         path=tmpdir))
+            addons.append(server.get_url() + 'empty.xpi')
+
+            # install it with a restore=True AddonManager
+            am = mozprofile.addons.AddonManager(profile, restore=True)
+
+            for addon in addons:
+                am.install_from_path(addon)
+
+            # now its there
+            self.assertEqual(os.listdir(profile), ['extensions'])
+            staging_folder = os.path.join(profile, 'extensions', 'staged')
+            self.assertTrue(os.path.exists(staging_folder))
+            self.assertEqual(len(os.listdir(staging_folder)), 2)
+
+            # del addons; now its gone though the directory tree exists
+            downloaded_addons = am.downloaded_addons
+            del am
+
+            self.assertEqual(os.listdir(profile), ['extensions'])
+            self.assertTrue(os.path.exists(staging_folder))
+            self.assertEqual(os.listdir(staging_folder), [])
+
+            for addon in downloaded_addons:
+                self.assertFalse(os.path.isfile(addon))
+
+        finally:
+            mozfile.rmtree(tmpdir)
+            mozfile.rmtree(profile)
+
+    def test_remove_addon(self):
+        addons = []
+        addons.append(generate_addon('test-addon-1@mozilla.org',
+                                     path=self.tmpdir))
+        addons.append(generate_addon('test-addon-2@mozilla.org',
+                                     path=self.tmpdir))
+
+        self.am.install_from_path(self.tmpdir)
+
+        extensions_path = os.path.join(self.profile_path, 'extensions')
+        staging_path = os.path.join(extensions_path, 'staged')
+
+        # Fake a run by virtually installing one of the staged add-ons
+        shutil.move(os.path.join(staging_path, 'test-addon-1@mozilla.org.xpi'),
+                    extensions_path)
+
+        for addon in self.am._addons:
+            self.am.remove_addon(addon)
+
+        self.assertEqual(os.listdir(staging_path), [])
+        self.assertEqual(os.listdir(extensions_path), ['staged'])
+
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_clone_cleanup.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+import os
+import tempfile
+import unittest
+import mozfile
+
+from mozprofile.profile import Profile
+
+
+class CloneCleanupTest(unittest.TestCase):
+    """
+    test cleanup logic for the clone functionality
+    see https://bugzilla.mozilla.org/show_bug.cgi?id=642843
+    """
+
+    def setUp(self):
+        # make a profile with one preference
+        path = tempfile.mktemp()
+        self.addCleanup(mozfile.remove, path)
+        self.profile = Profile(path,
+                               preferences={'foo': 'bar'},
+                               restore=False)
+        user_js = os.path.join(self.profile.profile, 'user.js')
+        self.assertTrue(os.path.exists(user_js))
+
+    def test_restore_true(self):
+        # make a clone of this profile with restore=True
+        clone = Profile.clone(self.profile.profile, restore=True)
+        self.addCleanup(mozfile.remove, clone.profile)
+
+        clone.cleanup()
+
+        # clone should be deleted
+        self.assertFalse(os.path.exists(clone.profile))
+
+    def test_restore_false(self):
+        # make a clone of this profile with restore=False
+        clone = Profile.clone(self.profile.profile, restore=False)
+        self.addCleanup(mozfile.remove, clone.profile)
+
+        clone.cleanup()
+
+        # clone should still be around on the filesystem
+        self.assertTrue(os.path.exists(clone.profile))
+
+    def test_cleanup_on_garbage_collected(self):
+        clone = Profile.clone(self.profile.profile)
+        self.addCleanup(mozfile.remove, clone.profile)
+        profile_dir = clone.profile
+        self.assertTrue(os.path.exists(profile_dir))
+        del clone
+        # clone should be deleted
+        self.assertFalse(os.path.exists(profile_dir))
+
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_nonce.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+"""
+test nonce in prefs delimeters
+see https://bugzilla.mozilla.org/show_bug.cgi?id=722804
+"""
+
+import os
+import tempfile
+import unittest
+import mozfile
+from mozprofile.prefs import Preferences
+from mozprofile.profile import Profile
+
+
+class PreferencesNonceTest(unittest.TestCase):
+
+    def test_nonce(self):
+
+        # make a profile with one preference
+        path = tempfile.mktemp()
+        self.addCleanup(mozfile.remove, path)
+        profile = Profile(path,
+                          preferences={'foo': 'bar'},
+                          restore=False)
+        user_js = os.path.join(profile.profile, 'user.js')
+        self.assertTrue(os.path.exists(user_js))
+
+        # ensure the preference is correct
+        prefs = Preferences.read_prefs(user_js)
+        self.assertEqual(dict(prefs), {'foo': 'bar'})
+
+        del profile
+
+        # augment the profile with a second preference
+        profile = Profile(path,
+                          preferences={'fleem': 'baz'},
+                          restore=True)
+        prefs = Preferences.read_prefs(user_js)
+        self.assertEqual(dict(prefs), {'foo': 'bar', 'fleem': 'baz'})
+
+        # cleanup the profile;
+        # this should remove the new preferences but not the old
+        profile.cleanup()
+        prefs = Preferences.read_prefs(user_js)
+        self.assertEqual(dict(prefs), {'foo': 'bar'})
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_preferences.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozfile
+import mozhttpd
+import os
+import shutil
+import tempfile
+import unittest
+from mozprofile.cli import MozProfileCLI
+from mozprofile.prefs import Preferences
+from mozprofile.profile import Profile
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+class PreferencesTest(unittest.TestCase):
+    """test mozprofile preference handling"""
+
+    # preferences from files/prefs_with_comments.js
+    _prefs_with_comments = {'browser.startup.homepage': 'http://planet.mozilla.org',
+                            'zoom.minPercent': 30,
+                            'zoom.maxPercent': 300,
+                            'webgl.verbose': 'false'}
+
+    def run_command(self, *args):
+        """
+        invokes mozprofile command line via the CLI factory
+        - args : command line arguments (equivalent of sys.argv[1:])
+        """
+
+        # instantiate the factory
+        cli = MozProfileCLI(list(args))
+
+        # create the profile
+        profile = cli.profile()
+
+        # return path to profile
+        return profile.profile
+
+    def compare_generated(self, _prefs, commandline):
+        """
+        writes out to a new profile with mozprofile command line
+        reads the generated preferences with prefs.py
+        compares the results
+        cleans up
+        """
+        profile = self.run_command(*commandline)
+        prefs_file = os.path.join(profile, 'user.js')
+        self.assertTrue(os.path.exists(prefs_file))
+        read = Preferences.read_prefs(prefs_file)
+        if isinstance(_prefs, dict):
+            read = dict(read)
+        self.assertEqual(_prefs, read)
+        shutil.rmtree(profile)
+
+    def test_basic_prefs(self):
+        """test setting a pref from the command line entry point"""
+
+        _prefs = {"browser.startup.homepage": "http://planet.mozilla.org/"}
+        commandline = []
+        _prefs = _prefs.items()
+        for pref, value in _prefs:
+            commandline += ["--pref", "%s:%s" % (pref, value)]
+        self.compare_generated(_prefs, commandline)
+
+    def test_ordered_prefs(self):
+        """ensure the prefs stay in the right order"""
+        _prefs = [("browser.startup.homepage", "http://planet.mozilla.org/"),
+                  ("zoom.minPercent", 30),
+                  ("zoom.maxPercent", 300),
+                  ("webgl.verbose", 'false')]
+        commandline = []
+        for pref, value in _prefs:
+            commandline += ["--pref", "%s:%s" % (pref, value)]
+        _prefs = [(i, Preferences.cast(j)) for i, j in _prefs]
+        self.compare_generated(_prefs, commandline)
+
+    def test_ini(self):
+
+        # write the .ini file
+        _ini = """[DEFAULT]
+browser.startup.homepage = http://planet.mozilla.org/
+
+[foo]
+browser.startup.homepage = http://github.com/
+"""
+        try:
+            fd, name = tempfile.mkstemp(suffix='.ini')
+            os.write(fd, _ini)
+            os.close(fd)
+            commandline = ["--preferences", name]
+
+            # test the [DEFAULT] section
+            _prefs = {'browser.startup.homepage': 'http://planet.mozilla.org/'}
+            self.compare_generated(_prefs, commandline)
+
+            # test a specific section
+            _prefs = {'browser.startup.homepage': 'http://github.com/'}
+            commandline[-1] = commandline[-1] + ':foo'
+            self.compare_generated(_prefs, commandline)
+
+        finally:
+            # cleanup
+            os.remove(name)
+
+    def test_ini_keep_case(self):
+        """
+        Read a preferences config file with a preference in camel-case style.
+        Check that the read preference name has not been lower-cased
+        """
+        # write the .ini file
+        _ini = """[DEFAULT]
+general.warnOnAboutConfig = False
+"""
+        try:
+            fd, name = tempfile.mkstemp(suffix='.ini')
+            os.write(fd, _ini)
+            os.close(fd)
+            commandline = ["--preferences", name]
+
+            # test the [DEFAULT] section
+            _prefs = {'general.warnOnAboutConfig': 'False'}
+            self.compare_generated(_prefs, commandline)
+
+        finally:
+            # cleanup
+            os.remove(name)
+
+    def test_reset_should_remove_added_prefs(self):
+        """Check that when we call reset the items we expect are updated"""
+        profile = Profile()
+        prefs_file = os.path.join(profile.profile, 'user.js')
+
+        # we shouldn't have any initial preferences
+        initial_prefs = Preferences.read_prefs(prefs_file)
+        self.assertFalse(initial_prefs)
+        initial_prefs = file(prefs_file).read().strip()
+        self.assertFalse(initial_prefs)
+
+        # add some preferences
+        prefs1 = [("mr.t.quotes", "i aint getting on no plane!")]
+        profile.set_preferences(prefs1)
+        self.assertEqual(prefs1, Preferences.read_prefs(prefs_file))
+        lines = file(prefs_file).read().strip().splitlines()
+        self.assertTrue(any(line.startswith('#MozRunner Prefs Start') for line in lines))
+        self.assertTrue(any(line.startswith('#MozRunner Prefs End') for line in lines))
+
+        profile.reset()
+        self.assertNotEqual(prefs1,
+                            Preferences.read_prefs(os.path.join(profile.profile, 'user.js')),
+                            "I pity the fool who left my pref")
+
+    def test_reset_should_keep_user_added_prefs(self):
+        """Check that when we call reset the items we expect are updated"""
+        profile = Profile()
+        prefs_file = os.path.join(profile.profile, 'user.js')
+
+        # we shouldn't have any initial preferences
+        initial_prefs = Preferences.read_prefs(prefs_file)
+        self.assertFalse(initial_prefs)
+        initial_prefs = file(prefs_file).read().strip()
+        self.assertFalse(initial_prefs)
+
+        # add some preferences
+        prefs1 = [("mr.t.quotes", "i aint getting on no plane!")]
+        profile.set_persistent_preferences(prefs1)
+        self.assertEqual(prefs1, Preferences.read_prefs(prefs_file))
+        lines = file(prefs_file).read().strip().splitlines()
+        self.assertTrue(any(line.startswith('#MozRunner Prefs Start') for line in lines))
+        self.assertTrue(any(line.startswith('#MozRunner Prefs End') for line in lines))
+
+        profile.reset()
+        self.assertEqual(prefs1,
+                         Preferences.read_prefs(os.path.join(profile.profile, 'user.js')),
+                         "I pity the fool who left my pref")
+
+    def test_magic_markers(self):
+        """ensure our magic markers are working"""
+
+        profile = Profile()
+        prefs_file = os.path.join(profile.profile, 'user.js')
+
+        # we shouldn't have any initial preferences
+        initial_prefs = Preferences.read_prefs(prefs_file)
+        self.assertFalse(initial_prefs)
+        initial_prefs = file(prefs_file).read().strip()
+        self.assertFalse(initial_prefs)
+
+        # add some preferences
+        prefs1 = [("browser.startup.homepage", "http://planet.mozilla.org/"),
+                  ("zoom.minPercent", 30)]
+        profile.set_preferences(prefs1)
+        self.assertEqual(prefs1, Preferences.read_prefs(prefs_file))
+        lines = file(prefs_file).read().strip().splitlines()
+        self.assertTrue(bool([line for line in lines
+                              if line.startswith('#MozRunner Prefs Start')]))
+        self.assertTrue(bool([line for line in lines
+                              if line.startswith('#MozRunner Prefs End')]))
+
+        # add some more preferences
+        prefs2 = [("zoom.maxPercent", 300),
+                  ("webgl.verbose", 'false')]
+        profile.set_preferences(prefs2)
+        self.assertEqual(prefs1 + prefs2, Preferences.read_prefs(prefs_file))
+        lines = file(prefs_file).read().strip().splitlines()
+        self.assertTrue(len([line for line in lines
+                             if line.startswith('#MozRunner Prefs Start')]) == 2)
+        self.assertTrue(len([line for line in lines
+                             if line.startswith('#MozRunner Prefs End')]) == 2)
+
+        # now clean it up
+        profile.clean_preferences()
+        final_prefs = Preferences.read_prefs(prefs_file)
+        self.assertFalse(final_prefs)
+        lines = file(prefs_file).read().strip().splitlines()
+        self.assertTrue('#MozRunner Prefs Start' not in lines)
+        self.assertTrue('#MozRunner Prefs End' not in lines)
+
+    def test_preexisting_preferences(self):
+        """ensure you don't clobber preexisting preferences"""
+
+        # make a pretend profile
+        tempdir = tempfile.mkdtemp()
+
+        try:
+            # make a user.js
+            contents = """
+user_pref("webgl.enabled_for_all_sites", true);
+user_pref("webgl.force-enabled", true);
+"""
+            user_js = os.path.join(tempdir, 'user.js')
+            f = file(user_js, 'w')
+            f.write(contents)
+            f.close()
+
+            # make sure you can read it
+            prefs = Preferences.read_prefs(user_js)
+            original_prefs = [('webgl.enabled_for_all_sites', True), ('webgl.force-enabled', True)]
+            self.assertTrue(prefs == original_prefs)
+
+            # now read this as a profile
+            profile = Profile(tempdir, preferences={"browser.download.dir": "/home/jhammel"})
+
+            # make sure the new pref is now there
+            new_prefs = original_prefs[:] + [("browser.download.dir", "/home/jhammel")]
+            prefs = Preferences.read_prefs(user_js)
+            self.assertTrue(prefs == new_prefs)
+
+            # clean up the added preferences
+            profile.cleanup()
+            del profile
+
+            # make sure you have the original preferences
+            prefs = Preferences.read_prefs(user_js)
+            self.assertTrue(prefs == original_prefs)
+        finally:
+            shutil.rmtree(tempdir)
+
+    def test_can_read_prefs_with_multiline_comments(self):
+        """
+        Ensure that multiple comments in the file header do not break reading
+        the prefs (https://bugzilla.mozilla.org/show_bug.cgi?id=1233534).
+        """
+        user_js = tempfile.NamedTemporaryFile(suffix='.js', delete=False)
+        self.addCleanup(mozfile.remove, user_js.name)
+        with user_js:
+            user_js.write("""
+# Mozilla User Preferences
+
+/* Do not edit this file.
+ *
+ * If you make changes to this file while the application is running,
+ * the changes will be overwritten when the application exits.
+ *
+ * To make a manual change to preferences, you can visit the URL about:config
+ */
+
+user_pref("webgl.enabled_for_all_sites", true);
+user_pref("webgl.force-enabled", true);
+""")
+        self.assertEqual(
+            Preferences.read_prefs(user_js.name),
+            [('webgl.enabled_for_all_sites', True),
+             ('webgl.force-enabled', True)]
+        )
+
+    def test_json(self):
+        _prefs = {"browser.startup.homepage": "http://planet.mozilla.org/"}
+        json = '{"browser.startup.homepage": "http://planet.mozilla.org/"}'
+
+        # just repr it...could use the json module but we don't need it here
+        with mozfile.NamedTemporaryFile(suffix='.json') as f:
+            f.write(json)
+            f.flush()
+
+            commandline = ["--preferences", f.name]
+            self.compare_generated(_prefs, commandline)
+
+    def test_prefs_write(self):
+        """test that the Preferences.write() method correctly serializes preferences"""
+
+        _prefs = {'browser.startup.homepage': "http://planet.mozilla.org",
+                  'zoom.minPercent': 30,
+                  'zoom.maxPercent': 300}
+
+        # make a Preferences manager with the testing preferences
+        preferences = Preferences(_prefs)
+
+        # write them to a temporary location
+        path = None
+        read_prefs = None
+        try:
+            with mozfile.NamedTemporaryFile(suffix='.js', delete=False) as f:
+                path = f.name
+                preferences.write(f, _prefs)
+
+            # read them back and ensure we get what we put in
+            read_prefs = dict(Preferences.read_prefs(path))
+
+        finally:
+            # cleanup
+            if path and os.path.exists(path):
+                os.remove(path)
+
+        self.assertEqual(read_prefs, _prefs)
+
+    def test_read_prefs_with_comments(self):
+        """test reading preferences from a prefs.js file that contains comments"""
+
+        path = os.path.join(here, 'files', 'prefs_with_comments.js')
+        self.assertEqual(dict(Preferences.read_prefs(path)), self._prefs_with_comments)
+
+    def test_read_prefs_with_interpolation(self):
+        """test reading preferences from a prefs.js file whose values
+        require interpolation"""
+
+        expected_prefs = {
+            "browser.foo": "http://server-name",
+            "zoom.minPercent": 30,
+            "webgl.verbose": "false",
+            "browser.bar": "somethingxyz"
+        }
+        values = {
+            "server": "server-name",
+            "abc": "something"
+        }
+        path = os.path.join(here, 'files', 'prefs_with_interpolation.js')
+        read_prefs = Preferences.read_prefs(path, interpolation=values)
+        self.assertEqual(dict(read_prefs), expected_prefs)
+
+    def test_read_prefs_ttw(self):
+        """test reading preferences through the web via mozhttpd"""
+
+        # create a MozHttpd instance
+        docroot = os.path.join(here, 'files')
+        host = '127.0.0.1'
+        port = 8888
+        httpd = mozhttpd.MozHttpd(host=host, port=port, docroot=docroot)
+
+        # create a preferences instance
+        prefs = Preferences()
+
+        try:
+            # start server
+            httpd.start(block=False)
+
+            # read preferences through the web
+            read = prefs.read_prefs('http://%s:%d/prefs_with_comments.js' % (host, port))
+            self.assertEqual(dict(read), self._prefs_with_comments)
+        finally:
+            httpd.stop()
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_profile.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+import os
+from mozprofile import Profile
+
+
+class TestProfile(unittest.TestCase):
+
+    def test_with_profile_should_cleanup(self):
+        with Profile() as profile:
+            self.assertTrue(os.path.exists(profile.profile))
+        # profile is cleaned
+        self.assertFalse(os.path.exists(profile.profile))
+
+    def test_with_profile_should_cleanup_even_on_exception(self):
+        with self.assertRaises(ZeroDivisionError):
+            with Profile() as profile:
+                self.assertTrue(os.path.exists(profile.profile))
+                1 / 0  # will raise ZeroDivisionError
+        # profile is cleaned
+        self.assertFalse(os.path.exists(profile.profile))
+
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_profile_view.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import mozfile
+import mozprofile
+import os
+import tempfile
+import unittest
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+class TestProfilePrint(unittest.TestCase):
+
+    def test_profileprint(self):
+        """
+        test the summary function
+        """
+
+        keys = set(['Files', 'Path', 'user.js'])
+        ff_prefs = mozprofile.FirefoxProfile.preferences  # shorthand
+        pref_string = '\n'.join(['%s: %s' % (key, ff_prefs[key])
+                                 for key in sorted(ff_prefs.keys())])
+
+        tempdir = tempfile.mkdtemp()
+        try:
+            profile = mozprofile.FirefoxProfile(tempdir)
+            parts = profile.summary(return_parts=True)
+            parts = dict(parts)
+
+            self.assertEqual(parts['Path'], tempdir)
+            self.assertEqual(set(parts.keys()), keys)
+            self.assertEqual(pref_string, parts['user.js'].strip())
+
+        except:
+            raise
+        finally:
+            mozfile.rmtree(tempdir)
+
+    def test_strcast(self):
+        """
+        test casting to a string
+        """
+
+        profile = mozprofile.Profile()
+        self.assertEqual(str(profile), profile.summary())
+
+    def test_profile_diff(self):
+        profile1 = mozprofile.Profile()
+        profile2 = mozprofile.Profile(preferences=dict(foo='bar'))
+
+        # diff a profile against itself; no difference
+        self.assertEqual([], mozprofile.diff(profile1, profile1))
+
+        # diff two profiles
+        diff = dict(mozprofile.diff(profile1, profile2))
+        self.assertEqual(diff.keys(), ['user.js'])
+        lines = [line.strip() for line in diff['user.js'].splitlines()]
+        self.assertTrue('+foo: bar' in lines)
+
+        # diff a blank vs FirefoxProfile
+        ff_profile = mozprofile.FirefoxProfile()
+        diff = dict(mozprofile.diff(profile2, ff_profile))
+        self.assertEqual(diff.keys(), ['user.js'])
+        lines = [line.strip() for line in diff['user.js'].splitlines()]
+        self.assertTrue('-foo: bar' in lines)
+        ff_pref_lines = ['+%s: %s' % (key, value)
+                         for key, value in mozprofile.FirefoxProfile.preferences.items()]
+        self.assertTrue(set(ff_pref_lines).issubset(lines))
+
+if __name__ == '__main__':
+    unittest.main()

testing/mozbase/mozprofile/tests/test_webapps.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python
+
+"""
+test installing and managing webapps in a profile
+"""
+
+import os
+import shutil
+import unittest
+from tempfile import mkdtemp
+
+from mozprofile.webapps import WebappCollection, Webapp, WebappFormatException
+
+here = os.path.dirname(os.path.abspath(__file__))
+
+
+class WebappTest(unittest.TestCase):
+    """Tests reading, installing and cleaning webapps
+    from a profile.
+    """
+    manifest_path_1 = os.path.join(here, 'files', 'webapps1.json')
+    manifest_path_2 = os.path.join(here, 'files', 'webapps2.json')
+
+    def setUp(self):
+        self.profile = mkdtemp(prefix='test_webapp')
+        self.webapps_dir = os.path.join(self.profile, 'webapps')
+        self.webapps_json_path = os.path.join(self.webapps_dir, 'webapps.json')
+
+    def tearDown(self):
+        shutil.rmtree(self.profile)
+
+    def test_read_json_manifest(self):
+        """Tests WebappCollection.read_json"""
+        # Parse a list of webapp objects and verify it worked
+        manifest_json_1 = WebappCollection.read_json(self.manifest_path_1)
+        self.assertEqual(len(manifest_json_1), 7)
+        for app in manifest_json_1:
+            self.assertIsInstance(app, Webapp)
+            for key in Webapp.required_keys:
+                self.assertIn(key, app)
+
+        # Parse a dictionary of webapp objects and verify it worked
+        manifest_json_2 = WebappCollection.read_json(self.manifest_path_2)
+        self.assertEqual(len(manifest_json_2), 5)
+        for app in manifest_json_2:
+            self.assertIsInstance(app, Webapp)
+            for key in Webapp.required_keys:
+                self.assertIn(key, app)
+
+    def test_invalid_webapp(self):
+        """Tests a webapp with a missing required key"""
+        webapps = WebappCollection(self.profile)
+        # Missing the required key "description", exception should be raised
+        self.assertRaises(WebappFormatException, webapps.append, {'name': 'foo'})
+
+    def test_webapp_collection(self):
+        """Tests the methods of the WebappCollection object"""
+        webapp_1 = {'name': 'test_app_1',
+                    'description': 'a description',
+                    'manifestURL': 'http://example.com/1/manifest.webapp',
+                    'appStatus': 1}
+
+        webapp_2 = {'name': 'test_app_2',
+                    'description': 'another description',
+                    'manifestURL': 'http://example.com/2/manifest.webapp',
+                    'appStatus': 2}
+
+        webapp_3 = {'name': 'test_app_2',
+                    'description': 'a third description',
+                    'manifestURL': 'http://example.com/3/manifest.webapp',
+                    'appStatus': 3}
+
+        webapps = WebappCollection(self.profile)
+        self.assertEqual(len(webapps), 0)
+
+        # WebappCollection should behave like a list
+        def invalid_index():
+            webapps[0]
+        self.assertRaises(IndexError, invalid_index)
+
+        # Append a webapp object
+        webapps.append(webapp_1)
+        self.assertTrue(len(webapps), 1)
+        self.assertIsInstance(webapps[0], Webapp)
+        self.assertEqual(len(webapps[0]), len(webapp_1))
+        self.assertEqual(len(set(webapps[0].items()) & set(webapp_1.items())), len(webapp_1))
+
+        # Remove a webapp object
+        webapps.remove(webapp_1)
+        self.assertEqual(len(webapps), 0)
+
+        # Extend a list of webapp objects
+        webapps.extend([webapp_1, webapp_2])
+        self.assertEqual(len(webapps), 2)
+        self.assertTrue(webapp_1 in webapps)
+        self.assertTrue(webapp_2 in webapps)
+        self.assertNotEquals(webapps[0], webapps[1])
+
+        # Insert a webapp object
+        webapps.insert(1, webapp_3)
+        self.assertEqual(len(webapps), 3)
+        self.assertEqual(webapps[1], webapps[2])
+        for app in webapps:
+            self.assertIsInstance(app, Webapp)
+
+        # Assigning an invalid type (must be accepted by the dict() constructor) should throw
+        def invalid_type():
+            webapps[2] = 1
+        self.assertRaises(WebappFormatException, invalid_type)
+
+    def test_install_webapps(self):
+        """Test installing webapps into a profile that has no prior webapps"""
+        webapps = WebappCollection(self.profile, apps=self.manifest_path_1)
+        self.assertFalse(os.path.exists(self.webapps_dir))
+
+        # update the webapp manifests for the first time
+        webapps.update_manifests()
+        self.assertFalse(os.path.isdir(os.path.join(self.profile, webapps.backup_dir)))
+        self.assertTrue(os.path.isfile(self.webapps_json_path))
+
+        webapps_json = webapps.read_json(self.webapps_json_path, description="fake description")
+        self.assertEqual(len(webapps_json), 7)
+        for app in webapps_json:
+            self.assertIsInstance(app, Webapp)
+
+        manifest_json_1 = webapps.read_json(self.manifest_path_1)
+        manifest_json_2 = webapps.read_json(self.manifest_path_2)
+        self.assertEqual(len(webapps_json), len(manifest_json_1))
+        for app in webapps_json:
+            self.assertTrue(app in manifest_json_1)
+
+        # Remove one of the webapps from WebappCollection after it got installed
+        removed_app = manifest_json_1[2]
+        webapps.remove(removed_app)
+        # Add new webapps to the collection
+        webapps.extend(manifest_json_2)
+
+        # update the webapp manifests a second time
+        webapps.update_manifests()
+        self.assertFalse(os.path.isdir(os.path.join(self.profile, webapps.backup_dir)))
+        self.assertTrue(os.path.isfile(self.webapps_json_path))
+
+        webapps_json = webapps.read_json(self.webapps_json_path, description="a description")
+        self.assertEqual(len(webapps_json), 11)
+
+        # The new apps should be added
+        for app in webapps_json:
+            self.assertIsInstance(app, Webapp)
+            self.assertTrue(os.path.isfile(os.path.join(self.webapps_dir, app['name'],
+                                                        'manifest.webapp')))
+        # The removed app should not exist in the manifest
+        self.assertNotIn(removed_app, webapps_json)
+        self.assertFalse(os.path.exists(os.path.join(self.webapps_dir, removed_app['name'])))
+
+        # Cleaning should delete the webapps directory entirely
+        # since there was nothing there before
+        webapps.clean()
+        self.assertFalse(os.path.isdir(self.webapps_dir))
+
+    def test_install_webapps_preexisting(self):
+        """Tests installing webapps when the webapps directory already exists"""
+        manifest_json_2 = WebappCollection.read_json(self.manifest_path_2)
+
+        # Synthesize a pre-existing webapps directory
+        os.mkdir(self.webapps_dir)
+        shutil.copyfile(self.manifest_path_2, self.webapps_json_path)
+        for app in manifest_json_2:
+            app_path = os.path.join(self.webapps_dir, app['name'])
+            os.mkdir(app_path)
+            f = open(os.path.join(app_path, 'manifest.webapp'), 'w')
+            f.close()
+
+        webapps = WebappCollection(self.profile, apps=self.manifest_path_1)
+        self.assertTrue(os.path.exists(self.webapps_dir))
+
+        # update webapp manifests for the first time
+        webapps.update_manifests()
+        # A backup should be created
+        self.assertTrue(os.path.isdir(os.path.join(self.profile, webapps.backup_dir)))
+
+        # Both manifests should remain installed
+        webapps_json = webapps.read_json(self.webapps_json_path, description='a fake description')
+        self.assertEqual(len(webapps_json), 12)
+        for app in webapps_json:
+            self.assertIsInstance(app, Webapp)
+            self.assertTrue(os.path.isfile(os.path.join(self.webapps_dir, app['name'],
+                                                        'manifest.webapp')))
+
+        # Upon cleaning the backup should be restored
+        webapps.clean()
+        self.assertFalse(os.path.isdir(os.path.join(self.profile, webapps.backup_dir)))
+
+        # The original webapps should still be installed
+        webapps_json = webapps.read_json(self.webapps_json_path)
+        for app in webapps_json:
+            self.assertIsInstance(app, Webapp)
+            self.assertTrue(os.path.isfile(os.path.join(self.webapps_dir, app['name'],
+                                                        'manifest.webapp')))
+        self.assertEqual(webapps_json, manifest_json_2)
+
+if __name__ == '__main__':
+    unittest.main()