import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

testing/mozharness/scripts/release/antivirus.py

@@ -0,0 +1,193 @@
+from multiprocessing.pool import ThreadPool
+import os
+import re
+import sys
+import shutil
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+
+
+class AntivirusScan(BaseScript, VirtualenvMixin):
+    config_options = [
+        [["--product"], {
+            "dest": "product",
+            "help": "Product being released, eg: firefox, thunderbird",
+        }],
+        [["--version"], {
+            "dest": "version",
+            "help": "Version of release, eg: 39.0b5",
+        }],
+        [["--build-number"], {
+            "dest": "build_number",
+            "help": "Build number of release, eg: 2",
+        }],
+        [["--bucket-name"], {
+            "dest": "bucket_name",
+            "help": "S3 Bucket to retrieve files from",
+        }],
+        [["--exclude"], {
+            "dest": "excludes",
+            "action": "append",
+            "help": "List of filename patterns to exclude. See script source for default",
+        }],
+        [["-d", "--download-parallelization"], {
+            "dest": "download_parallelization",
+            "default": 6,
+            "type": "int",
+            "help": "Number of concurrent file downloads",
+        }],
+        [["-s", "--scan-parallelization"], {
+            "dest": "scan_parallelization",
+            "default": 4,
+            "type": "int",
+            "help": "Number of concurrent file scans",
+        }],
+        [["--tools-repo"], {
+            "dest": "tools_repo",
+            "default": "https://hg.mozilla.org/build/tools",
+        }],
+        [["--tools-revision"], {
+            "dest": "tools_revision",
+            "help": "Revision of tools repo to use when downloading extract_and_run_command.py",
+        }],
+        ] + virtualenv_config_options
+
+    DEFAULT_EXCLUDES = [
+        r"^.*tests.*$",
+        r"^.*crashreporter.*$",
+        r"^.*\.zip(\.asc)?$",
+        r"^.*\.log$",
+        r"^.*\.txt$",
+        r"^.*\.asc$",
+        r"^.*/partner-repacks.*$",
+        r"^.*.checksums(\.asc)?$",
+        r"^.*/logs/.*$",
+        r"^.*/jsshell.*$",
+        r"^.*json$",
+        r"^.*/host.*$",
+        r"^.*/mar-tools/.*$",
+        r"^.*robocop.apk$",
+        r"^.*contrib.*"
+    ]
+    CACHE_DIR = 'cache'
+
+    def __init__(self):
+        BaseScript.__init__(self,
+            config_options=self.config_options,
+            require_config_file=False,
+            config={
+                "virtualenv_modules": [
+                    "boto",
+                    "redo",
+                    "mar",
+                ],
+                "virtualenv_path": "venv",
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "get-extract-script",
+                "get-files",
+                "scan-files",
+                "cleanup-cache",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "get-extract-script",
+                "get-files",
+                "scan-files",
+                "cleanup-cache",
+            ],
+        )
+        self.excludes = self.config.get('excludes', self.DEFAULT_EXCLUDES)
+        self.dest_dir = self.CACHE_DIR
+
+    def _get_candidates_prefix(self):
+        return "pub/{}/candidates/{}-candidates/build{}/".format(
+            self.config['product'],
+            self.config["version"],
+            self.config["build_number"]
+        )
+
+    def _matches_exclude(self, keyname):
+        for exclude in self.excludes:
+            if re.search(exclude, keyname):
+                return True
+        return False
+
+    def get_extract_script(self):
+        """Gets a copy of extract_and_run_command.py from tools, and the supporting mar.py,
+        so that we can unpack various files for clam to scan them."""
+        remote_file = "{}/raw-file/{}/stage/extract_and_run_command.py".format(self.config["tools_repo"],
+                                                                               self.config["tools_revision"])
+        self.download_file(remote_file, file_name="extract_and_run_command.py")
+
+    def get_files(self):
+        """Pull the candidate files down from S3 for scanning, using parallel requests"""
+        from boto.s3.connection import S3Connection
+        from boto.exception import S3CopyError, S3ResponseError
+        from redo import retry
+        from httplib import HTTPException
+
+        # suppress boto debug logging, it's too verbose with --loglevel=debug
+        import logging
+        logging.getLogger('boto').setLevel(logging.INFO)
+
+        self.info("Connecting to S3")
+        conn = S3Connection(anon=True)
+        self.info("Getting bucket {}".format(self.config["bucket_name"]))
+        bucket = conn.get_bucket(self.config["bucket_name"])
+
+        if os.path.exists(self.dest_dir):
+            self.info('Emptying {}'.format(self.dest_dir))
+            shutil.rmtree(self.dest_dir)
+        os.makedirs(self.dest_dir)
+
+        def worker(item):
+            source, destination = item
+
+            self.info("Downloading {} to {}".format(source, destination))
+            key = bucket.get_key(source)
+            return retry(key.get_contents_to_filename,
+                         args=(destination, ),
+                         sleeptime=30, max_sleeptime=150,
+                         retry_exceptions=(S3CopyError, S3ResponseError,
+                                           IOError, HTTPException))
+
+        def find_release_files():
+            candidates_prefix = self._get_candidates_prefix()
+            self.info("Getting key names from candidates")
+            for key in bucket.list(prefix=candidates_prefix):
+                keyname = key.name
+                if self._matches_exclude(keyname):
+                    self.debug("Excluding {}".format(keyname))
+                else:
+                    destination = os.path.join(self.dest_dir, keyname.replace(candidates_prefix, ''))
+                    dest_dir = os.path.dirname(destination)
+                    if not os.path.isdir(dest_dir):
+                        os.makedirs(dest_dir)
+                    yield (keyname, destination)
+
+        pool = ThreadPool(self.config["download_parallelization"])
+        pool.map(worker, find_release_files())
+
+    def scan_files(self):
+        """Scan the files we've collected. We do the download and scan concurrently to make
+        it easier to have a coherent log afterwards. Uses the venv python."""
+        self.run_command([self.query_python_path(), 'extract_and_run_command.py',
+                          '-j{}'.format(self.config['scan_parallelization']),
+                          'clamdscan', '-m', '--no-summary', '--', self.dest_dir])
+
+    def cleanup_cache(self):
+        """If we have simultaneous releases in flight an av slave may end up doing another
+        av job before being recycled, and we need to make sure the full disk is available."""
+        shutil.rmtree(self.dest_dir)
+
+
+if __name__ == "__main__":
+    myScript = AntivirusScan()
+    myScript.run_and_exit()

testing/mozharness/scripts/release/beet_mover.py

@@ -0,0 +1,372 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""beet_mover.py.
+
+downloads artifacts, scans them and uploads them to s3
+"""
+import hashlib
+import sys
+import os
+import pprint
+import re
+from os import listdir
+from os.path import isfile, join
+import sh
+import redo
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+from mozharness.base.log import FATAL
+from mozharness.base.python import VirtualenvMixin
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.aws import pop_aws_auth_from_env
+import mozharness
+import mimetypes
+
+
+def get_hash(content, hash_type="md5"):
+    h = hashlib.new(hash_type)
+    h.update(content)
+    return h.hexdigest()
+
+
+CONFIG_OPTIONS = [
+    [["--template"], {
+        "dest": "template",
+        "help": "Specify jinja2 template file",
+    }],
+    [['--locale', ], {
+        "action": "extend",
+        "dest": "locales",
+        "type": "string",
+        "help": "Specify the locale(s) to upload."}],
+    [["--platform"], {
+        "dest": "platform",
+        "help": "Specify the platform of the build",
+    }],
+    [["--version"], {
+        "dest": "version",
+        "help": "full release version based on gecko and tag/stage identifier. e.g. '44.0b1'"
+    }],
+    [["--app-version"], {
+        "dest": "app_version",
+        "help": "numbered version based on gecko. e.g. '44.0'"
+    }],
+    [["--partial-version"], {
+        "dest": "partial_version",
+        "help": "the partial version the mar is based off of"
+    }],
+    [["--artifact-subdir"], {
+        "dest": "artifact_subdir",
+        "default": 'build',
+        "help": "subdir location for taskcluster artifacts after public/ base.",
+    }],
+    [["--build-num"], {
+        "dest": "build_num",
+        "help": "the release build identifier"
+    }],
+    [["--taskid"], {
+        "dest": "taskid",
+        "help": "taskcluster task id to download artifacts from",
+    }],
+    [["--bucket"], {
+        "dest": "bucket",
+        "help": "s3 bucket to move beets to.",
+    }],
+    [["--product"], {
+        "dest": "product",
+        "help": "product for which artifacts are beetmoved",
+    }],
+    [["--exclude"], {
+        "dest": "excludes",
+        "action": "append",
+        "help": "List of filename patterns to exclude. See script source for default",
+    }],
+    [["-s", "--scan-parallelization"], {
+        "dest": "scan_parallelization",
+        "default": 4,
+        "type": "int",
+        "help": "Number of concurrent file scans",
+    }],
+]
+
+DEFAULT_EXCLUDES = [
+    r"^.*tests.*$",
+    r"^.*crashreporter.*$",
+    r"^.*\.zip(\.asc)?$",
+    r"^.*\.log$",
+    r"^.*\.txt$",
+    r"^.*\.asc$",
+    r"^.*/partner-repacks.*$",
+    r"^.*.checksums(\.asc)?$",
+    r"^.*/logs/.*$",
+    r"^.*/jsshell.*$",
+    r"^.*json$",
+    r"^.*/host.*$",
+    r"^.*/mar-tools/.*$",
+    r"^.*robocop.apk$",
+    r"^.*contrib.*"
+]
+CACHE_DIR = 'cache'
+
+MIME_MAP = {
+    '': 'text/plain',
+    '.asc': 'text/plain',
+    '.beet': 'text/plain',
+    '.bundle': 'application/octet-stream',
+    '.bz2': 'application/octet-stream',
+    '.checksums': 'text/plain',
+    '.dmg': 'application/x-iso9660-image',
+    '.mar': 'application/octet-stream',
+    '.xpi': 'application/x-xpinstall'
+}
+
+HASH_FORMATS = ["sha512", "sha256"]
+
+
+class BeetMover(BaseScript, VirtualenvMixin, object):
+    def __init__(self, aws_creds):
+        beetmover_kwargs = {
+            'config_options': CONFIG_OPTIONS,
+            'all_actions': [
+                # 'clobber',
+                'create-virtualenv',
+                'activate-virtualenv',
+                'generate-candidates-manifest',
+                'refresh-antivirus',
+                'verify-bits',  # beets
+                'download-bits', # beets
+                'scan-bits',     # beets
+                'upload-bits',  # beets
+            ],
+            'require_config_file': False,
+            # Default configuration
+            'config': {
+                # base index url where to find taskcluster artifact based on taskid
+                "artifact_base_url": 'https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/{subdir}',
+                "virtualenv_modules": [
+                    "boto",
+                    "PyYAML",
+                    "Jinja2",
+                    "redo",
+                    "cryptography==2.0.3",
+                    "mar",
+                ],
+                "virtualenv_path": "venv",
+            },
+        }
+        #todo do excludes need to be configured via command line for specific builds?
+        super(BeetMover, self).__init__(**beetmover_kwargs)
+
+        c = self.config
+        self.manifest = {}
+        # assigned in _post_create_virtualenv
+        self.virtualenv_imports = None
+        self.bucket = c['bucket']
+        if not all(aws_creds):
+            self.fatal('credentials must be passed in env: "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY"')
+        self.aws_key_id, self.aws_secret_key = aws_creds
+        # if excludes is set from command line, use it otherwise use defaults
+        self.excludes = self.config.get('excludes', DEFAULT_EXCLUDES)
+        dirs = self.query_abs_dirs()
+        self.dest_dir = os.path.join(dirs['abs_work_dir'], CACHE_DIR)
+        self.mime_fix()
+
+    def activate_virtualenv(self):
+        """
+        activates virtualenv and adds module imports to a instance wide namespace.
+
+        creating and activating a virtualenv onto the currently executing python interpreter is a
+        bit black magic. Rather than having import statements added in various places within the
+        script, we import them here immediately after we activate the newly created virtualenv
+        """
+        VirtualenvMixin.activate_virtualenv(self)
+
+        import boto
+        import yaml
+        import jinja2
+        self.virtualenv_imports = {
+            'boto': boto,
+            'yaml': yaml,
+            'jinja2': jinja2,
+        }
+        self.log("activated virtualenv with the modules: {}".format(str(self.virtualenv_imports)))
+
+    def _get_template_vars(self):
+        return {
+            "platform": self.config['platform'],
+            "locales": self.config.get('locales'),
+            "version": self.config['version'],
+            "app_version": self.config.get('app_version', ''),
+            "partial_version": self.config.get('partial_version', ''),
+            "build_num": self.config['build_num'],
+            # keep the trailing slash
+            "s3_prefix": 'pub/{prod}/candidates/{ver}-candidates/{n}/'.format(
+                prod=self.config['product'], ver=self.config['version'],
+                n=self.config['build_num']
+            ),
+            "artifact_base_url": self.config['artifact_base_url'].format(
+                    taskid=self.config['taskid'], subdir=self.config['artifact_subdir']
+            )
+        }
+
+    def generate_candidates_manifest(self):
+        """
+        generates and outputs a manifest that maps expected Taskcluster artifact names
+        to release deliverable names
+        """
+        self.log('generating manifest from {}...'.format(self.config['template']))
+        template_dir, template_file = os.path.split(os.path.abspath(self.config['template']))
+        jinja2 = self.virtualenv_imports['jinja2']
+        yaml = self.virtualenv_imports['yaml']
+
+        jinja_env = jinja2.Environment(loader=jinja2.FileSystemLoader(template_dir),
+                                       undefined=jinja2.StrictUndefined)
+        template = jinja_env.get_template(template_file)
+        self.manifest = yaml.safe_load(template.render(**self._get_template_vars()))
+
+        self.log("manifest generated:")
+        self.log(pprint.pformat(self.manifest['mapping']))
+
+    def verify_bits(self):
+        """
+        inspects each artifact and verifies that they were created by trustworthy tasks
+        """
+        # TODO
+        self.log('skipping verification. unimplemented...')
+
+    def refresh_antivirus(self):
+        self.info("Refreshing clamav db...")
+        try:
+            redo.retry(lambda:
+                       sh.freshclam("--stdout", "--verbose", _timeout=300,
+                                    _err_to_out=True))
+            self.info("Done.")
+        except sh.ErrorReturnCode:
+            self.warning("Freshclam failed, skipping DB update")
+
+    def download_bits(self):
+        """
+        downloads list of artifacts to self.dest_dir dir based on a given manifest
+        """
+        self.log('downloading and uploading artifacts to self_dest_dir...')
+        dirs = self.query_abs_dirs()
+
+        for locale in self.manifest['mapping']:
+            for deliverable in self.manifest['mapping'][locale]:
+                self.log("downloading '{}' deliverable for '{}' locale".format(deliverable, locale))
+                source = self.manifest['mapping'][locale][deliverable]['artifact']
+                self.retry(
+                    self.download_file,
+                    args=[source],
+                    kwargs={'parent_dir': dirs['abs_work_dir']},
+                    error_level=FATAL)
+        self.log('Success!')
+
+    def _strip_prefix(self, s3_key):
+        """Return file name relative to prefix"""
+        # "abc/def/hfg".split("abc/de")[-1] == "f/hfg"
+        return s3_key.split(self._get_template_vars()["s3_prefix"])[-1]
+
+    def upload_bits(self):
+        """
+        uploads list of artifacts to s3 candidates dir based on a given manifest
+        """
+        self.log('uploading artifacts to s3...')
+        dirs = self.query_abs_dirs()
+
+        # connect to s3
+        boto = self.virtualenv_imports['boto']
+        conn = boto.connect_s3(self.aws_key_id, self.aws_secret_key)
+        bucket = conn.get_bucket(self.bucket)
+
+        for locale in self.manifest['mapping']:
+            for deliverable in self.manifest['mapping'][locale]:
+                self.log("uploading '{}' deliverable for '{}' locale".format(deliverable, locale))
+                # we have already downloaded the files locally so we can use that version
+                source = self.manifest['mapping'][locale][deliverable]['artifact']
+                s3_key = self.manifest['mapping'][locale][deliverable]['s3_key']
+                downloaded_file = os.path.join(dirs['abs_work_dir'], self.get_filename_from_url(source))
+                # generate checksums for every uploaded file
+                beet_file_name = '{}.beet'.format(downloaded_file)
+                # upload checksums to a separate subdirectory
+                beet_dest = '{prefix}beetmover-checksums/{f}.beet'.format(
+                    prefix=self._get_template_vars()["s3_prefix"],
+                    f=self._strip_prefix(s3_key)
+                )
+                beet_contents = '\n'.join([
+                    '{hash} {fmt} {size} {name}'.format(
+                        hash=self.get_hash_for_file(downloaded_file, hash_type=fmt),
+                        fmt=fmt,
+                        size=os.path.getsize(downloaded_file),
+                        name=self._strip_prefix(s3_key)) for fmt in HASH_FORMATS
+                ])
+                self.write_to_file(beet_file_name, beet_contents)
+                self.upload_bit(source=downloaded_file, s3_key=s3_key,
+                                bucket=bucket)
+                self.upload_bit(source=beet_file_name, s3_key=beet_dest,
+                                bucket=bucket)
+        self.log('Success!')
+
+
+    def upload_bit(self, source, s3_key, bucket):
+        boto = self.virtualenv_imports['boto']
+        self.info('uploading to s3 with key: {}'.format(s3_key))
+        key = boto.s3.key.Key(bucket)  # create new key
+        key.key = s3_key  # set key name
+
+        self.info("Checking if `{}` already exists".format(s3_key))
+        key = bucket.get_key(s3_key)
+        if not key:
+            self.info("Uploading to `{}`".format(s3_key))
+            key = bucket.new_key(s3_key)
+            # set key value
+            mime_type, _ = mimetypes.guess_type(source)
+            self.retry(lambda: key.set_contents_from_filename(source, headers={'Content-Type': mime_type}),
+                       error_level=FATAL),
+        else:
+            if not get_hash(key.get_contents_as_string()) == get_hash(open(source).read()):
+                # for now, let's halt. If necessary, we can revisit this and allow for overwrites
+                #  to the same buildnum release with different bits
+                self.fatal("`{}` already exists with different checksum.".format(s3_key))
+            self.log("`{}` has the same MD5 checksum, not uploading".format(s3_key))
+
+    def scan_bits(self):
+
+        dirs = self.query_abs_dirs()
+
+        filenames = [f for f in listdir(dirs['abs_work_dir']) if isfile(join(dirs['abs_work_dir'], f))]
+        self.mkdir_p(self.dest_dir)
+        for file_name in filenames:
+            if self._matches_exclude(file_name):
+                self.info("Excluding {} from virus scan".format(file_name))
+            else:
+                self.info('Copying {} to {}'.format(file_name,self.dest_dir))
+                self.copyfile(os.path.join(dirs['abs_work_dir'], file_name), os.path.join(self.dest_dir,file_name))
+        self._scan_files()
+        self.info('Emptying {}'.format(self.dest_dir))
+        self.rmtree(self.dest_dir)
+
+    def _scan_files(self):
+        """Scan the files we've collected. We do the download and scan concurrently to make
+        it easier to have a coherent log afterwards. Uses the venv python."""
+        external_tools_path = os.path.join(
+                              os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))), 'external_tools')
+        self.run_command([self.query_python_path(), os.path.join(external_tools_path,'extract_and_run_command.py'),
+                         '-j{}'.format(self.config['scan_parallelization']),
+                         'clamscan', '--no-summary', '--', self.dest_dir])
+
+    def _matches_exclude(self, keyname):
+         return any(re.search(exclude, keyname) for exclude in self.excludes)
+
+    def mime_fix(self):
+        """ Add mimetypes for custom extensions """
+        mimetypes.init()
+        map(lambda (ext, mime_type,): mimetypes.add_type(mime_type, ext), MIME_MAP.items())
+
+if __name__ == '__main__':
+    beet_mover = BeetMover(pop_aws_auth_from_env())
+    beet_mover.run_and_exit()

testing/mozharness/scripts/release/generate-checksums.py

@@ -0,0 +1,284 @@
+from multiprocessing.pool import ThreadPool
+import os
+from os import path
+import re
+import sys
+import posixpath
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+from mozharness.base.vcs.vcsbase import VCSMixin
+from mozharness.mozilla.checksums import parse_checksums_file
+from mozharness.mozilla.signing import SigningMixin
+from mozharness.mozilla.buildbot import BuildbotMixin
+
+class ChecksumsGenerator(BaseScript, VirtualenvMixin, SigningMixin, VCSMixin, BuildbotMixin):
+    config_options = [
+        [["--stage-product"], {
+            "dest": "stage_product",
+            "help": "Name of product used in file server's directory structure, eg: firefox, mobile",
+        }],
+        [["--version"], {
+            "dest": "version",
+            "help": "Version of release, eg: 39.0b5",
+        }],
+        [["--build-number"], {
+            "dest": "build_number",
+            "help": "Build number of release, eg: 2",
+        }],
+        [["--bucket-name-prefix"], {
+            "dest": "bucket_name_prefix",
+            "help": "Prefix of bucket name, eg: net-mozaws-prod-delivery. This will be used to generate a full bucket name (such as net-mozaws-prod-delivery-{firefox,archive}.",
+        }],
+        [["--bucket-name-full"], {
+            "dest": "bucket_name_full",
+            "help": "Full bucket name, eg: net-mozaws-prod-delivery-firefox",
+        }],
+        [["-j", "--parallelization"], {
+            "dest": "parallelization",
+            "default": 20,
+            "type": int,
+            "help": "Number of checksums file to download concurrently",
+        }],
+        [["-f", "--format"], {
+            "dest": "formats",
+            "default": [],
+            "action": "append",
+            "help": "Format(s) to generate big checksums file for. Default: sha512",
+        }],
+        [["--include"], {
+            "dest": "includes",
+            "default": [],
+            "action": "append",
+            "help": "List of patterns to include in big checksums file. See script source for default.",
+        }],
+        [["--tools-repo"], {
+            "dest": "tools_repo",
+            "default": "https://hg.mozilla.org/build/tools",
+        }],
+        [["--credentials"], {
+            "dest": "credentials",
+            "help": "File containing access key and secret access key for S3",
+        }],
+    ] + virtualenv_config_options
+
+    def __init__(self):
+        BaseScript.__init__(self,
+            config_options=self.config_options,
+            require_config_file=False,
+            config={
+                "virtualenv_modules": [
+                    "pip==1.5.5",
+                    "boto",
+                ],
+                "virtualenv_path": "venv",
+                'buildbot_json_path': 'buildprops.json',
+            },
+            all_actions=[
+                "create-virtualenv",
+                "collect-individual-checksums",
+                "create-big-checksums",
+                "sign",
+                "upload",
+                "copy-info-files",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "collect-individual-checksums",
+                "create-big-checksums",
+                "sign",
+                "upload",
+            ],
+        )
+
+        self.checksums = {}
+        self.bucket = None
+        self.bucket_name = self._get_bucket_name()
+        self.file_prefix = self._get_file_prefix()
+        # set the env var for boto to read our special config file
+        # rather than anything else we have at ~/.boto
+        os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"])
+
+    def _pre_config_lock(self, rw_config):
+        super(ChecksumsGenerator, self)._pre_config_lock(rw_config)
+
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        # TODO: version should come from repo
+        props = self.buildbot_config["properties"]
+        for prop in ['version', 'build_number']:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+
+        # These defaults are set here rather in the config because default
+        # lists cannot be completely overidden, only appended to.
+        if not self.config.get("formats"):
+            self.config["formats"] = ["sha512", "sha256"]
+
+        if not self.config.get("includes"):
+            self.config["includes"] = [
+                r"^.*\.tar\.bz2$",
+                r"^.*\.tar\.xz$",
+                r"^.*\.dmg$",
+                r"^.*\.bundle$",
+                r"^.*\.mar$",
+                r"^.*Setup.*\.exe$",
+                r"^.*\.xpi$",
+                r"^.*fennec.*\.apk$",
+            ]
+
+    def _get_bucket_name(self):
+        if self.config.get('bucket_name_full'):
+            return self.config['bucket_name_full']
+
+        suffix = "archive"
+        # Firefox has a special bucket, per https://github.com/mozilla-services/product-delivery-tools/blob/master/bucketmap.go
+        if self.config["stage_product"] == "firefox":
+            suffix = "firefox"
+
+        return "{}-{}".format(self.config["bucket_name_prefix"], suffix)
+
+    def _get_file_prefix(self):
+        return "pub/{}/candidates/{}-candidates/build{}/".format(
+            self.config["stage_product"], self.config["version"], self.config["build_number"]
+        )
+
+    def _get_sums_filename(self, format_):
+        return "{}SUMS".format(format_.upper())
+
+    def _get_bucket(self):
+        if not self.bucket:
+            self.activate_virtualenv()
+            from boto.s3.connection import S3Connection
+
+            self.info("Connecting to S3")
+            conn = S3Connection()
+            self.debug("Successfully connected to S3")
+            self.info("Connecting to bucket {}".format(self.bucket_name))
+            self.bucket = conn.get_bucket(self.bucket_name)
+        return self.bucket
+
+    def collect_individual_checksums(self):
+        """This step grabs all of the small checksums files for the release,
+        filters out any unwanted files from within them, and adds the remainder
+        to self.checksums for subsequent steps to use."""
+        bucket = self._get_bucket()
+        self.info("File prefix is: {}".format(self.file_prefix))
+
+        # Temporary holding place for checksums
+        raw_checksums = []
+        def worker(item):
+            self.debug("Downloading {}".format(item))
+            # TODO: It would be nice to download the associated .asc file
+            # and verify against it.
+            sums = bucket.get_key(item).get_contents_as_string()
+            raw_checksums.append(sums)
+
+        def find_checksums_files():
+            self.info("Getting key names from bucket")
+            checksum_files = {"beets": [], "checksums": []}
+            for key in bucket.list(prefix=self.file_prefix):
+                if key.key.endswith(".checksums"):
+                    self.debug("Found checksums file: {}".format(key.key))
+                    checksum_files["checksums"].append(key.key)
+                elif key.key.endswith(".beet"):
+                    self.debug("Found beet file: {}".format(key.key))
+                    checksum_files["beets"].append(key.key)
+                else:
+                    self.debug("Ignoring non-checksums file: {}".format(key.key))
+            if checksum_files["beets"]:
+                self.log("Using beet format")
+                return checksum_files["beets"]
+            else:
+                self.log("Using checksums format")
+                return checksum_files["checksums"]
+
+        pool = ThreadPool(self.config["parallelization"])
+        pool.map(worker, find_checksums_files())
+
+        for c in raw_checksums:
+            for f, info in parse_checksums_file(c).iteritems():
+                for pattern in self.config["includes"]:
+                    if re.search(pattern, f):
+                        if f in self.checksums:
+                            self.fatal("Found duplicate checksum entry for {}, don't know which one to pick.".format(f))
+                        if not set(self.config["formats"]) <= set(info["hashes"]):
+                            self.fatal("Missing necessary format for file {}".format(f))
+                        self.debug("Adding checksums for file: {}".format(f))
+                        self.checksums[f] = info
+                        break
+                else:
+                    self.debug("Ignoring checksums for file: {}".format(f))
+
+    def create_big_checksums(self):
+        for fmt in self.config["formats"]:
+            sums = self._get_sums_filename(fmt)
+            self.info("Creating big checksums file: {}".format(sums))
+            with open(sums, "w+") as output_file:
+                for fn in sorted(self.checksums):
+                    output_file.write("{}  {}\n".format(self.checksums[fn]["hashes"][fmt], fn))
+
+    def sign(self):
+        dirs = self.query_abs_dirs()
+
+        tools_dir = path.join(dirs["abs_work_dir"], "tools")
+        self.vcs_checkout(
+            repo=self.config["tools_repo"],
+            branch="default",
+            vcs="hg",
+            dest=tools_dir,
+        )
+
+        sign_cmd = self.query_moz_sign_cmd(formats=["gpg"])
+
+        for fmt in self.config["formats"]:
+            sums = self._get_sums_filename(fmt)
+            self.info("Signing big checksums file: {}".format(sums))
+            retval = self.run_command(sign_cmd + [sums])
+            if retval != 0:
+                self.fatal("Failed to sign {}".format(sums))
+
+    def upload(self):
+        # we need to provide the public side of the gpg key so that people can
+        # verify the detached signatures
+        dirs = self.query_abs_dirs()
+        tools_dir = path.join(dirs["abs_work_dir"], "tools")
+        self.copyfile(os.path.join(tools_dir, 'scripts', 'release', 'KEY'),
+                      'KEY')
+        files = ['KEY']
+
+        for fmt in self.config["formats"]:
+            files.append(self._get_sums_filename(fmt))
+            files.append("{}.asc".format(self._get_sums_filename(fmt)))
+
+        bucket = self._get_bucket()
+        for f in files:
+            dest = posixpath.join(self.file_prefix, f)
+            self.info("Uploading {} to {}".format(f, dest))
+            key = bucket.new_key(dest)
+            key.set_contents_from_filename(f, headers={'Content-Type': 'text/plain'})
+
+    def copy_info_files(self):
+        bucket = self._get_bucket()
+
+        for key in bucket.list(prefix=self.file_prefix):
+            if re.search(r'/en-US/android.*_info\.txt$', key.name):
+                self.info("Found      {}".format(key.name))
+                dest = posixpath.join(self.file_prefix, posixpath.basename(key.name))
+                self.info("Copying to {}".format(dest))
+                bucket.copy_key(new_key_name=dest,
+                                src_bucket_name=self.bucket_name,
+                                src_key_name=key.name,
+                                metadata={'Content-Type': 'text/plain'})
+
+
+if __name__ == "__main__":
+    myScript = ChecksumsGenerator()
+    myScript.run_and_exit()

testing/mozharness/scripts/release/postrelease_bouncer_aliases.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" postrelease_bouncer_aliases.py
+
+A script to replace the old-fashion way of updating the bouncer aliaes through
+tools script.
+"""
+
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+
+
+# PostReleaseBouncerAliases {{{1
+class PostReleaseBouncerAliases(BaseScript, VirtualenvMixin, BuildbotMixin):
+    config_options = virtualenv_config_options
+
+    def __init__(self, require_config_file=True):
+        super(PostReleaseBouncerAliases, self).__init__(
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            config={
+                "virtualenv_modules": [
+                    "redo",
+                    "requests",
+                ],
+                "virtualenv_path": "venv",
+                'credentials_file': 'oauth.txt',
+                'buildbot_json_path': 'buildprops.json',
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "update-bouncer-aliases",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "update-bouncer-aliases",
+            ],
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(PostReleaseBouncerAliases, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        props = self.buildbot_config["properties"]
+        for prop in ['tuxedo_server_url', 'version']:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+            else:
+                self.warning("%s could not be found within buildprops" % prop)
+                return
+
+    def _update_bouncer_alias(self, tuxedo_server_url, auth,
+                              related_product, alias):
+        from redo import retry
+        import requests
+
+        url = "%s/create_update_alias" % tuxedo_server_url
+        data = {"alias": alias, "related_product": related_product}
+        self.log("Updating {} to point to {} using {}".format(alias,
+                                                              related_product,
+                                                              url))
+
+        # Wrap the real call to hide credentials from retry's logging
+        def do_update_bouncer_alias():
+            r = requests.post(url, data=data, auth=auth,
+                              verify=False, timeout=60)
+            r.raise_for_status()
+
+        retry(do_update_bouncer_alias)
+
+    def update_bouncer_aliases(self):
+        tuxedo_server_url = self.config['tuxedo_server_url']
+        credentials_file = os.path.join(os.getcwd(),
+                                        self.config['credentials_file'])
+        credentials = {}
+        execfile(credentials_file, credentials)
+        auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
+        version = self.config['version']
+        for product, info in self.config["products"].iteritems():
+            if "alias" in info:
+                product_template = info["product-name"]
+                related_product = product_template % {"version": version}
+                self._update_bouncer_alias(tuxedo_server_url, auth,
+                                           related_product, info["alias"])
+
+
+# __main__ {{{1
+if __name__ == '__main__':
+    PostReleaseBouncerAliases().run_and_exit()

testing/mozharness/scripts/release/postrelease_mark_as_shipped.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" postrelease_mark_as_shipped.py
+
+A script to automate the manual way of updating a release as shipped in Ship-it
+following its successful ship-to-the-door opertion.
+"""
+import os
+import sys
+from datetime import datetime
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+
+
+def build_release_name(product, version, buildno):
+    """Function to reconstruct the name of the release based on product,
+    version and buildnumber
+    """
+    return "{}-{}-build{}".format(product.capitalize(),
+                                  str(version), str(buildno))
+
+
+class MarkReleaseAsShipped(BaseScript, VirtualenvMixin, BuildbotMixin):
+    config_options = virtualenv_config_options
+
+    def __init__(self, require_config_file=True):
+        super(MarkReleaseAsShipped, self).__init__(
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            config={
+                "virtualenv_modules": [
+                    "shipitapi",
+                ],
+                "virtualenv_path": "venv",
+                "credentials_file": "oauth.txt",
+                "buildbot_json_path": "buildprops.json",
+                "timeout": 60,
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "mark-as-shipped",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "mark-as-shipped",
+            ],
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(MarkReleaseAsShipped, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        props = self.buildbot_config['properties']
+        mandatory_props = ['product', 'version', 'build_number']
+        missing_props = []
+        for prop in mandatory_props:
+            if prop in props:
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+            else:
+                self.warning("%s could not be found within buildprops" % prop)
+                missing_props.append(prop)
+
+        if missing_props:
+            raise Exception("%s not found in configs" % missing_props)
+
+        self.config['name'] = build_release_name(self.config['product'],
+                                                 self.config['version'],
+                                                 self.config['build_number'])
+
+    def mark_as_shipped(self):
+        """Method to make a simple call to Ship-it API to change a release
+        status to 'shipped'
+        """
+        credentials_file = os.path.join(os.getcwd(),
+                                        self.config["credentials_file"])
+        credentials = {}
+        execfile(credentials_file, credentials)
+        ship_it_credentials = credentials["ship_it_credentials"]
+        auth = (self.config["ship_it_username"],
+                ship_it_credentials.get(self.config["ship_it_username"]))
+        api_root = self.config['ship_it_root']
+
+        from shipitapi import Release
+        release_api = Release(auth, api_root=api_root,
+                              timeout=self.config['timeout'])
+        shipped_at = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
+
+        self.info("Mark the release as shipped with %s timestamp" % shipped_at)
+        release_api.update(self.config['name'],
+                           status='shipped', shippedAt=shipped_at)
+
+
+if __name__ == '__main__':
+    MarkReleaseAsShipped().run_and_exit()

testing/mozharness/scripts/release/postrelease_version_bump.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" postrelease_version_bump.py
+
+A script to increase in-tree version number after shipping a release.
+"""
+
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
+
+
+# PostReleaseVersionBump {{{1
+class PostReleaseVersionBump(MercurialScript, BuildbotMixin,
+                             MercurialRepoManipulationMixin):
+    config_options = [
+        [['--hg-user', ], {
+            "action": "store",
+            "dest": "hg_user",
+            "type": "string",
+            "default": "ffxbld <release@mozilla.com>",
+            "help": "Specify what user to use to commit to hg.",
+        }],
+        [['--next-version', ], {
+            "action": "store",
+            "dest": "next_version",
+            "type": "string",
+            "help": "Next version used in version bump",
+        }],
+        [['--ssh-user', ], {
+            "action": "store",
+            "dest": "ssh_user",
+            "type": "string",
+            "help": "SSH username with hg.mozilla.org permissions",
+        }],
+        [['--ssh-key', ], {
+            "action": "store",
+            "dest": "ssh_key",
+            "type": "string",
+            "help": "Path to SSH key.",
+        }],
+        [['--product', ], {
+            "action": "store",
+            "dest": "product",
+            "type": "string",
+            "help": "Product name",
+        }],
+        [['--version', ], {
+            "action": "store",
+            "dest": "version",
+            "type": "string",
+            "help": "Version",
+        }],
+        [['--build-number', ], {
+            "action": "store",
+            "dest": "build_number",
+            "type": "string",
+            "help": "Build number",
+        }],
+        [['--revision', ], {
+            "action": "store",
+            "dest": "revision",
+            "type": "string",
+            "help": "HG revision to tag",
+        }],
+    ]
+
+    def __init__(self, require_config_file=True):
+        super(PostReleaseVersionBump, self).__init__(
+            config_options=self.config_options,
+            all_actions=[
+                'clobber',
+                'clean-repos',
+                'pull',
+                'bump_postrelease',
+                'commit-changes',
+                'tag',
+                'push',
+            ],
+            default_actions=[
+                'clean-repos',
+                'pull',
+                'bump_postrelease',
+                'commit-changes',
+                'tag',
+                'push',
+            ],
+            config={
+                'buildbot_json_path': 'buildprops.json',
+            },
+            require_config_file=require_config_file
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(PostReleaseVersionBump, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+        else:
+            props = self.buildbot_config["properties"]
+            for prop in ['next_version', 'product', 'version', 'build_number',
+                         'revision']:
+                if props.get(prop):
+                    self.info("Overriding %s with %s" % (prop, props[prop]))
+                    self.config[prop] = props.get(prop)
+
+        if not self.config.get("next_version"):
+            self.fatal("Next version has to be set. Use --next-version or "
+                       "pass `next_version' via buildbot properties.")
+
+    def query_abs_dirs(self):
+        """ Allow for abs_from_dir and abs_to_dir
+            """
+        if self.abs_dirs:
+            return self.abs_dirs
+        self.abs_dirs = super(PostReleaseVersionBump, self).query_abs_dirs()
+        self.abs_dirs["abs_gecko_dir"] = os.path.join(
+                self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"])
+        return self.abs_dirs
+
+    def query_repos(self):
+        """Build a list of repos to clone."""
+        return [self.config["repo"]]
+
+    def query_commit_dirs(self):
+        return [self.query_abs_dirs()["abs_gecko_dir"]]
+
+    def query_commit_message(self):
+        return "Automatic version bump. CLOSED TREE NO BUG a=release"
+
+    def query_push_dirs(self):
+        return self.query_commit_dirs()
+
+    def query_push_args(self, cwd):
+        # cwd is not used here
+        hg_ssh_opts = "ssh -l {user} -i {key}".format(
+            user=self.config["ssh_user"],
+            key=os.path.expanduser(self.config["ssh_key"])
+        )
+        return ["-e", hg_ssh_opts, "-r", "."]
+
+    def pull(self):
+        super(PostReleaseVersionBump, self).pull(
+                repos=self.query_repos())
+
+    def bump_postrelease(self, *args, **kwargs):
+        """Bump version"""
+        dirs = self.query_abs_dirs()
+        for f in self.config["version_files"]:
+            curr_version = ".".join(
+                self.get_version(dirs['abs_gecko_dir'], f["file"]))
+            self.replace(os.path.join(dirs['abs_gecko_dir'], f["file"]),
+                         curr_version, self.config["next_version"])
+
+    def tag(self):
+        dirs = self.query_abs_dirs()
+        tags = ["{product}_{version}_BUILD{build_number}",
+                "{product}_{version}_RELEASE"]
+        tags = [t.format(product=self.config["product"].upper(),
+                         version=self.config["version"].replace(".", "_"),
+                         build_number=self.config["build_number"])
+                for t in tags]
+        message = "No bug - Tagging {revision} with {tags} a=release CLOSED TREE"
+        message = message.format(
+            revision=self.config["revision"],
+            tags=', '.join(tags))
+        self.hg_tag(cwd=dirs["abs_gecko_dir"], tags=tags,
+                    revision=self.config["revision"], message=message,
+                    user=self.config["hg_user"], force=True)
+
+# __main__ {{{1
+if __name__ == '__main__':
+    PostReleaseVersionBump().run_and_exit()

testing/mozharness/scripts/release/publish_balrog.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" updates.py
+
+A script publish a release to Balrog.
+
+"""
+
+import os
+import sys
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+
+# PublishBalrog {{{1
+
+
+class PublishBalrog(MercurialScript, BuildbotMixin):
+
+    def __init__(self, require_config_file=True):
+        super(PublishBalrog, self).__init__(
+            all_actions=[
+                'clobber',
+                'pull',
+                'submit-to-balrog',
+            ],
+            default_actions=[
+                'clobber',
+                'pull',
+                'submit-to-balrog',
+            ],
+            config={
+                'buildbot_json_path': 'buildprops.json',
+                'credentials_file': 'oauth.txt',
+            },
+            require_config_file=require_config_file
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(PublishBalrog, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        # TODO: version and appVersion should come from repo
+        props = self.buildbot_config["properties"]
+        for prop in ['product', 'version', 'build_number', 'channels',
+                     'balrog_api_root', 'schedule_at', 'background_rate']:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        self.abs_dirs = super(PublishBalrog, self).query_abs_dirs()
+        self.abs_dirs["abs_tools_dir"] = os.path.join(
+            self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"])
+        return self.abs_dirs
+
+    def query_channel_configs(self):
+        """Return a list of channel configs.
+        For RC builds it returns "release" and "beta" using
+        "enabled_if_version_matches" to match RC.
+
+        :return: list
+         """
+        return [(n, c) for n, c in self.config["update_channels"].items() if
+            n in self.config["channels"]]
+
+    def query_repos(self):
+        """Build a list of repos to clone."""
+        return [self.config["repo"]]
+
+    def pull(self):
+        super(PublishBalrog, self).pull(
+            repos=self.query_repos())
+
+
+    def submit_to_balrog(self):
+        for _, channel_config in self.query_channel_configs():
+            self._submit_to_balrog(channel_config)
+
+    def _submit_to_balrog(self, channel_config):
+        dirs = self.query_abs_dirs()
+        auth = os.path.join(os.getcwd(), self.config['credentials_file'])
+        cmd = [
+            self.query_exe("python"),
+            os.path.join(dirs["abs_tools_dir"],
+                         "scripts/build-promotion/balrog-release-shipper.py")]
+        cmd.extend([
+            "--api-root", self.config["balrog_api_root"],
+            "--credentials-file", auth,
+            "--username", self.config["balrog_username"],
+            "--version", self.config["version"],
+            "--product", self.config["product"],
+            "--build-number", str(self.config["build_number"]),
+            "--verbose",
+        ])
+        for r in channel_config["publish_rules"]:
+            cmd.extend(["--rules", r])
+        if self.config.get("schedule_at"):
+            cmd.extend(["--schedule-at", self.config["schedule_at"]])
+        if self.config.get("background_rate"):
+            cmd.extend(["--background-rate", str(self.config["background_rate"])])
+
+        self.retry(lambda: self.run_command(cmd, halt_on_failure=True))
+
+# __main__ {{{1
+if __name__ == '__main__':
+    PublishBalrog().run_and_exit()

testing/mozharness/scripts/release/push-candidate-to-releases.py

@@ -0,0 +1,200 @@
+from multiprocessing.pool import ThreadPool
+import os
+import re
+import sys
+
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.aws import pop_aws_auth_from_env
+
+
+class ReleasePusher(BaseScript, VirtualenvMixin):
+    config_options = [
+        [["--product"], {
+            "dest": "product",
+            "help": "Product being released, eg: firefox, thunderbird",
+        }],
+        [["--version"], {
+            "dest": "version",
+            "help": "Version of release, eg: 39.0b5",
+        }],
+        [["--build-number"], {
+            "dest": "build_number",
+            "help": "Build number of release, eg: 2",
+        }],
+        [["--bucket-name"], {
+            "dest": "bucket_name",
+            "help": "Bucket to copy files from candidates/ to releases/",
+        }],
+        [["--credentials"], {
+            "dest": "credentials",
+            "help": "File containing access key and secret access key",
+        }],
+        [["--exclude"], {
+            "dest": "excludes",
+            "default": [
+                r"^.*tests.*$",
+                r"^.*crashreporter.*$",
+                r"^.*[^k]\.zip(\.asc)?$",
+                r"^.*\.log$",
+                r"^.*\.txt$",
+                r"^.*/partner-repacks.*$",
+                r"^.*.checksums(\.asc)?$",
+                r"^.*/logs/.*$",
+                r"^.*/jsshell.*$",
+                r"^.*json$",
+                r"^.*/host.*$",
+                r"^.*/mar-tools/.*$",
+                r"^.*robocop.apk$",
+                r"^.*bouncer.apk$",
+                r"^.*contrib.*",
+                r"^.*/beetmover-checksums/.*$",
+            ],
+            "action": "append",
+            "help": "List of patterns to exclude from copy. The list can be "
+                    "extended by passing multiple --exclude arguments.",
+        }],
+        [["-j", "--parallelization"], {
+            "dest": "parallelization",
+            "default": 20,
+            "type": "int",
+            "help": "Number of copy requests to run concurrently",
+        }],
+    ] + virtualenv_config_options
+
+    def __init__(self, aws_creds):
+        BaseScript.__init__(self,
+            config_options=self.config_options,
+            require_config_file=False,
+            config={
+                "virtualenv_modules": [
+                    "boto",
+                    "redo",
+                ],
+                "virtualenv_path": "venv",
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "push-to-releases",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "push-to-releases",
+            ],
+        )
+
+        # validate aws credentials
+        if not (all(aws_creds) or self.config.get('credentials')):
+            self.fatal("aws creds not defined. please add them to your config or env.")
+        if any(aws_creds) and self.config.get('credentials'):
+            self.fatal("aws creds found in env and self.config. please declare in one place only.")
+
+        # set aws credentials
+        if all(aws_creds):
+            self.aws_key_id, self.aws_secret_key = aws_creds
+        else:  # use
+            self.aws_key_id, self.aws_secret_key = None, None
+            # set the env var for boto to read our special config file
+            # rather than anything else we have at ~/.boto
+            os.environ["BOTO_CONFIG"] = os.path.abspath(self.config["credentials"])
+
+    def _get_candidates_prefix(self):
+        return "pub/{}/candidates/{}-candidates/build{}/".format(
+            self.config['product'],
+            self.config["version"],
+            self.config["build_number"]
+        )
+
+    def _get_releases_prefix(self):
+        return "pub/{}/releases/{}/".format(
+            self.config["product"],
+            self.config["version"]
+        )
+
+    def _matches_exclude(self, keyname):
+        for exclude in self.config["excludes"]:
+            if re.search(exclude, keyname):
+                return True
+        return False
+
+    def push_to_releases(self):
+        """This step grabs the list of files in the candidates dir,
+        filters out any unwanted files from within them, and copies
+        the remainder."""
+        from boto.s3.connection import S3Connection
+        from boto.exception import S3CopyError, S3ResponseError
+        from redo import retry
+
+        # suppress boto debug logging, it's too verbose with --loglevel=debug
+        import logging
+        logging.getLogger('boto').setLevel(logging.INFO)
+
+        self.info("Connecting to S3")
+        conn = S3Connection(aws_access_key_id=self.aws_key_id,
+                            aws_secret_access_key=self.aws_secret_key)
+        self.info("Getting bucket {}".format(self.config["bucket_name"]))
+        bucket = conn.get_bucket(self.config["bucket_name"])
+
+        # ensure the destination is empty
+        self.info("Checking destination {} is empty".format(self._get_releases_prefix()))
+        keys = [k for k in bucket.list(prefix=self._get_releases_prefix())]
+        if keys:
+            self.warning("Destination already exists with %s keys" % len(keys))
+
+        def worker(item):
+            source, destination = item
+
+            def copy_key():
+                source_key = bucket.get_key(source)
+                dest_key = bucket.get_key(destination)
+                # According to http://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+                # S3 key MD5 is represented as ETag, except when objects are
+                # uploaded using multipart method. In this case objects's ETag
+                # is constructed using its MD5, minus symbol, and number of
+                # part. See http://stackoverflow.com/questions/12186993/what-is-the-algorithm-to-compute-the-amazon-s3-etag-for-a-file-larger-than-5gb#answer-19896823
+                source_md5 = source_key.etag.split("-")[0]
+                if dest_key:
+                    dest_md5 = dest_key.etag.split("-")[0]
+                else:
+                    dest_md5 = None
+
+                if not dest_key:
+                    self.info("Copying {} to {}".format(source, destination))
+                    bucket.copy_key(destination, self.config["bucket_name"],
+                                    source)
+                elif source_md5 == dest_md5:
+                    self.warning(
+                        "{} already exists with the same content ({}), skipping copy".format(
+                            destination, dest_md5))
+                else:
+                    self.fatal(
+                        "{} already exists with the different content (src ETag: {}, dest ETag: {}), aborting".format(
+                            destination, source_key.etag, dest_key.etag))
+
+            return retry(copy_key, sleeptime=5, max_sleeptime=60,
+                         retry_exceptions=(S3CopyError, S3ResponseError))
+
+        def find_release_files():
+            candidates_prefix = self._get_candidates_prefix()
+            release_prefix = self._get_releases_prefix()
+            self.info("Getting key names from candidates")
+            for key in bucket.list(prefix=candidates_prefix):
+                keyname = key.name
+                if self._matches_exclude(keyname):
+                    self.debug("Excluding {}".format(keyname))
+                else:
+                    destination = keyname.replace(candidates_prefix,
+                                                  release_prefix)
+                    yield (keyname, destination)
+
+        pool = ThreadPool(self.config["parallelization"])
+        pool.map(worker, find_release_files())
+
+if __name__ == "__main__":
+    myScript = ReleasePusher(pop_aws_auth_from_env())
+    myScript.run_and_exit()

testing/mozharness/scripts/release/updates.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" updates.py
+
+A script to bump patcher configs, generate update verification configs, and
+publish top-level release blob information to Balrog.
+
+It clones the tools repo, modifies the existing patcher config to include
+current release build information, generates update verification configs,
+commits the changes and tags the repo using tags by Releng convention.
+After the changes are pushed to the repo, the script submits top-level release
+information to Balrog.
+"""
+
+import os
+import re
+import sys
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+from mozharness.mozilla.repo_manupulation import MercurialRepoManipulationMixin
+from mozharness.mozilla.release import get_previous_version
+
+
+# UpdatesBumper {{{1
+class UpdatesBumper(MercurialScript, BuildbotMixin,
+                    MercurialRepoManipulationMixin):
+    config_options = [
+        [['--hg-user', ], {
+            "action": "store",
+            "dest": "hg_user",
+            "type": "string",
+            "default": "ffxbld <release@mozilla.com>",
+            "help": "Specify what user to use to commit to hg.",
+        }],
+        [['--ssh-user', ], {
+            "action": "store",
+            "dest": "ssh_user",
+            "type": "string",
+            "help": "SSH username with hg.mozilla.org permissions",
+        }],
+        [['--ssh-key', ], {
+            "action": "store",
+            "dest": "ssh_key",
+            "type": "string",
+            "help": "Path to SSH key.",
+        }],
+    ]
+
+    def __init__(self, require_config_file=True):
+        super(UpdatesBumper, self).__init__(
+            config_options=self.config_options,
+            all_actions=[
+                'clobber',
+                'pull',
+                'download-shipped-locales',
+                'bump-configs',
+                'commit-changes',
+                'tag',
+                'push',
+                'submit-to-balrog',
+            ],
+            default_actions=[
+                'clobber',
+                'pull',
+                'download-shipped-locales',
+                'bump-configs',
+                'commit-changes',
+                'tag',
+                'push',
+                'submit-to-balrog',
+            ],
+            config={
+                'buildbot_json_path': 'buildprops.json',
+                'credentials_file': 'oauth.txt',
+            },
+            require_config_file=require_config_file
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(UpdatesBumper, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        # TODO: version and appVersion should come from repo
+        props = self.buildbot_config["properties"]
+        for prop in ['product', 'version', 'build_number', 'revision',
+                     'appVersion', 'balrog_api_root', "channels"]:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+
+        partials = [v.strip() for v in props["partial_versions"].split(",")]
+        self.config["partial_versions"] = [v.split("build") for v in partials]
+        self.config["platforms"] = [p.strip() for p in
+                                    props["platforms"].split(",")]
+        self.config["channels"] = [c.strip() for c in
+                                   props["channels"].split(",")]
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        self.abs_dirs = super(UpdatesBumper, self).query_abs_dirs()
+        self.abs_dirs["abs_tools_dir"] = os.path.join(
+            self.abs_dirs['abs_work_dir'], self.config["repo"]["dest"])
+        return self.abs_dirs
+
+    def query_repos(self):
+        """Build a list of repos to clone."""
+        return [self.config["repo"]]
+
+    def query_commit_dirs(self):
+        return [self.query_abs_dirs()["abs_tools_dir"]]
+
+    def query_commit_message(self):
+        return "Automated configuration bump"
+
+    def query_push_dirs(self):
+        return self.query_commit_dirs()
+
+    def query_push_args(self, cwd):
+        # cwd is not used here
+        hg_ssh_opts = "ssh -l {user} -i {key}".format(
+            user=self.config["ssh_user"],
+            key=os.path.expanduser(self.config["ssh_key"])
+        )
+        return ["-e", hg_ssh_opts]
+
+    def query_shipped_locales_path(self):
+        dirs = self.query_abs_dirs()
+        return os.path.join(dirs["abs_work_dir"], "shipped-locales")
+
+    def query_channel_configs(self):
+        """Return a list of channel configs.
+        For RC builds it returns "release" and "beta" using
+        "enabled_if_version_matches" to match RC.
+
+        :return: list
+        """
+        return [(n, c) for n, c in self.config["update_channels"].items() if
+                n in self.config["channels"]]
+
+    def pull(self):
+        super(UpdatesBumper, self).pull(
+            repos=self.query_repos())
+
+    def download_shipped_locales(self):
+        dirs = self.query_abs_dirs()
+        self.mkdir_p(dirs["abs_work_dir"])
+        url = self.config["shipped-locales-url"].format(
+            revision=self.config["revision"])
+        if not self.download_file(url=url,
+                                  file_name=self.query_shipped_locales_path()):
+            self.fatal("Unable to fetch shipped-locales from %s" % url)
+
+    def bump_configs(self):
+        for channel, channel_config in self.query_channel_configs():
+            self.bump_patcher_config(channel_config)
+            self.bump_update_verify_configs(channel, channel_config)
+
+    def query_matching_partials(self, channel_config):
+        return [(v, b) for v, b in self.config["partial_versions"] if
+                re.match(channel_config["version_regex"], v)]
+
+    def query_patcher_config(self, channel_config):
+        dirs = self.query_abs_dirs()
+        patcher_config = os.path.join(
+            dirs["abs_tools_dir"], "release/patcher-configs",
+            channel_config["patcher_config"])
+        return patcher_config
+
+    def query_update_verify_config(self, channel, platform):
+        dirs = self.query_abs_dirs()
+        uvc = os.path.join(
+            dirs["abs_tools_dir"], "release/updates",
+            "{}-{}-{}.cfg".format(channel, self.config["product"], platform))
+        return uvc
+
+    def bump_patcher_config(self, channel_config):
+        # TODO: to make it possible to run this before we have files copied to
+        # the candidates directory, we need to add support to fetch build IDs
+        # from tasks.
+        dirs = self.query_abs_dirs()
+        env = {"PERL5LIB": os.path.join(dirs["abs_tools_dir"], "lib/perl")}
+        partial_versions = [v[0] for v in
+                            self.query_matching_partials(channel_config)]
+        script = os.path.join(
+            dirs["abs_tools_dir"], "release/patcher-config-bump.pl")
+        patcher_config = self.query_patcher_config(channel_config)
+        cmd = [self.query_exe("perl"), script]
+        cmd.extend([
+            "-p", self.config["product"],
+            "-r", self.config["product"].capitalize(),
+            "-v", self.config["version"],
+            "-a", self.config["appVersion"],
+            "-o", get_previous_version(
+                self.config["version"], partial_versions),
+            "-b", str(self.config["build_number"]),
+            "-c", patcher_config,
+            "-f", self.config["archive_domain"],
+            "-d", self.config["download_domain"],
+            "-l", self.query_shipped_locales_path(),
+        ])
+        for v in partial_versions:
+            cmd.extend(["--partial-version", v])
+        for p in self.config["platforms"]:
+            cmd.extend(["--platform", p])
+        for mar_channel_id in channel_config["mar_channel_ids"]:
+            cmd.extend(["--mar-channel-id", mar_channel_id])
+        self.run_command(cmd, halt_on_failure=True, env=env)
+
+    def bump_update_verify_configs(self, channel, channel_config):
+        dirs = self.query_abs_dirs()
+        script = os.path.join(
+            dirs["abs_tools_dir"],
+            "scripts/build-promotion/create-update-verify-config.py")
+        patcher_config = self.query_patcher_config(channel_config)
+        for platform in self.config["platforms"]:
+            cmd = [self.query_exe("python"), script]
+            output = self.query_update_verify_config(channel, platform)
+            cmd.extend([
+                "--config", patcher_config,
+                "--platform", platform,
+                "--update-verify-channel",
+                channel_config["update_verify_channel"],
+                "--output", output,
+                "--archive-prefix", self.config["archive_prefix"],
+                "--previous-archive-prefix",
+                self.config["previous_archive_prefix"],
+                "--product", self.config["product"],
+                "--balrog-url", self.config["balrog_url"],
+                "--build-number", str(self.config["build_number"]),
+            ])
+
+            self.run_command(cmd, halt_on_failure=True)
+
+    def tag(self):
+        dirs = self.query_abs_dirs()
+        tags = ["{product}_{version}_BUILD{build_number}_RUNTIME",
+                "{product}_{version}_RELEASE_RUNTIME"]
+        tags = [t.format(product=self.config["product"].upper(),
+                         version=self.config["version"].replace(".", "_"),
+                         build_number=self.config["build_number"])
+                for t in tags]
+        self.hg_tag(cwd=dirs["abs_tools_dir"], tags=tags,
+                    user=self.config["hg_user"], force=True)
+
+    def submit_to_balrog(self):
+        for _, channel_config in self.query_channel_configs():
+            self._submit_to_balrog(channel_config)
+
+    def _submit_to_balrog(self, channel_config):
+        dirs = self.query_abs_dirs()
+        auth = os.path.join(os.getcwd(), self.config['credentials_file'])
+        cmd = [
+            self.query_exe("python"),
+            os.path.join(dirs["abs_tools_dir"],
+                         "scripts/build-promotion/balrog-release-pusher.py")]
+        cmd.extend([
+            "--api-root", self.config["balrog_api_root"],
+            "--download-domain", self.config["download_domain"],
+            "--archive-domain", self.config["archive_domain"],
+            "--credentials-file", auth,
+            "--product", self.config["product"],
+            "--version", self.config["version"],
+            "--build-number", str(self.config["build_number"]),
+            "--app-version", self.config["appVersion"],
+            "--username", self.config["balrog_username"],
+            "--verbose",
+        ])
+        for c in channel_config["channel_names"]:
+            cmd.extend(["--channel", c])
+        for r in channel_config["rules_to_update"]:
+            cmd.extend(["--rule-to-update", r])
+        for p in self.config["platforms"]:
+            cmd.extend(["--platform", p])
+        for v, build_number in self.query_matching_partials(channel_config):
+            partial = "{version}build{build_number}".format(
+                version=v, build_number=build_number)
+            cmd.extend(["--partial-update", partial])
+        if channel_config["requires_mirrors"]:
+            cmd.append("--requires-mirrors")
+        if self.config["balrog_use_dummy_suffix"]:
+            cmd.append("--dummy")
+
+        self.retry(lambda: self.run_command(cmd, halt_on_failure=True))
+
+# __main__ {{{1
+if __name__ == '__main__':
+    UpdatesBumper().run_and_exit()

testing/mozharness/scripts/release/uptake_monitoring.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" uptake_monitoring.py
+
+A script to replace the old-fashion way of computing the uptake monitoring
+from the scheduler within the slaves.
+"""
+
+import os
+import sys
+import datetime
+import time
+import xml.dom.minidom
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+
+
+def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os):
+    return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url,
+                                            related_product, os)
+
+
+class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin):
+    config_options = virtualenv_config_options
+
+    def __init__(self, require_config_file=True):
+        super(UptakeMonitoring, self).__init__(
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            config={
+                "virtualenv_modules": [
+                    "redo",
+                    "requests",
+                ],
+
+                "virtualenv_path": "venv",
+                "credentials_file": "oauth.txt",
+                "buildbot_json_path": "buildprops.json",
+                "poll_interval": 60,
+                "poll_timeout": 20*60,
+                "min_uptake": 10000,
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "monitor-uptake",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "monitor-uptake",
+            ],
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(UptakeMonitoring, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        props = self.buildbot_config["properties"]
+        for prop in ['tuxedo_server_url', 'version']:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+            else:
+                self.warning("%s could not be found within buildprops" % prop)
+                return
+        partials = [v.strip() for v in props["partial_versions"].split(",")]
+        self.config["partial_versions"] = [v.split("build")[0] for v in partials]
+        self.config["platforms"] = [p.strip() for p in
+                                    props["platforms"].split(",")]
+
+    def _get_product_uptake(self, tuxedo_server_url, auth,
+                            related_product, os):
+        from redo import retry
+        import requests
+
+        url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os)
+        self.info("Requesting {} from tuxedo".format(url))
+
+        def get_tuxedo_page():
+            r = requests.get(url, auth=auth,
+                             verify=False, timeout=60)
+            r.raise_for_status()
+            return r.content
+
+        def calculateUptake(page):
+            doc = xml.dom.minidom.parseString(page)
+            uptake_values = []
+
+            for element in doc.getElementsByTagName('available'):
+                for node in element.childNodes:
+                    if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
+                            node.data.isdigit():
+                        uptake_values.append(int(node.data))
+            if not uptake_values:
+                uptake_values = [0]
+            return min(uptake_values)
+
+        page = retry(get_tuxedo_page)
+        uptake = calculateUptake(page)
+        self.info("Current uptake for {} is {}".format(related_product, uptake))
+        return uptake
+
+    def _get_release_uptake(self, auth):
+        assert isinstance(self.config["platforms"], (list, tuple))
+
+        # handle the products first
+        tuxedo_server_url = self.config["tuxedo_server_url"]
+        version = self.config["version"]
+        dl = []
+
+        for product, info in self.config["products"].iteritems():
+            if info.get("check_uptake"):
+                product_template = info["product-name"]
+                related_product = product_template % {"version": version}
+
+                enUS_platforms = set(self.config["platforms"])
+                paths_platforms = set(info["paths"].keys())
+                platforms = enUS_platforms.intersection(paths_platforms)
+
+                for platform in platforms:
+                    bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
+                    dl.append(self._get_product_uptake(tuxedo_server_url, auth,
+                                                       related_product, bouncer_platform))
+        # handle the partials as well
+        prev_versions = self.config["partial_versions"]
+        for product, info in self.config["partials"].iteritems():
+            if info.get("check_uptake"):
+                product_template = info["product-name"]
+                for prev_version in prev_versions:
+                    subs = {
+                        "version": version,
+                        "prev_version": prev_version
+                    }
+                    related_product = product_template % subs
+
+                    enUS_platforms = set(self.config["platforms"])
+                    paths_platforms = set(info["paths"].keys())
+                    platforms = enUS_platforms.intersection(paths_platforms)
+
+                    for platform in platforms:
+                        bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
+                        dl.append(self._get_product_uptake(tuxedo_server_url, auth,
+                                                           related_product, bouncer_platform))
+        return min(dl)
+
+    def monitor_uptake(self):
+        credentials_file = os.path.join(os.getcwd(),
+                                        self.config["credentials_file"])
+        credentials = {}
+        execfile(credentials_file, credentials)
+        auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
+        self.info("Starting the loop to determine the uptake monitoring ...")
+
+        start_time = datetime.datetime.now()
+        while True:
+            delta = (datetime.datetime.now() - start_time).seconds
+            if delta > self.config["poll_timeout"]:
+                self.error("Uptake monitoring sadly timed-out")
+                raise Exception("Time-out during uptake monitoring")
+
+            uptake = self._get_release_uptake(auth)
+            self.info("Current uptake value to check is {}".format(uptake))
+
+            if uptake >= self.config["min_uptake"]:
+                self.info("Uptake monitoring is complete!")
+                break
+            else:
+                self.info("Mirrors not yet updated, sleeping for a bit ...")
+                time.sleep(self.config["poll_interval"])
+
+
+if __name__ == '__main__':
+    myScript = UptakeMonitoring()
+    myScript.run_and_exit()