import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

testing/docker/rust-build/Dockerfile

@@ -0,0 +1,37 @@
+FROM          quay.io/rust/rust-buildbot
+MAINTAINER    Ralph Giles <giles@mozilla.com>
+
+# Reset user/workdir from parent image so we can install software.
+WORKDIR /
+USER root
+
+# Update base.
+RUN yum upgrade -y
+RUN yum clean all
+
+# Install tooltool directly from github.
+RUN mkdir /builds
+ADD https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py /build/tooltool.py
+RUN chmod +rx /build/tooltool.py
+
+# Add build scripts.
+ADD             fetch_rust.sh build_rust.sh /build/
+ADD             fetch_cargo.sh build_cargo.sh /build/
+ADD             package_rust.sh upload_rust.sh /build/
+ADD             repack_rust.py /build/
+RUN             chmod +x /build/*
+
+# Create user for doing the build.
+ENV USER worker
+ENV HOME /home/${USER}
+
+RUN useradd -d ${HOME} -m ${USER}
+
+# Set up the user's tree
+WORKDIR ${HOME}
+
+# Invoke our build scripts by default, but allow other commands.
+USER ${USER}
+ENTRYPOINT /build/fetch_rust.sh && /build/build_rust.sh && \
+  /build/fetch_cargo.sh && /build/build_cargo.sh && \
+  /build/package_rust.sh && /build/upload_rust.sh

testing/docker/rust-build/README.md

@@ -0,0 +1,2 @@
+This is a docker script for building rust toolchains for
+use in Mozilla's build clusters.

testing/docker/rust-build/REGISTRY

@@ -0,0 +1 @@
+quay.io/rust

testing/docker/rust-build/VERSION

@@ -0,0 +1 @@
+0.3.0

testing/docker/rust-build/build_cargo.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Configure and build cargo.
+
+if test $(uname -s) = "Darwin"; then
+  export MACOSX_DEPLOYMENT_TARGET=10.7
+fi
+
+pushd ${WORKSPACE}/cargo
+./configure --prefix=${WORKSPACE}/rustc --local-rust-root=${WORKSPACE}/rustc
+make
+make dist
+make install
+popd

testing/docker/rust-build/build_rust.sh

@@ -0,0 +1,26 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/home/worker}
+
+CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
+
+set -v
+
+# Configure and build rust.
+OPTIONS="--enable-llvm-static-stdcpp --disable-docs"
+OPTIONS+="--enable-debuginfo"
+OPTIONS+="--release-channel=stable"
+x32="i686-unknown-linux-gnu"
+x64="x86_64-unknown-linux-gnu"
+arm="arm-linux-androideabi"
+
+mkdir -p ${WORKSPACE}/rust-build
+pushd ${WORKSPACE}/rust-build
+${WORKSPACE}/rust/configure --prefix=${WORKSPACE}/rustc \
+  --target=${x64},${x32} ${OPTIONS}
+make -j ${CORES}
+make dist
+make install
+popd

testing/docker/rust-build/build_rust_mac.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -vex
+
+set -e
+
+: WORKSPACE ${WORKSPACE:=$PWD}
+: TOOLTOOL ${TOOLTOOL:=python $WORKSPACE/tooltool.py}
+
+CORES=$(nproc || grep -c ^processor /proc/cpuinfo || sysctl -n hw.ncpu)
+echo Building on $CORES cpus...
+
+OPTIONS="--enable-debuginfo --disable-docs"
+TARGETS="x86_64-apple-darwin,i686-apple-darwin"
+
+PREFIX=${WORKSPACE}/rustc
+
+set -v
+
+mkdir -p ${WORKSPACE}/gecko-rust-mac
+pushd ${WORKSPACE}/gecko-rust-mac
+
+export MACOSX_DEPLOYMENT_TARGET=10.7
+${WORKSPACE}/rust/configure --prefix=${PREFIX} --target=${TARGETS} ${OPTIONS}
+make -j ${CORES}
+
+rm -rf ${PREFIX}
+mkdir ${PREFIX}
+make dist
+make install
+popd
+
+# Package the toolchain for upload.
+pushd ${WORKSPACE}
+rustc/bin/rustc --version
+tar cvjf rustc.tar.bz2 rustc/*
+${TOOLTOOL} add --visibility=public --unpack rustc.tar.bz2
+popd

testing/docker/rust-build/fetch_cargo.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -vex
+
+set -x -e
+
+# Inputs, with defaults
+
+: REPOSITORY   ${REPOSITORY:=https://github.com/rust-lang/cargo}
+: BRANCH       ${BRANCH:=master}
+
+: WORKSPACE    ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Check out rust sources
+SRCDIR=${WORKSPACE}/cargo
+git clone --recursive $REPOSITORY -b $BRANCH ${SRCDIR}
+
+# Report version
+VERSION=$(git -C ${SRCDIR} describe --tags --dirty)
+COMMIT=$(git -C ${SRCDIR} rev-parse HEAD)
+echo "cargo ${VERSION} (commit ${COMMIT})" | tee cargo-version

testing/docker/rust-build/fetch_rust.sh

@@ -0,0 +1,20 @@
+#!/bin/bash -vex
+
+set -x -e
+
+# Inputs, with defaults
+
+: RUST_REPOSITORY ${RUST_REPOSITORY:=https://github.com/rust-lang/rust}
+: RUST_BRANCH     ${RUST_BRANCH:=stable}
+
+: WORKSPACE       ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Check out rust sources
+git clone $RUST_REPOSITORY -b $RUST_BRANCH ${WORKSPACE}/rust
+
+# Report version
+VERSION=$(git -C ${WORKSPACE}/rust describe --tags --dirty)
+COMMIT=$(git -C ${WORKSPACE}/rust rev-parse HEAD)
+echo "rust ${VERSION} (commit ${COMMIT})" | tee rust-version

testing/docker/rust-build/package_rust.sh

@@ -0,0 +1,13 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Package the toolchain for upload.
+pushd ${WORKSPACE}
+tar cvJf rustc.tar.xz rustc/*
+/build/tooltool.py add --visibility=public --unpack rustc.tar.xz
+popd

testing/docker/rust-build/repack_rust.py

@@ -0,0 +1,177 @@
+#!/bin/env python
+'''
+This script downloads and repacks official rust language builds
+with the necessary tool and target support for the Firefox
+build environment.
+'''
+
+import os.path
+import requests
+import subprocess
+import toml
+
+def fetch_file(url):
+  '''Download a file from the given url if it's not already present.'''
+  filename = os.path.basename(url)
+  if os.path.exists(filename):
+    return
+  r = requests.get(url, stream=True)
+  r.raise_for_status()
+  with open(filename, 'wb') as fd:
+    for chunk in r.iter_content(4096):
+      fd.write(chunk)
+
+def fetch(url):
+  '''Download and verify a package url.'''
+  base = os.path.basename(url)
+  print('Fetching %s...' % base)
+  fetch_file(url + '.asc')
+  fetch_file(url)
+  fetch_file(url + '.sha256')
+  fetch_file(url + '.asc.sha256')
+  print('Verifying %s...' % base)
+  subprocess.check_call(['shasum', '-c', base + '.sha256'])
+  subprocess.check_call(['shasum', '-c', base + '.asc.sha256'])
+  subprocess.check_call(['gpg', '--verify', base + '.asc', base])
+  subprocess.check_call(['keybase', 'pgp', 'verify',
+      '-d', base + '.asc',
+      '-i', base,
+  ])
+
+def install(filename, target):
+  '''Run a package's installer script against the given target directory.'''
+  print(' Unpacking %s...' % filename)
+  subprocess.check_call(['tar', 'xf', filename])
+  basename = filename.split('.tar')[0]
+  print(' Installing %s...' % basename)
+  install_cmd = [os.path.join(basename, 'install.sh')]
+  install_cmd += ['--prefix=' + os.path.abspath(target)]
+  install_cmd += ['--disable-ldconfig']
+  subprocess.check_call(install_cmd)
+  print(' Cleaning %s...' % basename)
+  subprocess.check_call(['rm', '-rf', basename])
+
+def package(manifest, pkg, target):
+  '''Pull out the package dict for a particular package and target
+  from the given manifest.'''
+  version = manifest['pkg'][pkg]['version']
+  info = manifest['pkg'][pkg]['target'][target]
+  return (version, info)
+
+def fetch_package(manifest, pkg, host):
+  version, info = package(manifest, pkg, host)
+  print('%s %s\n  %s\n  %s' % (pkg, version, info['url'], info['hash']))
+  if not info['available']:
+    print('%s marked unavailable for %s' % (pkg, host))
+    raise AssertionError
+  fetch(info['url'])
+  return info
+
+def fetch_std(manifest, targets):
+  stds = []
+  for target in targets:
+      info = fetch_package(manifest, 'rust-std', target)
+      stds.append(info)
+  return stds
+
+def tar_for_host(host):
+  if 'linux' in host:
+      tar_options = 'cJf'
+      tar_ext = '.tar.xz'
+  else:
+      tar_options = 'cjf'
+      tar_ext = '.tar.bz2'
+  return tar_options, tar_ext
+
+def repack(host, targets, channel='stable', suffix=''):
+  print("Repacking rust for %s..." % host)
+  url = 'https://static.rust-lang.org/dist/channel-rust-' + channel + '.toml'
+  req = requests.get(url)
+  req.raise_for_status()
+  manifest = toml.loads(req.content)
+  if manifest['manifest-version'] != '2':
+    print('ERROR: unrecognized manifest version %s.' % manifest['manifest-version'])
+    return
+  print('Using manifest for rust %s as of %s.' % (channel, manifest['date']))
+  print('Fetching packages...')
+  rustc = fetch_package(manifest, 'rustc', host)
+  cargo = fetch_package(manifest, 'cargo', host)
+  stds = fetch_std(manifest, targets)
+  print('Installing packages...')
+  tar_basename = 'rustc-' + host
+  if suffix:
+      tar_basename += '-' + suffix
+  tar_basename += '-repack'
+  install_dir = 'rustc'
+  subprocess.check_call(['rm', '-rf', install_dir])
+  install(os.path.basename(rustc['url']), install_dir)
+  install(os.path.basename(cargo['url']), install_dir)
+  for std in stds:
+    install(os.path.basename(std['url']), install_dir)
+    pass
+  print('Tarring %s...' % tar_basename)
+  tar_options, tar_ext = tar_for_host(host)
+  subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
+  subprocess.check_call(['rm', '-rf', install_dir])
+
+def repack_cargo(host, channel='nightly'):
+  print("Repacking cargo for %s..." % host)
+  # Cargo doesn't seem to have a .toml manifest.
+  base_url = 'https://static.rust-lang.org/cargo-dist/'
+  req = requests.get(os.path.join(base_url, 'channel-cargo-' + channel))
+  req.raise_for_status()
+  file = ''
+  for line in req.iter_lines():
+      if line.find(host) != -1:
+          file = line.strip()
+  if not file:
+      print('No manifest entry for %s!' % host)
+      return
+  manifest = {
+          'date': req.headers['Last-Modified'],
+          'pkg': {
+              'cargo': {
+                  'version': channel,
+                  'target': {
+                      host: {
+                          'url': os.path.join(base_url, file),
+                          'hash': None,
+                          'available': True,
+                      },
+                  },
+              },
+          },
+  }
+  print('Using manifest for cargo %s.' % channel)
+  print('Fetching packages...')
+  cargo = fetch_package(manifest, 'cargo', host)
+  print('Installing packages...')
+  install_dir = 'cargo'
+  subprocess.check_call(['rm', '-rf', install_dir])
+  install(os.path.basename(cargo['url']), install_dir)
+  tar_basename = 'cargo-%s-repack' % host
+  print('Tarring %s...' % tar_basename)
+  tar_options, tar_ext = tar_for_host(host)
+  subprocess.check_call(['tar', tar_options, tar_basename + tar_ext, install_dir])
+  subprocess.check_call(['rm', '-rf', install_dir])
+
+# rust platform triples
+android="armv7-linux-androideabi"
+linux64="x86_64-unknown-linux-gnu"
+linux32="i686-unknown-linux-gnu"
+mac64="x86_64-apple-darwin"
+mac32="i686-apple-darwin"
+win64="x86_64-pc-windows-msvc"
+win32="i686-pc-windows-msvc"
+
+if __name__ == '__main__':
+  repack(mac64, [mac64, mac32])
+  repack(win32, [win32])
+  repack(win64, [win64])
+  repack(linux64, [linux64, linux32])
+  repack(linux64, [linux64, mac64, mac32], suffix='mac-cross')
+  repack(linux64, [linux64, android], suffix='android-cross')
+  repack_cargo(mac64)
+  repack_cargo(win32)
+  repack_cargo(win64)
+  repack_cargo(linux64)

testing/docker/rust-build/task.json

@@ -0,0 +1,37 @@
+{
+    "provisionerId": "aws-provisioner-v1",
+    "workerType": "rustbuild",
+    "created": "{task_created}",
+    "deadline": "{task_deadline}",
+    "payload": {
+        "image": "quay.io/rust/gecko-rust-build",
+        "env": {
+            "RUST_BRANCH": "{rust_branch}"
+        },
+        "artifacts": {
+            "public/rustc.tar.xz": {
+              "path": "/home/worker/rustc.tar.xz",
+              "expires": "{artifacts_expires}",
+              "type": "file"
+            },
+            "public/manifest.tt": {
+              "path": "/home/worker/manifest.tt",
+              "expires": "{artifacts_expires}",
+              "type": "file"
+            }
+        },
+        "features": {
+          "relengAPIProxy": true
+        },
+        "maxRunTime": 6000
+    },
+    "scopes": [
+      "docker-worker:relengapi-proxy:tooltool.upload.public"
+    ],
+    "metadata": {
+        "name": "Rust toolchain build",
+        "description": "Builds the rust toolchain for use in gecko builders.",
+        "owner": "giles@mozilla.com",
+        "source": "https://github.com/rillian/rust-build/"
+    }
+}

testing/docker/rust-build/tcbuild.py

@@ -0,0 +1,206 @@
+#!/bin/env python
+'''
+This script triggers a taskcluster task, waits for it to finish,
+fetches the artifacts, uploads them to tooltool, and updates
+the in-tree tooltool manifests.
+'''
+
+from __future__ import print_function
+
+import requests.packages.urllib3
+requests.packages.urllib3.disable_warnings()
+
+import argparse
+import datetime
+import json
+import os
+import shutil
+import sys
+import taskcluster
+import tempfile
+import time
+import tooltool
+
+def local_file(filename):
+    '''
+    Return a path to a file next to this script.
+    '''
+    return os.path.join(os.path.dirname(__file__), filename)
+
+def read_tc_auth(tc_auth_file):
+    '''
+    Read taskcluster credentials from tc_auth_file and return them as a dict.
+    '''
+    return json.load(open(tc_auth_file, 'rb'))
+
+def fill_template_dict(d, keys):
+    for key, val in d.items():
+        if isinstance(val, basestring) and '{' in val:
+            d[key] = val.format(**keys)
+        elif isinstance(val, dict):
+            fill_template_dict(val, keys)
+
+def fill_template(template_file, keys):
+    '''
+    Take the file object template_file, parse it as JSON, and
+    interpolate (using str.template) its keys using keys.
+    '''
+    template = json.load(template_file)
+    fill_template_dict(template, keys)
+    return template
+
+def spawn_task(queue, args):
+    '''
+    Spawn a Taskcluster task in queue using args.
+    '''
+    task_id = taskcluster.utils.slugId()
+    with open(local_file('task.json'), 'rb') as template:
+        keys = vars(args)
+        now = datetime.datetime.utcnow()
+        keys['task_created'] = now.isoformat() + 'Z'
+        keys['task_deadline'] = (now + datetime.timedelta(hours=2)).isoformat() + 'Z'
+        keys['artifacts_expires'] = (now + datetime.timedelta(days=1)).isoformat() + 'Z'
+        payload = fill_template(template, keys)
+    queue.createTask(task_id, payload)
+    print('--- %s task %s submitted ---' % (now, task_id))
+    return task_id
+
+def wait_for_task(queue, task_id, initial_wait=5):
+    '''
+    Wait until queue reports that task task_id is completed, and return
+    its run id.
+
+    Sleep for initial_wait seconds before checking status the first time.
+    Then poll periodically and print a running log of the task status.
+    '''
+    time.sleep(initial_wait)
+    previous_state = None
+    have_ticks = False
+    while True:
+        res = queue.status(task_id)
+        state = res['status']['state']
+        if state != previous_state:
+            now = datetime.datetime.utcnow()
+            if have_ticks:
+              sys.stdout.write('\n')
+              have_ticks = False
+            print('--- %s task %s %s ---' % (now, task_id, state))
+            previous_state = state
+        if state == 'completed':
+            return len(res['status']['runs']) - 1
+        if state in ('failed', 'exception'):
+            raise Exception('Task failed')
+        sys.stdout.write('.')
+        sys.stdout.flush()
+        have_ticks = True
+        time.sleep(10)
+
+def fetch_artifact(queue, task_id, run_id, name, dest_dir):
+    '''
+    Fetch the artifact with name from task_id and run_id in queue,
+    write it to a file in dest_dir, and return the path to the written
+    file.
+    '''
+    url = queue.buildUrl('getArtifact', task_id, run_id, name)
+    fn = os.path.join(dest_dir, os.path.basename(name))
+    print('Fetching %s...' % name)
+    try:
+        r = requests.get(url, stream=True)
+        r.raise_for_status()
+        with open(fn, 'wb') as f:
+            for chunk in r.iter_content(1024):
+                f.write(chunk)
+    except requests.exceptions.HTTPError:
+        print('HTTP Error %d fetching %s' % (r.status_code, name))
+        return None
+    return fn
+
+def make_artifact_dir(task_id, run_id):
+    prefix = 'tc-artifacts.%s.%d.' % (task_id, run_id)
+    print('making artifact dir %s' % prefix)
+    return tempfile.mkdtemp(prefix=prefix)
+
+def fetch_artifacts(queue, task_id, run_id):
+    '''
+    Fetch all artifacts from task_id and run_id in queue, write them to
+    temporary files, and yield the path to each.
+    '''
+    try:
+        tempdir = make_artifact_dir(task_id, run_id)
+        res = queue.listArtifacts(task_id, run_id)
+        for a in res['artifacts']:
+            # Skip logs
+            if a['name'].startswith('public/logs'):
+                continue
+            # Skip interfaces
+            if a['name'].startswith('private/docker-worker'):
+                continue
+            yield fetch_artifact(queue, task_id, run_id, a['name'], tempdir)
+    finally:
+        if os.path.isdir(tempdir):
+            #shutil.rmtree(tempdir)
+            print('Artifacts downloaded to %s' % tempdir)
+            pass
+
+def upload_to_tooltool(tooltool_auth, task_id, artifact):
+    '''
+    Upload artifact to tooltool using tooltool_auth as the authentication token.
+    Return the path to the generated tooltool manifest.
+    '''
+    try:
+        oldcwd = os.getcwd()
+        os.chdir(os.path.dirname(artifact))
+        manifest = artifact + '.manifest'
+        tooltool.main([
+            'tooltool.py',
+            'add',
+            '--visibility=public',
+            '-m', manifest,
+            artifact
+        ])
+        tooltool.main([
+            'tooltool.py',
+            'upload',
+            '-m', manifest,
+            '--authentication-file', tooltool_auth,
+            '--message', 'Built from taskcluster task {}'.format(task_id),
+        ])
+        return manifest
+    finally:
+        os.chdir(oldcwd)
+
+def update_manifest(artifact, manifest, local_gecko_clone):
+    platform = linux
+    manifest_dir = os.path.join(local_gecko_clone,
+                                'testing', 'config', 'tooltool-manifests')
+    platform_dir = [p for p in os.listdir(manifest_dir)
+                    if p.startswith(platform)][0]
+    tree_manifest = os.path.join(manifest_dir, platform_dir, 'releng.manifest')
+    print('%s -> %s' % (manifest, tree_manifest))
+    shutil.copyfile(manifest, tree_manifest)
+
+def main():
+    parser = argparse.ArgumentParser(description='Build and upload binaries')
+    parser.add_argument('taskcluster_auth', help='Path to a file containing Taskcluster client ID and authentication token as a JSON file in the form {"clientId": "...", "accessToken": "..."}')
+    parser.add_argument('--tooltool-auth', help='Path to a file containing a tooltool authentication token valid for uploading files')
+    parser.add_argument('--local-gecko-clone', help='Path to a local Gecko clone whose tooltool manifests will be updated with the newly-built binaries')
+    parser.add_argument('--rust-branch', default='stable',
+                        help='Revision of the rust repository to use')
+    parser.add_argument('--task', help='Use an existing task')
+
+    args = parser.parse_args()
+    tc_auth = read_tc_auth(args.taskcluster_auth)
+    queue = taskcluster.Queue({'credentials': tc_auth})
+    if args.task:
+        task_id, initial_wait = args.task, 0
+    else:
+        task_id, initial_wait = spawn_task(queue, args), 25
+    run_id = wait_for_task(queue, task_id, initial_wait)
+    for artifact in fetch_artifacts(queue, task_id, run_id):
+        if args.tooltool_auth:
+            manifest = upload_to_tooltool(args.tooltool_auth, task_id, artifact)
+        if args.local_gecko_clone:
+            update_manifest(artifact, manifest, args.local_gecko_clone)
+
+if __name__ == '__main__':
+    main()

testing/docker/rust-build/upload_rust.sh

@@ -0,0 +1,22 @@
+#!/bin/bash -vex
+
+set -x -e
+
+: WORKSPACE ${WORKSPACE:=/home/worker}
+
+set -v
+
+# Upload artifacts packaged by the build script.
+pushd ${WORKSPACE}
+if test -n "$TASK_ID"; then
+  # If we're running on task cluster, use the upload-capable tunnel.
+  TOOLTOOL_OPTS="--url=http://relengapi/tooltool/"
+  MESSAGE="Taskcluster upload ${TASK_ID}/${RUN_ID} $0"
+else
+  MESSAGE="Rust toolchain build for gecko"
+fi
+if test -r rust-version; then
+  MESSAGE="$MESSAGE $(cat rust-version)"
+fi
+/build/tooltool.py upload ${TOOLTOOL_OPTS} --message="${MESSAGE}"
+popd