wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-05 07:18:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

Browse source
This commit is contained in:
Roy Tam 2018-01-19 03:59:58 +08:00
commit dcd9973243
150858 changed files with 23884658 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

0
taskcluster/taskgraph/task/__init__.py Normal file
View file

108
taskcluster/taskgraph/task/base.py Normal file
View file

@ -0,0 +1,108 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import abc
class Task(object):
"""
Representation of a task in a TaskGraph. Each Task has, at creation:
- kind: the name of the task kind
- label; the label for this task
- attributes: a dictionary of attributes for this task (used for filtering)
- task: the task definition (JSON-able dictionary)
And later, as the task-graph processing proceeds:
- task_id -- TaskCluster taskId under which this task will be created
- optimized -- true if this task need not be performed
A kind represents a collection of tasks that share common characteristics.
For example, all build jobs. Each instance of a kind is intialized with a
path from which it draws its task configuration. The instance is free to
store as much local state as it needs.
"""
__metaclass__ = abc.ABCMeta
def __init__(self, kind, label, attributes, task):
self.kind = kind
self.label = label
self.attributes = attributes
self.task = task
self.task_id = None
self.optimized = False
self.attributes['kind'] = kind
def __eq__(self, other):
return self.kind == other.kind and \
self.label == other.label and \
self.attributes == other.attributes and \
self.task == other.task and \
self.task_id == other.task_id
@classmethod
@abc.abstractmethod
def load_tasks(cls, kind, path, config, parameters, loaded_tasks):
"""
Load the tasks for a given kind.
The `kind` is the name of the kind; the configuration for that kind
named this class.
The `path` is the path to the configuration directory for the kind. This
can be used to load extra data, templates, etc.
The `parameters` give details on which to base the task generation.
See `taskcluster/docs/parameters.rst` for details.
At the time this method is called, all kinds on which this kind depends
(that is, specified in the `kind-dependencies` key in `self.config`
have already loaded their tasks, and those tasks are available in
the list `loaded_tasks`.
The return value is a list of Task instances.
"""
@abc.abstractmethod
def get_dependencies(self, taskgraph):
"""
Get the set of task labels this task depends on, by querying the full
task set, given as `taskgraph`.
Returns a list of (task_label, dependency_name) pairs describing the
dependencies.
"""
def optimize(self, params):
"""
Determine whether this task can be optimized, and if it can, what taskId
it should be replaced with.
The return value is a tuple `(optimized, taskId)`. If `optimized` is
true, then the task will be optimized (in other words, not included in
the task graph). If the second argument is a taskid, then any
dependencies on this task will isntead depend on that taskId. It is an
error to return no taskId for a task on which other tasks depend.
The default never optimizes.
"""
return False, None
@classmethod
def from_json(cls, task_dict):
"""
Given a data structure as produced by taskgraph.to_json, re-construct
the original Task object. This is used to "resume" the task-graph
generation process, for example in Action tasks.
"""
return cls(
kind=task_dict['attributes']['kind'],
label=task_dict['label'],
attributes=task_dict['attributes'],
task=task_dict['task'])

130
taskcluster/taskgraph/task/docker_image.py Normal file
View file

@ -0,0 +1,130 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
import json
import os
import urllib2
from . import base
from taskgraph.util.docker import (
docker_image,
generate_context_hash,
INDEX_PREFIX,
)
from taskgraph.util.templates import Templates
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class DockerImageTask(base.Task):
def __init__(self, *args, **kwargs):
self.index_paths = kwargs.pop('index_paths')
super(DockerImageTask, self).__init__(*args, **kwargs)
def __eq__(self, other):
return super(DockerImageTask, self).__eq__(other) and \
self.index_paths == other.index_paths
@classmethod
def load_tasks(cls, kind, path, config, params, loaded_tasks):
parameters = {
'pushlog_id': params.get('pushlog_id', 0),
'pushdate': params['moz_build_date'],
'pushtime': params['moz_build_date'][8:],
'year': params['moz_build_date'][0:4],
'month': params['moz_build_date'][4:6],
'day': params['moz_build_date'][6:8],
'project': params['project'],
'docker_image': docker_image,
'base_repository': params['base_repository'] or params['head_repository'],
'head_repository': params['head_repository'],
'head_ref': params['head_ref'] or params['head_rev'],
'head_rev': params['head_rev'],
'owner': params['owner'],
'level': params['level'],
'source': '{repo}file/{rev}/taskcluster/ci/docker-image/image.yml'
.format(repo=params['head_repository'], rev=params['head_rev']),
'index_image_prefix': INDEX_PREFIX,
'artifact_path': 'public/image.tar.zst',
}
tasks = []
templates = Templates(path)
for image_name, image_symbol in config['images'].iteritems():
context_path = os.path.join('testing', 'docker', image_name)
context_hash = generate_context_hash(GECKO, context_path, image_name)
image_parameters = dict(parameters)
image_parameters['image_name'] = image_name
image_parameters['context_hash'] = context_hash
image_task = templates.load('image.yml', image_parameters)
attributes = {'image_name': image_name}
# unique symbol for different docker image
if 'extra' in image_task['task']:
image_task['task']['extra']['treeherder']['symbol'] = image_symbol
# As an optimization, if the context hash exists for a high level, that image
# task ID will be used. The reasoning behind this is that eventually everything ends
# up on level 3 at some point if most tasks use this as a common image
# for a given context hash, a worker within Taskcluster does not need to contain
# the same image per branch.
index_paths = ['{}.level-{}.{}.hash.{}'.format(
INDEX_PREFIX, level, image_name, context_hash)
for level in range(int(params['level']), 4)]
tasks.append(cls(kind, 'build-docker-image-' + image_name,
task=image_task['task'], attributes=attributes,
index_paths=index_paths))
return tasks
def get_dependencies(self, taskgraph):
return []
def optimize(self, params):
for index_path in self.index_paths:
try:
url = INDEX_URL.format(index_path)
existing_task = json.load(urllib2.urlopen(url))
# Only return the task ID if the artifact exists for the indexed
# task. Otherwise, continue on looking at each of the branches. Method
# continues trying other branches in case mozilla-central has an expired
# artifact, but 'project' might not. Only return no task ID if all
# branches have been tried
request = urllib2.Request(
ARTIFACT_URL.format(existing_task['taskId'], 'public/image.tar.zst'))
request.get_method = lambda: 'HEAD'
urllib2.urlopen(request)
# HEAD success on the artifact is enough
return True, existing_task['taskId']
except urllib2.HTTPError:
pass
return False, None
@classmethod
def from_json(cls, task_dict):
# Generating index_paths for optimization
imgMeta = task_dict['task']['extra']['imageMeta']
image_name = imgMeta['imageName']
context_hash = imgMeta['contextHash']
index_paths = ['{}.level-{}.{}.hash.{}'.format(
INDEX_PREFIX, level, image_name, context_hash)
for level in range(int(imgMeta['level']), 4)]
docker_image_task = cls(kind='docker-image',
label=task_dict['label'],
attributes=task_dict['attributes'],
task=task_dict['task'],
index_paths=index_paths)
return docker_image_task

53
taskcluster/taskgraph/task/post_build.py Normal file
View file

@ -0,0 +1,53 @@
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
import logging
from . import transform
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
class PostBuildTask(transform.TransformTask):
"""
A task implementing a post-build job. These depend on jobs and perform
various followup tasks after a build has completed.
The `only-for-build-platforms` kind configuration, if specified, will limit
the build platforms for which a post-build task will be created.
The `job-template' kind configuration points to a yaml file which will
be used to create the input to the transforms. It will have added to it
keys `build-label`, the label for the build task, and `build-platform`, its
platform.
"""
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
if config.get('kind-dependencies', []) != ["build"]:
raise Exception("PostBuildTask kinds must depend on builds")
only_platforms = config.get('only-for-build-platforms')
prototype = load_yaml(path, config.get('job-template'))
for task in loaded_tasks:
if task.kind != 'build':
continue
build_platform = task.attributes.get('build_platform')
build_type = task.attributes.get('build_type')
if not build_platform or not build_type:
continue
platform = "{}/{}".format(build_platform, build_type)
if only_platforms and platform not in only_platforms:
continue
post_task = copy.deepcopy(prototype)
post_task['build-label'] = task.label
post_task['build-platform'] = platform
yield post_task

64
taskcluster/taskgraph/task/signing.py Normal file
View file

@ -0,0 +1,64 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
import os
from . import base
from taskgraph.util.templates import Templates
logger = logging.getLogger(__name__)
GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class SigningTask(base.Task):
def __init__(self, kind, name, task, attributes):
self.unsigned_artifact_label = task['unsigned-task']['label']
super(SigningTask, self).__init__(kind, name, task=task['task'],
attributes=attributes)
@classmethod
def load_tasks(cls, kind, path, config, params, loaded_tasks):
root = os.path.abspath(path)
tasks = []
for filename in config.get('jobs-from', []):
templates = Templates(root)
jobs = templates.load(filename, {})
for name, job in jobs.iteritems():
for artifact in job['unsigned-task']['artifacts']:
url = ARTIFACT_URL.format('<{}>'.format('unsigned-artifact'), artifact)
job['task']['payload']['unsignedArtifacts'].append({
'task-reference': url
})
attributes = job.setdefault('attributes', {})
attributes.update({'kind': 'signing'})
tasks.append(cls(kind, name, job, attributes=attributes))
return tasks
def get_dependencies(self, taskgraph):
return [(self.unsigned_artifact_label, 'unsigned-artifact')]
def optimize(self, params):
return False, None
@classmethod
def from_json(cls, task_dict):
unsigned_task_label = task_dict['dependencies']['unsigned-artifact']
task_dict['unsigned-task'] = {
'label': unsigned_task_label
}
signing_task = cls(kind='build-signing',
name=task_dict['label'],
attributes=task_dict['attributes'],
task=task_dict)
return signing_task

112
taskcluster/taskgraph/task/test.py Normal file
View file

@ -0,0 +1,112 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
import logging
from . import transform
from ..util.yaml import load_yaml
logger = logging.getLogger(__name__)
class TestTask(transform.TransformTask):
"""
A task implementing a Gecko test.
"""
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
# the kind on which this one depends
if len(config.get('kind-dependencies', [])) != 1:
raise Exception("TestTask kinds must have exactly one item in kind-dependencies")
dep_kind = config['kind-dependencies'][0]
# get build tasks, keyed by build platform
builds_by_platform = cls.get_builds_by_platform(dep_kind, loaded_tasks)
# get the test platforms for those build tasks
test_platforms_cfg = load_yaml(path, 'test-platforms.yml')
test_platforms = cls.get_test_platforms(test_platforms_cfg, builds_by_platform)
# expand the test sets for each of those platforms
test_sets_cfg = load_yaml(path, 'test-sets.yml')
test_platforms = cls.expand_tests(test_sets_cfg, test_platforms)
# load the test descriptions
test_descriptions = load_yaml(path, 'tests.yml')
# generate all tests for all test platforms
for test_platform_name, test_platform in test_platforms.iteritems():
for test_name in test_platform['test-names']:
test = copy.deepcopy(test_descriptions[test_name])
test['build-platform'] = test_platform['build-platform']
test['test-platform'] = test_platform_name
test['build-label'] = test_platform['build-label']
test['test-name'] = test_name
logger.debug("Generating tasks for {} test {} on platform {}".format(
kind, test_name, test['test-platform']))
yield test
@classmethod
def get_builds_by_platform(cls, dep_kind, loaded_tasks):
"""Find the build tasks on which tests will depend, keyed by
platform/type. Returns a dictionary mapping build platform to task
label."""
builds_by_platform = {}
for task in loaded_tasks:
if task.kind != dep_kind:
continue
build_platform = task.attributes.get('build_platform')
build_type = task.attributes.get('build_type')
if not build_platform or not build_type:
continue
platform = "{}/{}".format(build_platform, build_type)
if platform in builds_by_platform:
raise Exception("multiple build jobs for " + platform)
builds_by_platform[platform] = task.label
return builds_by_platform
@classmethod
def get_test_platforms(cls, test_platforms_cfg, builds_by_platform):
"""Get the test platforms for which test tasks should be generated,
based on the available build platforms. Returns a dictionary mapping
test platform to {test-set, build-platform, build-label}."""
test_platforms = {}
for test_platform, cfg in test_platforms_cfg.iteritems():
build_platform = cfg['build-platform']
if build_platform not in builds_by_platform:
logger.warning(
"No build task with platform {}; ignoring test platform {}".format(
build_platform, test_platform))
continue
test_platforms[test_platform] = {
'test-set': cfg['test-set'],
'build-platform': build_platform,
'build-label': builds_by_platform[build_platform],
}
return test_platforms
@classmethod
def expand_tests(cls, test_sets_cfg, test_platforms):
"""Expand the test sets in `test_platforms` out to sets of test names.
Returns a dictionary like `get_test_platforms`, with an additional
`test-names` key for each test platform, containing a set of test
names."""
rv = {}
for test_platform, cfg in test_platforms.iteritems():
test_set = cfg['test-set']
if test_set not in test_sets_cfg:
raise Exception(
"Test set '{}' for test platform {} is not defined".format(
test_set, test_platform))
test_names = test_sets_cfg[test_set]
rv[test_platform] = cfg.copy()
rv[test_platform]['test-names'] = test_names
return rv

109
taskcluster/taskgraph/task/transform.py Normal file
View file

@ -0,0 +1,109 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import logging
import itertools
from . import base
from .. import files_changed
from ..util.python_path import find_object
from ..util.templates import merge
from ..util.yaml import load_yaml
from ..util.seta import is_low_value_task
from ..transforms.base import TransformSequence, TransformConfig
logger = logging.getLogger(__name__)
class TransformTask(base.Task):
"""
Tasks of this class are generated by applying transformations to a sequence
of input entities. By default, it gets those inputs from YAML data in the
kind directory, but subclasses may override `get_inputs` to produce them in
some other way.
"""
@classmethod
def get_inputs(cls, kind, path, config, params, loaded_tasks):
"""
Get the input elements that will be transformed into tasks. The
elements themselves are free-form, and become the input to the first
transform.
By default, this reads jobs from the `jobs` key, or from yaml files
named by `jobs-from`. The entities are read from mappings, and the
keys to those mappings are added in the `name` key of each entity.
If there is a `job-defaults` config, then every job is merged with it.
This provides a simple way to set default values for all jobs of a
kind. More complex defaults should be implemented with custom
transforms.
This method can be overridden in subclasses that need to perform more
complex calculations to generate the list of inputs.
"""
def jobs():
defaults = config.get('job-defaults')
jobs = config.get('jobs', {}).iteritems()
jobs_from = itertools.chain.from_iterable(
load_yaml(path, filename).iteritems()
for filename in config.get('jobs-from', {}))
for name, job in itertools.chain(jobs, jobs_from):
if defaults:
job = merge(defaults, job)
yield name, job
for name, job in jobs():
job['name'] = name
logger.debug("Generating tasks for {} {}".format(kind, name))
yield job
@classmethod
def load_tasks(cls, kind, path, config, params, loaded_tasks):
inputs = cls.get_inputs(kind, path, config, params, loaded_tasks)
transforms = TransformSequence()
for xform_path in config['transforms']:
transform = find_object(xform_path)
transforms.add(transform)
# perform the transformations
trans_config = TransformConfig(kind, path, config, params)
tasks = [cls(kind, t) for t in transforms(trans_config, inputs)]
return tasks
def __init__(self, kind, task):
self.dependencies = task['dependencies']
self.when = task['when']
super(TransformTask, self).__init__(kind, task['label'],
task['attributes'], task['task'])
def get_dependencies(self, taskgraph):
return [(label, name) for name, label in self.dependencies.items()]
def optimize(self, params):
if 'files-changed' in self.when:
changed = files_changed.check(
params, self.when['files-changed'])
if not changed:
logger.debug('no files found matching a pattern in `when.files-changed` for ' +
self.label)
return True, None
# we would like to return 'False, None' while it's high_value_task
# and we wouldn't optimize it. Otherwise, it will return 'True, None'
if is_low_value_task(self.label, params.get('project')):
# Always optimize away low-value tasks
return True, None
else:
return False, None
@classmethod
def from_json(cls, task_dict):
# when reading back from JSON, we lose the "when" information
task_dict['when'] = {}
return cls(task_dict['attributes']['kind'], task_dict)