mirror of
https://repo.dactyloidae.xyz/Dactyloidae/UXP.git
synced 2026-09-24 09:27:31 +09:00
1076 lines
41 KiB
Python
1076 lines
41 KiB
Python
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
import json
|
|
import os
|
|
import random
|
|
import re
|
|
import socket
|
|
import sys
|
|
import time
|
|
import traceback
|
|
import unittest
|
|
|
|
from argparse import ArgumentParser
|
|
from copy import deepcopy
|
|
|
|
import mozinfo
|
|
import moznetwork
|
|
import mozprofile
|
|
import mozversion
|
|
import serve
|
|
|
|
from manifestparser import TestManifest
|
|
from manifestparser.filters import tags
|
|
from marionette_driver.marionette import Marionette
|
|
from moztest.adapters.unit import StructuredTestResult, StructuredTestRunner
|
|
from moztest.results import TestResult, TestResultCollection, relevant_line
|
|
from serve import iter_proc, iter_url
|
|
|
|
here = os.path.abspath(os.path.dirname(__file__))
|
|
|
|
|
|
def update_mozinfo(path=None):
|
|
"""Walk up directories to find mozinfo.json and update the info."""
|
|
path = path or here
|
|
dirs = set()
|
|
while path != os.path.expanduser('~'):
|
|
if path in dirs:
|
|
break
|
|
dirs.add(path)
|
|
path = os.path.split(path)[0]
|
|
|
|
return mozinfo.find_and_update_from_json(*dirs)
|
|
|
|
|
|
class MarionetteTest(TestResult):
|
|
|
|
@property
|
|
def test_name(self):
|
|
if self.test_class is not None:
|
|
return '{0}.py {1}.{2}'.format(self.test_class.split('.')[0],
|
|
self.test_class,
|
|
self.name)
|
|
else:
|
|
return self.name
|
|
|
|
|
|
class MarionetteTestResult(StructuredTestResult, TestResultCollection):
|
|
|
|
resultClass = MarionetteTest
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
self.marionette = kwargs.pop('marionette')
|
|
TestResultCollection.__init__(self, 'MarionetteTest')
|
|
self.passed = 0
|
|
self.testsRun = 0
|
|
self.result_modifiers = [] # used by mixins to modify the result
|
|
StructuredTestResult.__init__(self, *args, **kwargs)
|
|
|
|
@property
|
|
def skipped(self):
|
|
return [t for t in self if t.result == 'SKIPPED']
|
|
|
|
@skipped.setter
|
|
def skipped(self, value):
|
|
pass
|
|
|
|
@property
|
|
def expectedFailures(self):
|
|
return [t for t in self if t.result == 'KNOWN-FAIL']
|
|
|
|
@expectedFailures.setter
|
|
def expectedFailures(self, value):
|
|
pass
|
|
|
|
@property
|
|
def unexpectedSuccesses(self):
|
|
return [t for t in self if t.result == 'UNEXPECTED-PASS']
|
|
|
|
@unexpectedSuccesses.setter
|
|
def unexpectedSuccesses(self, value):
|
|
pass
|
|
|
|
@property
|
|
def tests_passed(self):
|
|
return [t for t in self if t.result == 'PASS']
|
|
|
|
@property
|
|
def errors(self):
|
|
return [t for t in self if t.result == 'ERROR']
|
|
|
|
@errors.setter
|
|
def errors(self, value):
|
|
pass
|
|
|
|
@property
|
|
def failures(self):
|
|
return [t for t in self if t.result == 'UNEXPECTED-FAIL']
|
|
|
|
@failures.setter
|
|
def failures(self, value):
|
|
pass
|
|
|
|
@property
|
|
def duration(self):
|
|
if self.stop_time:
|
|
return self.stop_time - self.start_time
|
|
else:
|
|
return 0
|
|
|
|
def add_test_result(self, test, result_expected='PASS',
|
|
result_actual='PASS', output='', context=None, **kwargs):
|
|
def get_class(test):
|
|
return test.__class__.__module__ + '.' + test.__class__.__name__
|
|
|
|
name = str(test).split()[0]
|
|
test_class = get_class(test)
|
|
if hasattr(test, 'jsFile'):
|
|
name = os.path.basename(test.jsFile)
|
|
test_class = None
|
|
|
|
t = self.resultClass(name=name, test_class=test_class,
|
|
time_start=test.start_time, result_expected=result_expected,
|
|
context=context, **kwargs)
|
|
# call any registered result modifiers
|
|
for modifier in self.result_modifiers:
|
|
result_expected, result_actual, output, context = modifier(
|
|
t, result_expected, result_actual, output, context)
|
|
t.finish(result_actual,
|
|
time_end=time.time() if test.start_time else 0,
|
|
reason=relevant_line(output),
|
|
output=output)
|
|
self.append(t)
|
|
|
|
def addError(self, test, err):
|
|
self.add_test_result(test, output=self._exc_info_to_string(err, test),
|
|
result_actual='ERROR')
|
|
super(MarionetteTestResult, self).addError(test, err)
|
|
|
|
def addFailure(self, test, err):
|
|
self.add_test_result(test, output=self._exc_info_to_string(err, test),
|
|
result_actual='UNEXPECTED-FAIL')
|
|
super(MarionetteTestResult, self).addFailure(test, err)
|
|
|
|
def addSuccess(self, test):
|
|
self.passed += 1
|
|
self.add_test_result(test, result_actual='PASS')
|
|
super(MarionetteTestResult, self).addSuccess(test)
|
|
|
|
def addExpectedFailure(self, test, err):
|
|
"""Called when an expected failure/error occured."""
|
|
self.add_test_result(test, output=self._exc_info_to_string(err, test),
|
|
result_actual='KNOWN-FAIL')
|
|
super(MarionetteTestResult, self).addExpectedFailure(test, err)
|
|
|
|
def addUnexpectedSuccess(self, test):
|
|
"""Called when a test was expected to fail, but succeed."""
|
|
self.add_test_result(test, result_actual='UNEXPECTED-PASS')
|
|
super(MarionetteTestResult, self).addUnexpectedSuccess(test)
|
|
|
|
def addSkip(self, test, reason):
|
|
self.add_test_result(test, output=reason, result_actual='SKIPPED')
|
|
super(MarionetteTestResult, self).addSkip(test, reason)
|
|
|
|
def getInfo(self, test):
|
|
return test.test_name
|
|
|
|
def getDescription(self, test):
|
|
doc_first_line = test.shortDescription()
|
|
if self.descriptions and doc_first_line:
|
|
return '\n'.join((str(test), doc_first_line))
|
|
else:
|
|
desc = str(test)
|
|
if hasattr(test, 'jsFile'):
|
|
desc = "{0}, {1}".format(test.jsFile, desc)
|
|
return desc
|
|
|
|
def printLogs(self, test):
|
|
for testcase in test._tests:
|
|
if hasattr(testcase, 'loglines') and testcase.loglines:
|
|
# Don't dump loglines to the console if they only contain
|
|
# TEST-START and TEST-END.
|
|
skip_log = True
|
|
for line in testcase.loglines:
|
|
str_line = ' '.join(line)
|
|
if 'TEST-END' not in str_line and 'TEST-START' not in str_line:
|
|
skip_log = False
|
|
break
|
|
if skip_log:
|
|
return
|
|
self.logger.info('START LOG:')
|
|
for line in testcase.loglines:
|
|
self.logger.info(' '.join(line).encode('ascii', 'replace'))
|
|
self.logger.info('END LOG:')
|
|
|
|
def stopTest(self, *args, **kwargs):
|
|
unittest._TextTestResult.stopTest(self, *args, **kwargs)
|
|
if self.marionette.check_for_crash():
|
|
# this tells unittest.TestSuite not to continue running tests
|
|
self.shouldStop = True
|
|
test = next((a for a in args if isinstance(a, unittest.TestCase)),
|
|
None)
|
|
if test:
|
|
self.addError(test, sys.exc_info())
|
|
|
|
|
|
class MarionetteTextTestRunner(StructuredTestRunner):
|
|
|
|
resultclass = MarionetteTestResult
|
|
|
|
def __init__(self, **kwargs):
|
|
self.marionette = kwargs.pop('marionette')
|
|
self.capabilities = kwargs.pop('capabilities')
|
|
|
|
StructuredTestRunner.__init__(self, **kwargs)
|
|
|
|
def _makeResult(self):
|
|
return self.resultclass(self.stream,
|
|
self.descriptions,
|
|
self.verbosity,
|
|
marionette=self.marionette,
|
|
logger=self.logger,
|
|
result_callbacks=self.result_callbacks)
|
|
|
|
def run(self, test):
|
|
result = super(MarionetteTextTestRunner, self).run(test)
|
|
result.printLogs(test)
|
|
return result
|
|
|
|
|
|
class BaseMarionetteArguments(ArgumentParser):
|
|
# Bug 1336953 - Until we can remove the socket timeout parameter it has to be
|
|
# set a default value which is larger than the longest timeout as defined by the
|
|
# WebDriver spec. In that case its 300s for page load. Also add another minute
|
|
# so that slow builds have enough time to send the timeout error to the client.
|
|
socket_timeout_default = 360.0
|
|
|
|
def __init__(self, **kwargs):
|
|
ArgumentParser.__init__(self, **kwargs)
|
|
|
|
def dir_path(path):
|
|
path = os.path.abspath(os.path.expanduser(path))
|
|
if not os.access(path, os.F_OK):
|
|
os.makedirs(path)
|
|
return path
|
|
|
|
self.argument_containers = []
|
|
self.add_argument('tests',
|
|
nargs='*',
|
|
default=[],
|
|
help='Tests to run. '
|
|
'One or more paths to test files (Python or JS), '
|
|
'manifest files (.ini) or directories. '
|
|
'When a directory is specified, '
|
|
'all test files in the directory will be run.')
|
|
self.add_argument('--binary',
|
|
help='path to gecko executable to launch before running the test')
|
|
self.add_argument('--address',
|
|
help='host:port of running Gecko instance to connect to')
|
|
self.add_argument('--emulator',
|
|
action='store_true',
|
|
help='If no --address is given, then the harness will launch an '
|
|
'emulator. (See Remote options group.) '
|
|
'If --address is given, then the harness assumes you are '
|
|
'running an emulator already, and will launch gecko app '
|
|
'on that emulator.')
|
|
self.add_argument('--app',
|
|
help='application to use. see marionette_driver.geckoinstance')
|
|
self.add_argument('--app-arg',
|
|
dest='app_args',
|
|
action='append',
|
|
default=[],
|
|
help='specify a command line argument to be passed onto the application')
|
|
self.add_argument('--profile',
|
|
help='profile to use when launching the gecko process. If not passed, '
|
|
'then a profile will be constructed and used',
|
|
type=dir_path)
|
|
self.add_argument('--pref',
|
|
action='append',
|
|
dest='prefs_args',
|
|
help="A preference to set. Must be a key-value pair separated by a ':'.")
|
|
self.add_argument('--preferences',
|
|
action='append',
|
|
dest='prefs_files',
|
|
help="read preferences from a JSON or INI file. For INI, use "
|
|
"'file.ini:section' to specify a particular section.")
|
|
self.add_argument('--addon',
|
|
action='append',
|
|
dest='addons',
|
|
help="addon to install; repeat for multiple addons.")
|
|
self.add_argument('--repeat',
|
|
type=int,
|
|
default=0,
|
|
help='number of times to repeat the test(s)')
|
|
self.add_argument('--testvars',
|
|
action='append',
|
|
help='path to a json file with any test data required')
|
|
self.add_argument('--symbols-path',
|
|
help='absolute path to directory containing breakpad symbols, or the '
|
|
'url of a zip file containing symbols')
|
|
self.add_argument('--startup-timeout',
|
|
type=int,
|
|
default=60,
|
|
help='the max number of seconds to wait for a Marionette connection '
|
|
'after launching a binary')
|
|
self.add_argument('--shuffle',
|
|
action='store_true',
|
|
default=False,
|
|
help='run tests in a random order')
|
|
self.add_argument('--shuffle-seed',
|
|
type=int,
|
|
default=random.randint(0, sys.maxint),
|
|
help='Use given seed to shuffle tests')
|
|
self.add_argument('--total-chunks',
|
|
type=int,
|
|
help='how many chunks to split the tests up into')
|
|
self.add_argument('--this-chunk',
|
|
type=int,
|
|
help='which chunk to run')
|
|
self.add_argument('--server-root',
|
|
help='url to a webserver or path to a document root from which content '
|
|
'resources are served (default: {}).'.format(os.path.join(
|
|
os.path.dirname(here), 'www')))
|
|
self.add_argument('--gecko-log',
|
|
help="Define the path to store log file. If the path is"
|
|
" a directory, the real log file will be created"
|
|
" given the format gecko-(timestamp).log. If it is"
|
|
" a file, if will be used directly. '-' may be passed"
|
|
" to write to stdout. Default: './gecko.log'")
|
|
self.add_argument('--logger-name',
|
|
default='Marionette-based Tests',
|
|
help='Define the name to associate with the logger used')
|
|
self.add_argument('--jsdebugger',
|
|
action='store_true',
|
|
default=False,
|
|
help='Enable the jsdebugger for marionette javascript.')
|
|
self.add_argument('--pydebugger',
|
|
help='Enable python post-mortem debugger when a test fails.'
|
|
' Pass in the debugger you want to use, eg pdb or ipdb.')
|
|
self.add_argument('--socket-timeout',
|
|
type=float,
|
|
default=self.socket_timeout_default,
|
|
help='Set the global timeout for marionette socket operations.'
|
|
' Default: %(default)ss.')
|
|
self.add_argument('--disable-e10s',
|
|
action='store_false',
|
|
dest='e10s',
|
|
default=True,
|
|
help='Disable e10s when running marionette tests.')
|
|
self.add_argument('--tag',
|
|
action='append', dest='test_tags',
|
|
default=None,
|
|
help="Filter out tests that don't have the given tag. Can be "
|
|
"used multiple times in which case the test must contain "
|
|
"at least one of the given tags.")
|
|
self.add_argument('--workspace',
|
|
action='store',
|
|
default=None,
|
|
help="Path to directory for Marionette output. "
|
|
"(Default: .) (Default profile dest: TMP)",
|
|
type=dir_path)
|
|
self.add_argument('-v', '--verbose',
|
|
action='count',
|
|
help='Increase verbosity to include debug messages with -v, '
|
|
'and trace messages with -vv.')
|
|
self.register_argument_container(RemoteMarionetteArguments())
|
|
|
|
def register_argument_container(self, container):
|
|
group = self.add_argument_group(container.name)
|
|
|
|
for cli, kwargs in container.args:
|
|
group.add_argument(*cli, **kwargs)
|
|
|
|
self.argument_containers.append(container)
|
|
|
|
def parse_known_args(self, args=None, namespace=None):
|
|
args, remainder = ArgumentParser.parse_known_args(self, args, namespace)
|
|
for container in self.argument_containers:
|
|
if hasattr(container, 'parse_args_handler'):
|
|
container.parse_args_handler(args)
|
|
return (args, remainder)
|
|
|
|
def _get_preferences(self, prefs_files, prefs_args):
|
|
"""Return user defined profile preferences as a dict."""
|
|
# object that will hold the preferences
|
|
prefs = mozprofile.prefs.Preferences()
|
|
|
|
# add preferences files
|
|
if prefs_files:
|
|
for prefs_file in prefs_files:
|
|
prefs.add_file(prefs_file)
|
|
|
|
separator = ':'
|
|
cli_prefs = []
|
|
if prefs_args:
|
|
misformatted = []
|
|
for pref in prefs_args:
|
|
if separator not in pref:
|
|
misformatted.append(pref)
|
|
else:
|
|
cli_prefs.append(pref.split(separator, 1))
|
|
if misformatted:
|
|
self._print_message("Warning: Ignoring preferences not in key{}value format: {}\n"
|
|
.format(separator, ", ".join(misformatted)))
|
|
# string preferences
|
|
prefs.add(cli_prefs, cast=True)
|
|
|
|
return dict(prefs())
|
|
|
|
def verify_usage(self, args):
|
|
if not args.tests:
|
|
self.error('You must specify one or more test files, manifests, or directories.')
|
|
|
|
missing_tests = [path for path in args.tests if not os.path.exists(path)]
|
|
if missing_tests:
|
|
self.error("Test file(s) not found: " + " ".join([path for path in missing_tests]))
|
|
|
|
if not args.address and not args.binary and not args.emulator:
|
|
self.error('You must specify --binary, or --address, or --emulator')
|
|
|
|
if args.total_chunks is not None and args.this_chunk is None:
|
|
self.error('You must specify which chunk to run.')
|
|
|
|
if args.this_chunk is not None and args.total_chunks is None:
|
|
self.error('You must specify how many chunks to split the tests into.')
|
|
|
|
if args.total_chunks is not None:
|
|
if not 1 < args.total_chunks:
|
|
self.error('Total chunks must be greater than 1.')
|
|
if not 1 <= args.this_chunk <= args.total_chunks:
|
|
self.error('Chunk to run must be between 1 and {}.'.format(args.total_chunks))
|
|
|
|
if args.jsdebugger:
|
|
args.app_args.append('-jsdebugger')
|
|
args.socket_timeout = None
|
|
|
|
args.prefs = self._get_preferences(args.prefs_files, args.prefs_args)
|
|
|
|
for container in self.argument_containers:
|
|
if hasattr(container, 'verify_usage_handler'):
|
|
container.verify_usage_handler(args)
|
|
|
|
return args
|
|
|
|
|
|
class RemoteMarionetteArguments(object):
|
|
name = 'Remote (Emulator/Device)'
|
|
args = [
|
|
[['--emulator-binary'],
|
|
{'help': 'Path to emulator binary. By default mozrunner uses `which emulator`',
|
|
'dest': 'emulator_bin',
|
|
}],
|
|
[['--adb'],
|
|
{'help': 'Path to the adb. By default mozrunner uses `which adb`',
|
|
'dest': 'adb_path'
|
|
}],
|
|
[['--avd'],
|
|
{'help': ('Name of an AVD available in your environment.'
|
|
'See mozrunner.FennecEmulatorRunner'),
|
|
}],
|
|
[['--avd-home'],
|
|
{'help': 'Path to avd parent directory',
|
|
}],
|
|
[['--device'],
|
|
{'help': ('Serial ID to connect to as seen in `adb devices`,'
|
|
'e.g emulator-5444'),
|
|
'dest': 'device_serial',
|
|
}],
|
|
[['--package'],
|
|
{'help': 'Name of Android package, e.g. org.mozilla.fennec',
|
|
'dest': 'package_name',
|
|
}],
|
|
|
|
]
|
|
|
|
|
|
class Fixtures(object):
|
|
def where_is(self, uri, on="http"):
|
|
return serve.where_is(uri, on)
|
|
|
|
|
|
class BaseMarionetteTestRunner(object):
|
|
|
|
textrunnerclass = MarionetteTextTestRunner
|
|
driverclass = Marionette
|
|
|
|
def __init__(self, address=None,
|
|
app=None, app_args=None, binary=None, profile=None,
|
|
logger=None, logdir=None,
|
|
repeat=0, testvars=None,
|
|
symbols_path=None,
|
|
shuffle=False, shuffle_seed=random.randint(0, sys.maxint), this_chunk=1,
|
|
total_chunks=1,
|
|
server_root=None, gecko_log=None, result_callbacks=None,
|
|
prefs=None, test_tags=None,
|
|
socket_timeout=BaseMarionetteArguments.socket_timeout_default,
|
|
startup_timeout=None, addons=None, workspace=None,
|
|
verbose=0, e10s=True, emulator=False, **kwargs):
|
|
self._appinfo = None
|
|
self._appName = None
|
|
self._capabilities = None
|
|
self._filename_pattern = None
|
|
self._version_info = {}
|
|
|
|
self.fixture_servers = {}
|
|
self.fixtures = Fixtures()
|
|
self.extra_kwargs = kwargs
|
|
self.test_kwargs = deepcopy(kwargs)
|
|
self.address = address
|
|
self.app = app
|
|
self.app_args = app_args or []
|
|
self.bin = binary
|
|
self.emulator = emulator
|
|
self.profile = profile
|
|
self.addons = addons
|
|
self.logger = logger
|
|
self.marionette = None
|
|
self.logdir = logdir
|
|
self.repeat = repeat
|
|
self.symbols_path = symbols_path
|
|
self.socket_timeout = socket_timeout
|
|
self.shuffle = shuffle
|
|
self.shuffle_seed = shuffle_seed
|
|
self.server_root = server_root
|
|
self.this_chunk = this_chunk
|
|
self.total_chunks = total_chunks
|
|
self.mixin_run_tests = []
|
|
self.manifest_skipped_tests = []
|
|
self.tests = []
|
|
self.result_callbacks = result_callbacks or []
|
|
self.prefs = prefs or {}
|
|
self.test_tags = test_tags
|
|
self.startup_timeout = startup_timeout
|
|
self.workspace = workspace
|
|
# If no workspace is set, default location for gecko.log is .
|
|
# and default location for profile is TMP
|
|
self.workspace_path = workspace or os.getcwd()
|
|
self.verbose = verbose
|
|
self.e10s = e10s
|
|
if self.e10s:
|
|
self.prefs.update({
|
|
'browser.tabs.remote.autostart': True,
|
|
'browser.tabs.remote.force-enable': True,
|
|
'extensions.e10sBlocksEnabling': False
|
|
})
|
|
|
|
def gather_debug(test, status):
|
|
# No screenshots and page source for skipped tests
|
|
if status == "SKIP":
|
|
return
|
|
|
|
rv = {}
|
|
marionette = test._marionette_weakref()
|
|
|
|
# In the event we're gathering debug without starting a session,
|
|
# skip marionette commands
|
|
if marionette.session is not None:
|
|
try:
|
|
with marionette.using_context(marionette.CONTEXT_CHROME):
|
|
rv['screenshot'] = marionette.screenshot()
|
|
with marionette.using_context(marionette.CONTEXT_CONTENT):
|
|
rv['source'] = marionette.page_source
|
|
except Exception as exc:
|
|
self.logger.warning('Failed to gather test failure debug: {}'.format(exc))
|
|
return rv
|
|
|
|
self.result_callbacks.append(gather_debug)
|
|
|
|
# testvars are set up in self.testvars property
|
|
self._testvars = None
|
|
self.testvars_paths = testvars
|
|
|
|
self.test_handlers = []
|
|
|
|
self.reset_test_stats()
|
|
|
|
self.logger.info('Using workspace for temporary data: '
|
|
'"{}"'.format(self.workspace_path))
|
|
|
|
if not gecko_log:
|
|
self.gecko_log = os.path.join(self.workspace_path or '', 'gecko.log')
|
|
else:
|
|
self.gecko_log = gecko_log
|
|
|
|
self.results = []
|
|
|
|
@property
|
|
def filename_pattern(self):
|
|
if self._filename_pattern is None:
|
|
self._filename_pattern = re.compile(
|
|
"^test(((_.+?)+?\.((py)|(js)))|(([A-Z].*?)+?\.js))$")
|
|
|
|
return self._filename_pattern
|
|
|
|
@property
|
|
def testvars(self):
|
|
if self._testvars is not None:
|
|
return self._testvars
|
|
|
|
self._testvars = {}
|
|
|
|
def update(d, u):
|
|
"""Update a dictionary that may contain nested dictionaries."""
|
|
for k, v in u.iteritems():
|
|
o = d.get(k, {})
|
|
if isinstance(v, dict) and isinstance(o, dict):
|
|
d[k] = update(d.get(k, {}), v)
|
|
else:
|
|
d[k] = u[k]
|
|
return d
|
|
|
|
json_testvars = self._load_testvars()
|
|
for j in json_testvars:
|
|
self._testvars = update(self._testvars, j)
|
|
return self._testvars
|
|
|
|
def _load_testvars(self):
|
|
data = []
|
|
if self.testvars_paths is not None:
|
|
for path in list(self.testvars_paths):
|
|
path = os.path.abspath(os.path.expanduser(path))
|
|
if not os.path.exists(path):
|
|
raise IOError('--testvars file {} does not exist'.format(path))
|
|
try:
|
|
with open(path) as f:
|
|
data.append(json.loads(f.read()))
|
|
except ValueError as e:
|
|
exc, val, tb = sys.exc_info()
|
|
msg = "JSON file ({0}) is not properly formatted: {1}"
|
|
raise exc, msg.format(os.path.abspath(path), e.message), tb
|
|
return data
|
|
|
|
@property
|
|
def capabilities(self):
|
|
if self._capabilities:
|
|
return self._capabilities
|
|
|
|
self.marionette.start_session()
|
|
self._capabilities = self.marionette.session_capabilities
|
|
self.marionette.delete_session()
|
|
return self._capabilities
|
|
|
|
@property
|
|
def appinfo(self):
|
|
if self._appinfo:
|
|
return self._appinfo
|
|
|
|
self.marionette.start_session()
|
|
with self.marionette.using_context('chrome'):
|
|
self._appinfo = self.marionette.execute_script("""
|
|
try {
|
|
return Services.appinfo;
|
|
} catch (e) {
|
|
return null;
|
|
}""")
|
|
self.marionette.delete_session()
|
|
self._appinfo = self._appinfo or {}
|
|
return self._appinfo
|
|
|
|
@property
|
|
def appName(self):
|
|
if self._appName:
|
|
return self._appName
|
|
|
|
self._appName = self.capabilities.get('browserName')
|
|
return self._appName
|
|
|
|
@property
|
|
def bin(self):
|
|
return self._bin
|
|
|
|
@bin.setter
|
|
def bin(self, path):
|
|
"""Set binary and reset parts of runner accordingly.
|
|
Intended use: to change binary between calls to run_tests
|
|
"""
|
|
self._bin = path
|
|
self.tests = []
|
|
self.cleanup()
|
|
|
|
@property
|
|
def version_info(self):
|
|
if not self._version_info:
|
|
try:
|
|
# TODO: Get version_info in Fennec case
|
|
self._version_info = mozversion.get_version(binary=self.bin)
|
|
except Exception:
|
|
self.logger.warning("Failed to retrieve version information for {}".format(
|
|
self.bin))
|
|
return self._version_info
|
|
|
|
def reset_test_stats(self):
|
|
self.passed = 0
|
|
self.failed = 0
|
|
self.crashed = 0
|
|
self.unexpected_successes = 0
|
|
self.todo = 0
|
|
self.skipped = 0
|
|
self.failures = []
|
|
|
|
def _build_kwargs(self):
|
|
if self.logdir and not os.access(self.logdir, os.F_OK):
|
|
os.mkdir(self.logdir)
|
|
|
|
kwargs = {
|
|
'socket_timeout': self.socket_timeout,
|
|
'prefs': self.prefs,
|
|
'startup_timeout': self.startup_timeout,
|
|
'verbose': self.verbose,
|
|
'symbols_path': self.symbols_path,
|
|
}
|
|
if self.bin or self.emulator:
|
|
kwargs.update({
|
|
'host': 'localhost',
|
|
'port': 2828,
|
|
'app': self.app,
|
|
'app_args': self.app_args,
|
|
'profile': self.profile,
|
|
'addons': self.addons,
|
|
'gecko_log': self.gecko_log,
|
|
# ensure Marionette class takes care of starting gecko instance
|
|
'bin': True,
|
|
})
|
|
|
|
if self.bin:
|
|
kwargs.update({
|
|
'bin': self.bin,
|
|
})
|
|
|
|
if self.emulator:
|
|
kwargs.update({
|
|
'avd_home': self.extra_kwargs.get('avd_home'),
|
|
'adb_path': self.extra_kwargs.get('adb_path'),
|
|
'emulator_binary': self.extra_kwargs.get('emulator_bin'),
|
|
'avd': self.extra_kwargs.get('avd'),
|
|
'package_name': self.extra_kwargs.get('package_name'),
|
|
})
|
|
|
|
if self.address:
|
|
host, port = self.address.split(':')
|
|
kwargs.update({
|
|
'host': host,
|
|
'port': int(port),
|
|
})
|
|
if self.emulator:
|
|
kwargs.update({
|
|
'connect_to_running_emulator': True,
|
|
})
|
|
if not self.bin and not self.emulator:
|
|
try:
|
|
# Establish a socket connection so we can vertify the data come back
|
|
connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
connection.connect((host, int(port)))
|
|
connection.close()
|
|
except Exception as e:
|
|
exc, val, tb = sys.exc_info()
|
|
msg = "Connection attempt to {0}:{1} failed with error: {2}"
|
|
raise exc, msg.format(host, port, e), tb
|
|
if self.workspace:
|
|
kwargs['workspace'] = self.workspace_path
|
|
return kwargs
|
|
|
|
def record_crash(self):
|
|
crash = True
|
|
try:
|
|
crash = self.marionette.check_for_crash()
|
|
self.crashed += int(crash)
|
|
except Exception:
|
|
traceback.print_exc()
|
|
return crash
|
|
|
|
def _initialize_test_run(self, tests):
|
|
assert len(tests) > 0
|
|
assert len(self.test_handlers) > 0
|
|
self.reset_test_stats()
|
|
|
|
def _add_tests(self, tests):
|
|
for test in tests:
|
|
self.add_test(test)
|
|
|
|
invalid_tests = [t['filepath'] for t in self.tests
|
|
if not self._is_filename_valid(t['filepath'])]
|
|
if invalid_tests:
|
|
raise Exception("Test file names must be of the form "
|
|
"'test_something.py', 'test_something.js', or 'testSomething.js'."
|
|
" Invalid test names:\n {}".format('\n '.join(invalid_tests)))
|
|
|
|
def _is_filename_valid(self, filename):
|
|
filename = os.path.basename(filename)
|
|
return self.filename_pattern.match(filename)
|
|
|
|
def _log_skipped_tests(self):
|
|
for test in self.manifest_skipped_tests:
|
|
name = os.path.basename(test['path'])
|
|
self.logger.test_start(name)
|
|
self.logger.test_end(name,
|
|
'SKIP',
|
|
message=test['disabled'])
|
|
self.todo += 1
|
|
|
|
def run_tests(self, tests):
|
|
start_time = time.time()
|
|
self._initialize_test_run(tests)
|
|
|
|
if self.marionette is None:
|
|
self.marionette = self.driverclass(**self._build_kwargs())
|
|
self.logger.info("Profile path is %s" % self.marionette.profile_path)
|
|
|
|
if len(self.fixture_servers) == 0 or \
|
|
any(not server.is_alive for _, server in self.fixture_servers):
|
|
self.logger.info("Starting fixture servers")
|
|
self.fixture_servers = self.start_fixture_servers()
|
|
for url in iter_url(self.fixture_servers):
|
|
self.logger.info("Fixture server listening on %s" % url)
|
|
|
|
# backwards compatibility
|
|
self.marionette.baseurl = serve.where_is("/")
|
|
|
|
self._add_tests(tests)
|
|
|
|
device_info = None
|
|
if self.marionette.instance and self.emulator:
|
|
try:
|
|
device_info = self.marionette.instance.runner.device.dm.getInfo()
|
|
except Exception:
|
|
self.logger.warning('Could not get device info', exc_info=True)
|
|
|
|
appinfo_e10s = self.appinfo.get('browserTabsRemoteAutostart', False)
|
|
self.logger.info("e10s is {}".format("enabled" if appinfo_e10s else "disabled"))
|
|
if self.e10s != appinfo_e10s:
|
|
message_e10s = ("BaseMarionetteTestRunner configuration (self.e10s) does "
|
|
"not match browser appinfo")
|
|
self.cleanup()
|
|
raise AssertionError(message_e10s)
|
|
|
|
self.logger.suite_start(self.tests,
|
|
version_info=self.version_info,
|
|
device_info=device_info)
|
|
|
|
self._log_skipped_tests()
|
|
|
|
interrupted = None
|
|
try:
|
|
counter = self.repeat
|
|
while counter >= 0:
|
|
round_num = self.repeat - counter
|
|
if round_num > 0:
|
|
self.logger.info('\nREPEAT {}\n-------'.format(round_num))
|
|
self.run_test_sets()
|
|
counter -= 1
|
|
except KeyboardInterrupt:
|
|
# in case of KeyboardInterrupt during the test execution
|
|
# we want to display current test results.
|
|
# so we keep the exception to raise it later.
|
|
interrupted = sys.exc_info()
|
|
except:
|
|
# For any other exception we return immediately and have to
|
|
# cleanup running processes
|
|
self.cleanup()
|
|
raise
|
|
|
|
try:
|
|
self._print_summary(tests)
|
|
self.record_crash()
|
|
self.elapsedtime = time.time() - start_time
|
|
|
|
for run_tests in self.mixin_run_tests:
|
|
run_tests(tests)
|
|
if self.shuffle:
|
|
self.logger.info("Using shuffle seed: %d" % self.shuffle_seed)
|
|
|
|
self.logger.suite_end()
|
|
except:
|
|
# raise only the exception if we were not interrupted
|
|
if not interrupted:
|
|
raise
|
|
finally:
|
|
self.cleanup()
|
|
|
|
# reraise previous interruption now
|
|
if interrupted:
|
|
raise interrupted[0], interrupted[1], interrupted[2]
|
|
|
|
def _print_summary(self, tests):
|
|
self.logger.info('\nSUMMARY\n-------')
|
|
self.logger.info('passed: {}'.format(self.passed))
|
|
if self.unexpected_successes == 0:
|
|
self.logger.info('failed: {}'.format(self.failed))
|
|
else:
|
|
self.logger.info(
|
|
'failed: {0} (unexpected sucesses: {1})'.format(self.failed,
|
|
self.unexpected_successes))
|
|
if self.skipped == 0:
|
|
self.logger.info('todo: {}'.format(self.todo))
|
|
else:
|
|
self.logger.info('todo: {0} (skipped: {1})'.format(self.todo, self.skipped))
|
|
|
|
if self.failed > 0:
|
|
self.logger.info('\nFAILED TESTS\n-------')
|
|
for failed_test in self.failures:
|
|
self.logger.info('{}'.format(failed_test[0]))
|
|
|
|
def start_fixture_servers(self):
|
|
root = self.server_root or os.path.join(os.path.dirname(here), "www")
|
|
if self.appName == "fennec":
|
|
return serve.start(root, host=moznetwork.get_ip())
|
|
else:
|
|
return serve.start(root)
|
|
|
|
def add_test(self, test, expected='pass'):
|
|
filepath = os.path.abspath(test)
|
|
|
|
if os.path.isdir(filepath):
|
|
for root, dirs, files in os.walk(filepath):
|
|
for filename in files:
|
|
if filename.endswith('.ini'):
|
|
msg_tmpl = ("Ignoring manifest '{0}'; running all tests in '{1}'."
|
|
" See --help for details.")
|
|
relpath = os.path.relpath(os.path.join(root, filename), filepath)
|
|
self.logger.warning(msg_tmpl.format(relpath, filepath))
|
|
elif self._is_filename_valid(filename):
|
|
test_file = os.path.join(root, filename)
|
|
self.add_test(test_file)
|
|
return
|
|
|
|
file_ext = os.path.splitext(os.path.split(filepath)[-1])[1]
|
|
|
|
if file_ext == '.ini':
|
|
manifest = TestManifest()
|
|
manifest.read(filepath)
|
|
|
|
json_path = update_mozinfo(filepath)
|
|
self.logger.info("mozinfo updated from: {}".format(json_path))
|
|
self.logger.info("mozinfo is: {}".format(mozinfo.info))
|
|
|
|
filters = []
|
|
if self.test_tags:
|
|
filters.append(tags(self.test_tags))
|
|
|
|
values = {
|
|
"appname": self.appName,
|
|
"e10s": self.e10s,
|
|
"manage_instance": self.marionette.instance is not None,
|
|
}
|
|
values.update(mozinfo.info)
|
|
|
|
manifest_tests = manifest.active_tests(exists=False,
|
|
disabled=True,
|
|
filters=filters,
|
|
**values)
|
|
if len(manifest_tests) == 0:
|
|
self.logger.error("No tests to run using specified "
|
|
"combination of filters: {}".format(
|
|
manifest.fmt_filters()))
|
|
|
|
target_tests = []
|
|
for test in manifest_tests:
|
|
if test.get('disabled'):
|
|
self.manifest_skipped_tests.append(test)
|
|
else:
|
|
target_tests.append(test)
|
|
|
|
for i in target_tests:
|
|
if not os.path.exists(i["path"]):
|
|
raise IOError("test file: {} does not exist".format(i["path"]))
|
|
|
|
file_ext = os.path.splitext(os.path.split(i['path'])[-1])[-1]
|
|
|
|
self.add_test(i["path"], i["expected"])
|
|
return
|
|
|
|
self.tests.append({'filepath': filepath, 'expected': expected})
|
|
|
|
def run_test(self, filepath, expected):
|
|
testloader = unittest.TestLoader()
|
|
suite = unittest.TestSuite()
|
|
self.test_kwargs['expected'] = expected
|
|
mod_name = os.path.splitext(os.path.split(filepath)[-1])[0]
|
|
for handler in self.test_handlers:
|
|
if handler.match(os.path.basename(filepath)):
|
|
handler.add_tests_to_suite(mod_name,
|
|
filepath,
|
|
suite,
|
|
testloader,
|
|
self.marionette,
|
|
self.fixtures,
|
|
self.testvars,
|
|
**self.test_kwargs)
|
|
break
|
|
|
|
if suite.countTestCases():
|
|
runner = self.textrunnerclass(logger=self.logger,
|
|
marionette=self.marionette,
|
|
capabilities=self.capabilities,
|
|
result_callbacks=self.result_callbacks)
|
|
|
|
results = runner.run(suite)
|
|
self.results.append(results)
|
|
|
|
self.failed += len(results.failures) + len(results.errors)
|
|
if hasattr(results, 'skipped'):
|
|
self.skipped += len(results.skipped)
|
|
self.todo += len(results.skipped)
|
|
self.passed += results.passed
|
|
for failure in results.failures + results.errors:
|
|
self.failures.append((results.getInfo(failure), failure.output,
|
|
'TEST-UNEXPECTED-FAIL'))
|
|
if hasattr(results, 'unexpectedSuccesses'):
|
|
self.failed += len(results.unexpectedSuccesses)
|
|
self.unexpected_successes += len(results.unexpectedSuccesses)
|
|
for failure in results.unexpectedSuccesses:
|
|
self.failures.append((results.getInfo(failure), failure.output,
|
|
'TEST-UNEXPECTED-PASS'))
|
|
if hasattr(results, 'expectedFailures'):
|
|
self.todo += len(results.expectedFailures)
|
|
|
|
self.mixin_run_tests = []
|
|
for result in self.results:
|
|
result.result_modifiers = []
|
|
|
|
def run_test_set(self, tests):
|
|
if self.shuffle:
|
|
random.seed(self.shuffle_seed)
|
|
random.shuffle(tests)
|
|
|
|
for test in tests:
|
|
self.run_test(test['filepath'], test['expected'])
|
|
if self.record_crash():
|
|
break
|
|
|
|
def run_test_sets(self):
|
|
if len(self.tests) < 1:
|
|
raise Exception('There are no tests to run.')
|
|
elif self.total_chunks > len(self.tests):
|
|
raise ValueError('Total number of chunks must be between 1 and {}.'
|
|
.format(len(self.tests)))
|
|
if self.total_chunks > 1:
|
|
chunks = [[] for i in range(self.total_chunks)]
|
|
for i, test in enumerate(self.tests):
|
|
target_chunk = i % self.total_chunks
|
|
chunks[target_chunk].append(test)
|
|
|
|
self.logger.info('Running chunk {0} of {1} ({2} tests selected from a '
|
|
'total of {3})'.format(self.this_chunk, self.total_chunks,
|
|
len(chunks[self.this_chunk - 1]),
|
|
len(self.tests)))
|
|
self.tests = chunks[self.this_chunk - 1]
|
|
|
|
self.run_test_set(self.tests)
|
|
|
|
def cleanup(self):
|
|
for proc in iter_proc(self.fixture_servers):
|
|
proc.stop()
|
|
proc.kill()
|
|
self.fixture_servers = {}
|
|
|
|
if hasattr(self, 'marionette') and self.marionette:
|
|
if self.marionette.instance is not None:
|
|
self.marionette.instance.close()
|
|
self.marionette.instance = None
|
|
|
|
self.marionette.cleanup()
|
|
self.marionette = None
|
|
|
|
__del__ = cleanup
|