wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-05 07:18:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/master' into custom

Browse source
This commit is contained in:
roytam1 2021-02-05 07:37:11 +08:00
commit f9d748b5f0
23946 changed files with 541 additions and 1480141 deletions
Download patch file Download diff file Expand all files Collapse all files

4
build/mach_bootstrap.py
View file

@ -90,9 +90,6 @@ SEARCH_PATHS = [
'testing/mozbase/manifestparser',
'testing/taskcluster',
'testing/tools/autotry',
'testing/web-platform',
'testing/web-platform/harness',
'testing/web-platform/tests/tools/wptserve',
'testing/xpcshell',
'xpcom/idl-parser',
]
@ -121,7 +118,6 @@ MACH_MODULES = [
'testing/mochitest/mach_commands.py',
'testing/mozharness/mach_commands.py',
'testing/talos/mach_commands.py',
'testing/web-platform/mach_commands.py',
'testing/xpcshell/mach_commands.py',
'tools/docs/mach_commands.py',
'tools/lint/mach_commands.py',

21
js/src/builtin/RegExp.js
View file

@ -1166,3 +1166,24 @@ function RegExpStringIteratorNext() {
result.value = match;
return result;
}
// String.prototype.matchAll proposal.
// ES2020 draft rev e97c95d064750fb949b6778584702dd658cf5624
// 7.2.8 IsRegExp ( argument )
function IsRegExp(argument) {
// Step 1.
if (!IsObject(argument)) {
return false;
}
// Step 2.
var matcher = argument[std_match];
// Step 3.
if (matcher !== undefined) {
return !!matcher;
}
// Steps 4-5.
return IsPossiblyWrappedRegExpObject(argument);
}

109
js/src/builtin/String.js
View file

@ -224,6 +224,115 @@ function String_generic_replace(thisValue, searchValue, replaceValue) {
return callFunction(String_replace, thisValue, searchValue, replaceValue);
}
function String_replaceAll(searchValue, replaceValue) {
// Step 1.
RequireObjectCoercible(this);
// Step 2.
if (searchValue !== undefined && searchValue !== null) {
// Steps 2.a-b.
if (IsRegExp(searchValue)) {
// Step 2.b.i.
var flags = searchValue.flags;
// Step 2.b.ii.
if (flags === undefined || flags === null) {
ThrowTypeError(JSMSG_FLAGS_UNDEFINED_OR_NULL);
}
// Step 2.b.iii.
if (!callFunction(std_String_includes, ToString(flags), "g")) {
ThrowTypeError(JSMSG_REQUIRES_GLOBAL_REGEXP, "replaceAll");
}
}
// Step 2.c.
var replacer = GetMethod(searchValue, std_replace);
// Step 2.b.
if (replacer !== undefined) {
return callContentFunction(replacer, searchValue, this, replaceValue);
}
}
// Step 3.
var string = ToString(this);
// Step 4.
var searchString = ToString(searchValue);
// Steps 5-6.
if (!IsCallable(replaceValue)) {
// Steps 7-16.
return StringReplaceAllString(string, searchString, ToString(replaceValue));
}
// Step 7.
var searchLength = searchString.length;
// Step 8.
var advanceBy = std_Math_max(1, searchLength);
// Step 9 (not needed in this implementation).
// Step 12.
var endOfLastMatch = 0;
// Step 13.
var result = "";
// Steps 10-11, 14.
var position = 0;
while (true) {
// Steps 10-11.
//
// StringIndexOf doesn't clamp the |position| argument to the input
// string length, i.e. |StringIndexOf("abc", "", 4)| returns -1,
// whereas |"abc".indexOf("", 4)| returns 3. That means we need to
// exit the loop when |nextPosition| is smaller than |position| and
// not just when |nextPosition| is -1.
var nextPosition = callFunction(std_String_indexOf, string, searchString, position);
if (nextPosition < position) {
break;
}
position = nextPosition;
// Step 14.a.
var replacement = ToString(callContentFunction(replaceValue, undefined, searchString,
position, string));
// Step 14.b (not applicable).
// Step 14.c.
var stringSlice = Substring(string, endOfLastMatch, position - endOfLastMatch);
// Step 14.d.
result += stringSlice + replacement;
// Step 14.e.
endOfLastMatch = position + searchLength;
// Step 11.b.
position += advanceBy;
}
// Step 15.
if (endOfLastMatch < string.length) {
// Step 15.a.
result += Substring(string, endOfLastMatch, string.length - endOfLastMatch);
}
// Step 16.
return result;
}
function String_generic_replace(thisValue, searchValue, replaceValue) {
if (thisValue === undefined)
ThrowTypeError(JSMSG_MISSING_FUN_ARG, 0, 'String.replace');
return callFunction(String_replace, thisValue, searchValue, replaceValue);
}
function StringProtoHasNoSearch() {
var ObjectProto = GetBuiltinPrototype("Object");
var StringProto = GetBuiltinPrototype("String");

1
js/src/jit/InlinableNatives.h
View file

@ -66,6 +66,7 @@
_(RegExpSearcher) \
_(RegExpTester) \
_(IsRegExpObject) \
_(IntrinsicIsPossiblyWrappedRegExpObject)\
_(RegExpPrototypeOptimizable) \
_(RegExpInstanceOptimizable) \
_(GetFirstDollarIndex) \

1
js/src/jit/IonBuilder.h
View file

@ -867,6 +867,7 @@ class IonBuilder
InliningStatus inlineRegExpSearcher(CallInfo& callInfo);
InliningStatus inlineRegExpTester(CallInfo& callInfo);
InliningStatus inlineIsRegExpObject(CallInfo& callInfo);
InliningStatus inlineIsPossiblyWrappedRegExpObject(CallInfo& callInfo);
InliningStatus inlineRegExpPrototypeOptimizable(CallInfo& callInfo);
InliningStatus inlineRegExpInstanceOptimizable(CallInfo& callInfo);
InliningStatus inlineGetFirstDollarIndex(CallInfo& callInfo);

40
js/src/jit/MCallOptimize.cpp
View file

@ -185,6 +185,8 @@ IonBuilder::inlineNativeCall(CallInfo& callInfo, JSFunction* target)
return inlineRegExpTester(callInfo);
case InlinableNative::IsRegExpObject:
return inlineIsRegExpObject(callInfo);
case InlinableNative::IntrinsicIsPossiblyWrappedRegExpObject:
return inlineIsPossiblyWrappedRegExpObject(callInfo);
case InlinableNative::RegExpPrototypeOptimizable:
return inlineRegExpPrototypeOptimizable(callInfo);
case InlinableNative::RegExpInstanceOptimizable:
@ -1902,6 +1904,44 @@ IonBuilder::inlineIsRegExpObject(CallInfo& callInfo)
callInfo.setImplicitlyUsedUnchecked();
return InliningStatus_Inlined;
}
IonBuilder::InliningStatus
IonBuilder::inlineIsPossiblyWrappedRegExpObject(CallInfo& callInfo)
{
MOZ_ASSERT(!callInfo.constructing());
MOZ_ASSERT(callInfo.argc() == 1);
if (callInfo.getArg(0)->type() != MIRType::Object)
return InliningStatus_NotInlined;
if (getInlineReturnType() != MIRType::Boolean)
return InliningStatus_NotInlined;
MDefinition* arg = callInfo.getArg(0);
if (arg->type() != MIRType::Object) {
return InliningStatus_NotInlined;
}
TemporaryTypeSet* types = arg->resultTypeSet();
if (!types) {
return InliningStatus_NotInlined;
}
// Don't inline if the argument might be a wrapper.
if (types->forAllClasses(constraints(), IsProxyClass) !=
TemporaryTypeSet::ForAllResult::ALL_FALSE) {
return InliningStatus_NotInlined;
}
if (const Class* clasp = types->getKnownClass(constraints())) {
pushConstant(BooleanValue(clasp == &RegExpObject::class_));
} else {
MHasClass* hasClass = MHasClass::New(alloc(), arg, &RegExpObject::class_);
current->add(hasClass);
current->push(hasClass);
}
callInfo.setImplicitlyUsedUnchecked();
return InliningStatus_Inlined;
}
IonBuilder::InliningStatus
IonBuilder::inlineRegExpPrototypeOptimizable(CallInfo& callInfo)

261
js/src/jsstr.cpp
View file

@ -2111,9 +2111,13 @@ AppendDollarReplacement(StringBuffer& newReplaceChars, size_t firstDollarIndex,
const CharT* repChars, size_t repLength)
{
MOZ_ASSERT(firstDollarIndex < repLength);
MOZ_ASSERT(matchStart <= matchLimit);
MOZ_ASSERT(matchLimit <= text->length());
/* Move the pre-dollar chunk in bulk. */
newReplaceChars.infallibleAppend(repChars, firstDollarIndex);
if (!newReplaceChars.append(repChars, firstDollarIndex)) {
return false;
}
/* Move the rest char-by-char, interpreting dollars as we encounter them. */
const CharT* repLimit = repChars + repLength;
@ -2329,6 +2333,200 @@ js::str_flat_replace_string(JSContext *cx, HandleString string, HandleString pat
return str;
}
// https://tc39.es/proposal-string-replaceall/#sec-string.prototype.replaceall
// Steps 7-16 when functionalReplace is false and searchString is not empty.
//
// The steps are quite different, for performance. Loops in steps 11 and 14
// are fused. GetSubstitution is optimized away when possible.
template <typename StrChar, typename RepChar>
static JSString* ReplaceAll(JSContext* cx, JSLinearString* str,
JSLinearString* pat,
JSLinearString* rep) {
// Step 7.
const size_t stringLength = str->length();
const size_t searchLength = pat->length();
const size_t replaceLength = rep->length();
MOZ_ASSERT(stringLength > 0);
MOZ_ASSERT(searchLength > 0);
MOZ_ASSERT(stringLength >= searchLength);
// Step 8 (advanceBy is equal to searchLength when searchLength > 0).
// Step 9 (not needed in this implementation).
// Step 10.
// Find the first match.
int32_t position = StringMatch(str, pat, 0);
// Nothing to replace, so return early.
if (position < 0) {
return str;
}
// Step 11 (moved below).
// Step 12.
uint32_t endOfLastMatch = 0;
// Step 13.
StringBuffer result(cx);
if (std::is_same<StrChar, char16_t>::value ||
std::is_same<RepChar, char16_t>::value) {
if (!result.ensureTwoByteChars()) {
return nullptr;
}
}
{
AutoCheckCannotGC nogc;
const StrChar* strChars = str->chars<StrChar>(nogc);
const RepChar* repChars = rep->chars<RepChar>(nogc);
uint32_t dollarIndex = FindDollarIndex(repChars, replaceLength);
// If it's true, we are sure that the result's length is, at least, the same
// length as |str->length()|.
if (replaceLength >= searchLength) {
if (!result.reserve(stringLength)) {
return nullptr;
}
}
do {
// Step 14.c.
// Append the substring before the current match.
if (!result.append(strChars + endOfLastMatch,
position - endOfLastMatch)) {
return nullptr;
}
// Steps 14.a-b and 14.d.
// Append the replacement.
if (dollarIndex != UINT32_MAX) {
size_t matchLimit = position + searchLength;
if (!AppendDollarReplacement(result, dollarIndex, position, matchLimit,
str, repChars, replaceLength)) {
return nullptr;
}
} else {
if (!result.append(repChars, replaceLength)) {
return nullptr;
}
}
// Step 14.e.
endOfLastMatch = position + searchLength;
// Step 11.
// Find the next match.
position = StringMatch(str, pat, endOfLastMatch);
} while (position >= 0);
// Step 15.
// Append the substring after the last match.
if (!result.append(strChars + endOfLastMatch,
stringLength - endOfLastMatch)) {
return nullptr;
}
}
// Step 16.
return result.finishString();
}
// https://tc39.es/proposal-string-replaceall/#sec-string.prototype.replaceall
// Steps 7-16 when functionalReplace is false and searchString is the empty
// string.
//
// The steps are quite different, for performance. Loops in steps 11 and 14
// are fused. GetSubstitution is optimized away when possible.
template <typename StrChar, typename RepChar>
static JSString* ReplaceAllInterleave(JSContext* cx, JSLinearString* str,
JSLinearString* rep) {
// Step 7.
const size_t stringLength = str->length();
const size_t replaceLength = rep->length();
// Step 8 (advanceBy is 1 when searchString is the empty string).
// Steps 9-12 (trivial when searchString is the empty string).
// Step 13.
StringBuffer result(cx);
if (std::is_same<StrChar, char16_t>::value ||
std::is_same<RepChar, char16_t>::value) {
if (!result.ensureTwoByteChars()) {
return nullptr;
}
}
{
AutoCheckCannotGC nogc;
const StrChar* strChars = str->chars<StrChar>(nogc);
const RepChar* repChars = rep->chars<RepChar>(nogc);
uint32_t dollarIndex = FindDollarIndex(repChars, replaceLength);
if (dollarIndex != UINT32_MAX) {
if (!result.reserve(stringLength)) {
return nullptr;
}
} else {
// Compute the exact result length when no substitutions take place.
CheckedInt<uint32_t> strLength(stringLength);
CheckedInt<uint32_t> repLength(replaceLength);
CheckedInt<uint32_t> length = strLength + (strLength + 1) * repLength;
if (!length.isValid()) {
ReportAllocationOverflow(cx);
return nullptr;
}
if (!result.reserve(length.value())) {
return nullptr;
}
}
auto appendReplacement = [&](size_t match) {
if (dollarIndex != UINT32_MAX) {
return AppendDollarReplacement(result, dollarIndex, match, match,
str, repChars, replaceLength);
}
return result.append(repChars, replaceLength);
};
for (size_t index = 0; index < stringLength; index++) {
// Steps 11, 14.a-b and 14.d.
// The empty string matches before each character.
if (!appendReplacement(index)) {
return nullptr;
}
// Step 14.c.
if (!result.append(strChars[index])) {
return nullptr;
}
}
// Steps 11, 14.a-b and 14.d.
// The empty string also matches at the end of the string.
if (!appendReplacement(stringLength)) {
return nullptr;
}
// Step 15 (not applicable when searchString is the empty string).
}
// Step 16.
return result.finishString();
}
// String.prototype.replaceAll (Stage 3 proposal)
// https://tc39.es/proposal-string-replaceall/
//
// String.prototype.replaceAll ( searchValue, replaceValue )
//
// Steps 7-16 when functionalReplace is false.o
JSString*
js::str_replace_string_raw(JSContext* cx, HandleString string, HandleString pattern,
HandleString replacement)
@ -2371,6 +2569,62 @@ js::str_replace_string_raw(JSContext* cx, HandleString string, HandleString patt
return BuildFlatReplacement(cx, string, repl, match, patternLength);
}
JSString*
js::str_replaceAll_string_raw(JSContext* cx, HandleString string, HandleString pattern,
HandleString replacement)
{
const size_t stringLength = string->length();
const size_t searchLength = pattern->length();
// Directly return when we're guaranteed to find no match.
if (searchLength > stringLength) {
return string;
}
RootedLinearString str(cx, string->ensureLinear(cx));
if (!str) {
return nullptr;
}
RootedLinearString repl(cx, replacement->ensureLinear(cx));
if (!repl) {
return nullptr;
}
RootedLinearString search(cx, pattern->ensureLinear(cx));
if (!search) {
return nullptr;
}
// The pattern is empty, so we interleave the replacement string in-between
// each character.
if (searchLength == 0) {
if (str->hasTwoByteChars()) {
if (repl->hasTwoByteChars()) {
return ReplaceAllInterleave<char16_t, char16_t>(cx, str, repl);
}
return ReplaceAllInterleave<char16_t, Latin1Char>(cx, str, repl);
}
if (repl->hasTwoByteChars()) {
return ReplaceAllInterleave<Latin1Char, char16_t>(cx, str, repl);
}
return ReplaceAllInterleave<Latin1Char, Latin1Char>(cx, str, repl);
}
MOZ_ASSERT(stringLength > 0);
if (str->hasTwoByteChars()) {
if (repl->hasTwoByteChars()) {
return ReplaceAll<char16_t, char16_t>(cx, str, search, repl);
}
return ReplaceAll<char16_t, Latin1Char>(cx, str, search, repl);
}
if (repl->hasTwoByteChars()) {
return ReplaceAll<Latin1Char, char16_t>(cx, str, search, repl);
}
return ReplaceAll<Latin1Char, Latin1Char>(cx, str, search, repl);
}
// ES 2016 draft Mar 25, 2016 21.1.3.17 steps 4, 8, 12-18.
static JSObject*
SplitHelper(JSContext* cx, HandleLinearString str, uint32_t limit, HandleLinearString sep,
@ -2572,9 +2826,9 @@ static const JSFunctionSpec string_methods[] = {
JS_FN("endsWith", str_endsWith, 1,0),
JS_FN("trim", str_trim, 0,0),
JS_FN("trimLeft", str_trimStart, 0,0),
JS_FN("trimStart", str_trimStart, 0,0),
JS_FN("trimStart", str_trimStart, 0,0),
JS_FN("trimRight", str_trimEnd, 0,0),
JS_FN("trimEnd", str_trimEnd, 0,0),
JS_FN("trimEnd", str_trimEnd, 0,0),
JS_FN("toLocaleLowerCase", str_toLocaleLowerCase, 0,0),
JS_FN("toLocaleUpperCase", str_toLocaleUpperCase, 0,0),
JS_SELF_HOSTED_FN("localeCompare", "String_localeCompare", 1,0),
@ -2586,6 +2840,7 @@ static const JSFunctionSpec string_methods[] = {
JS_SELF_HOSTED_FN("matchAll", "String_matchAll", 1,0),
JS_SELF_HOSTED_FN("search", "String_search", 1,0),
JS_SELF_HOSTED_FN("replace", "String_replace", 2,0),
JS_SELF_HOSTED_FN("replaceAll", "String_replaceAll", 2,0),
JS_SELF_HOSTED_FN("split", "String_split", 2,0),
JS_SELF_HOSTED_FN("substr", "String_substr", 2,0),

4
js/src/jsstr.h
View file

@ -476,6 +476,10 @@ JSString*
str_replace_string_raw(JSContext* cx, HandleString string, HandleString pattern,
HandleString replacement);
JSString*
str_replaceAll_string_raw(JSContext* cx, HandleString string, HandleString pattern,
HandleString replacement);
extern bool
StringConstructor(JSContext* cx, unsigned argc, Value* vp);

63
js/src/tests/ecma_2021/manual/test-replaceall.html Normal file
View file

@ -0,0 +1,63 @@
<html><head>
<script>
function passfail(test) {
if (test) {
document.writeln('<span style="color:#008000;">PASS</span>');
} else {
document.writeln('<span style="color:#BF0000;">FAIL</span>');
}
}
</script>
</head>
<body>
Replace "a" with "b":
<script>
var input='1a234abcd';
var match='a';
var expected='1b234bbcd';
passfail(input.replaceAll(match,'b') === expected);
</script><br>
Replace "a" with "aa":
<script>
var input='aabbccdd';
var match='a';
var expected='aaaabbccdd';
passfail(input.replaceAll(match,'aa') === expected);
</script><br>
Replace "" with "b":
<script>
var input='aaaa';
var match='';
var expected='babababab';
passfail(input.replaceAll(match,'b') === expected);
</script><br>
Replace "old" with special pattern "new (was: $&)":
<script>
var input='This is the old thing of the old time.';
var match='old';
var expected='This is the new (was: old) thing of the new (was: old) time.';
passfail(input.replaceAll(match,'new (was: $&)') === expected);
</script><br>
Replace "/[0-9]/g" with "b":
<script>
var input='1a234abcd';
var match=/[0-9]/g;
var expected='babbbabcd';
passfail(input.replaceAll(match,'b') === expected);
</script><br>
Replace "/[0-9]/" with "b" (Throws):
<script>
var input='1a234abcd';
var match=/[0-9]/;
var expected='1a234abcd';
try {
test=input.replaceAll(match,'b');
passfail(false);
} catch(e) {
passfail(true);
document.writeln('('+e+')');
}
</script><br>
</body>
</html>

66
js/src/vm/SelfHosting.cpp
View file

@ -189,6 +189,28 @@ intrinsic_IsInstanceOfBuiltin(JSContext* cx, unsigned argc, Value* vp)
return true;
}
template<typename T>
static bool
intrinsic_IsPossiblyWrappedBuiltin(JSContext* cx, unsigned argc, Value* vp)
{
CallArgs args = CallArgsFromVp(argc, vp);
MOZ_ASSERT(args.length() == 1);
bool isTypeT = false;
if (args[0].isObject()) {
JSObject* obj = CheckedUnwrap(&args[0].toObject());
if (!obj) {
JS_ReportErrorASCII(cx, "Permission denied to access object");
return false;
}
isTypeT = obj->is<T>();
}
args.rval().setBoolean(isTypeT);
return true;
}
template<typename T>
static bool
intrinsic_GuardToBuiltin(JSContext* cx, unsigned argc, Value* vp)
@ -1172,27 +1194,6 @@ intrinsic_IsFloat32TypedArray(JSContext* cx, unsigned argc, Value* vp)
return intrinsic_IsSpecificTypedArray(cx, argc, vp, Scalar::Float32);
}
static bool
intrinsic_IsPossiblyWrappedTypedArray(JSContext* cx, unsigned argc, Value* vp)
{
CallArgs args = CallArgsFromVp(argc, vp);
MOZ_ASSERT(args.length() == 1);
bool isTypedArray = false;
if (args[0].isObject()) {
JSObject* obj = CheckedUnwrap(&args[0].toObject());
if (!obj) {
JS_ReportErrorASCII(cx, "Permission denied to access object");
return false;
}
isTypedArray = obj->is<TypedArrayObject>();
}
args.rval().setBoolean(isTypedArray);
return true;
}
static bool
intrinsic_TypedArrayBuffer(JSContext* cx, unsigned argc, Value* vp)
{
@ -1756,6 +1757,24 @@ intrinsic_StringReplaceString(JSContext* cx, unsigned argc, Value* vp)
return true;
}
static bool
intrinsic_StringReplaceAllString(JSContext * cx, unsigned argc, Value* vp)
{
CallArgs args = CallArgsFromVp(argc, vp);
MOZ_ASSERT(args.length() == 3);
RootedString string(cx, args[0].toString());
RootedString pattern(cx, args[1].toString());
RootedString replacement(cx, args[2].toString());
JSString* result = str_replaceAll_string_raw(cx, string, pattern, replacement);
if (!result) {
return false;
}
args.rval().setString(result);
return true;
}
bool
js::intrinsic_StringSplitString(JSContext* cx, unsigned argc, Value* vp)
{
@ -2401,7 +2420,7 @@ static const JSFunctionSpec intrinsic_functions[] = {
JS_INLINABLE_FN("IsTypedArray",
intrinsic_IsInstanceOfBuiltin<TypedArrayObject>, 1,0,
IntrinsicIsTypedArray),
JS_INLINABLE_FN("IsPossiblyWrappedTypedArray",intrinsic_IsPossiblyWrappedTypedArray,1,0,
JS_INLINABLE_FN("IsPossiblyWrappedTypedArray",intrinsic_IsPossiblyWrappedBuiltin<TypedArrayObject>,1,0,
IntrinsicIsPossiblyWrappedTypedArray),
JS_FN("TypedArrayBuffer", intrinsic_TypedArrayBuffer, 1,0),
@ -2513,6 +2532,8 @@ static const JSFunctionSpec intrinsic_functions[] = {
JS_INLINABLE_FN("IsRegExpObject",
intrinsic_IsInstanceOfBuiltin<RegExpObject>, 1,0,
IsRegExpObject),
JS_INLINABLE_FN("IsPossiblyWrappedRegExpObject", intrinsic_IsPossiblyWrappedBuiltin<RegExpObject>,1,0,
IntrinsicIsPossiblyWrappedRegExpObject),
JS_FN("CallRegExpMethodIfWrapped",
CallNonGenericSelfhostedMethod<Is<RegExpObject>>, 2,0),
JS_INLINABLE_FN("RegExpMatcher", RegExpMatcher, 4,0,
@ -2536,6 +2557,7 @@ static const JSFunctionSpec intrinsic_functions[] = {
JS_FN("FlatStringSearch", FlatStringSearch, 2,0),
JS_INLINABLE_FN("StringReplaceString", intrinsic_StringReplaceString, 3, 0,
IntrinsicStringReplaceString),
JS_FN("StringReplaceAllString", intrinsic_StringReplaceAllString, 3, 0),
JS_INLINABLE_FN("StringSplitString", intrinsic_StringSplitString, 2, 0,
IntrinsicStringSplitString),
JS_FN("StringSplitStringLimit", intrinsic_StringSplitStringLimit, 3, 0),

266
testing/web-platform/README.md
View file

@ -1,266 +0,0 @@
web-platform-tests
==================
This directory contains the W3C
[web-platform-tests](http://github.com/w3c/web-platform-tests). They
can be run using `mach`:
mach web-platform-tests
To limit the testrun to certain directories use the `--include` option;
for example:
mach web-platform-tests --include=dom
The testsuite contains a mix of javascript tests and reftests. To
limit the type of tests that get run, use `--test-type=testharness` for
javascript tests or `--test-type=reftest` for reftests.
FAQ
---
* I fixed a bug and some tests have started to pass. How do I fix the
UNEXPECTED-PASS messages when web-platform-tests is run?
You need to update the expectation data for those tests. See the
section on expectations below.
* I want to write some new tests for the web-platform-tests
testsuite. How do I do that?
See the section on tests below. You can commit the tests directly to
the Mozilla repository under `testing/web-platform/tests` and they
will be upstreamed next time the test is imported. For this reason
please ensure that any tests you write are testing correct-per-spec
behaviour even if we don't yet pass, get proper review, and have a
commit message that makes sense outside of the Mozilla
context. If you are writing tests that should not be upstreamed yet
for some reason they must be located under
`testing/web-platform/mozilla/tests`.
It is important to note that in order for the tests to run the
manifest file must be updated; this should not be done by hand, but
by running `mach wpt-manifest-update` (or `mach web-platform-tests
--manifest-update`, if you also wish to run some tests).
`mach web-platform-tests-create <path>` is a helper script designed
to help create new web-platform-tests. It opens a locally configured
editor at `<path>` with web-platform-tests boilerplate filled in,
and in the background runs `mach web-platform-tests
--manifest-update <path>`, so the test being developed is added to
the manifest and opened for interactive development.
* How do I write a test that requires the use of a Mozilla-specific
feature?
Tests in the `mozilla/tests/` directory use the same harness but are
not synced with any upstream. Be aware that these tests run on the
server with a `/_mozilla/` prefix to their URLs.
* A test is unstable; how do I disable it?
See the section on disabling tests.
Directories
-----------
`tests/` contains the tests themselves. This is a copy of a certain
revision of web-platform-tests. Any patches modifying this directory
will be upstreamed next time the tests are imported.
`harness/` contains the [wptrunner](http://github.com/w3c/wptrunner)
test runner. Again the contents of this directory will be overwritten
on update.
`meta/` contains Gecko-specific expectation data. This is explained in
the following section.
`mozilla/tests` contains tests that will not be upstreamed and may
make use of Mozilla-specific features.
`mozilla/meta` contains metadata for the Mozilla-specific tests.
Expectation Data
----------------
With the tests coming from upstream, it is not guaranteed that they
all pass in Gecko-based browsers. For this reason it is necessary to
provide metadata about the expected results of each test. This is
provided in a set of manifest files in the `meta/` subdirectories.
There is one manifest file per test with "non-default"
expectations. By default tests are expected to PASS, and tests with
subtests are expected to have an overall status of OK. The manifest
file of a test has the same path as the test file but under the `meta`
directory rather than the `tests` directory and has the suffix `.ini`.
The format of these files is similar to `ini` files, but with a couple
of important differences; sections can be nested using indentation,
and only `:` is permitted as a key-value separator. For example the
expectation file for a test with one failing subtest and one erroring
subtest might look like:
[filename.html]
type: testharness
[Subtest name for failing test]
expected: FAIL
[Subtest name for erroring test]
expected: ERROR
Expectations can also be made platform-specific using a simple
python-like conditional syntax e.g. for a test that times out on linux
but otherwise fails:
[filename.html]
type: reftest
expected:
if os == "linux": TIMEOUT
FAIL
The available variables for the conditions are those provided by
[mozinfo](http://mozbase.readthedocs.org/en/latest/mozinfo.html).
For more information on manifest files, see the
[wptrunner documentation](http://wptrunner.readthedocs.org/en/latest/expectation.html).
Autogenerating Expectation Data
-------------------------------
After changing some code it may be necessary to update the expectation
data for the relevant tests. This can of course be done manually, but
tools are available to automate much of the process.
First one must run the tests that have changed status, and save the
raw log output to a file:
mach web-platform-tests --include=url/of/test.html --log-raw=new_results.log
Then the `web-platform-tests-update` command may be run using this log
data to update the expectation files:
mach web-platform-tests-update --no-check-clean new_results.log
By default this only updates the results data for the current
platform. To forcibly overwrite all existing result data, use the
`--ignore-existing` option to the update command.
Disabling Tests
---------------
Tests are disabled using the same manifest files used to set
expectation values. For example, if a test is unstable on Windows, it
can be disabled using an ini file with the contents:
[filename.html]
type: testharness
disabled:
if os == "win": https://bugzilla.mozilla.org/show_bug.cgi?id=1234567
Enabling Prefs
--------------
Some tests require specific prefs to be enabled before running. These
prefs can be set in the expectation data using a `prefs` key with a
comma-seperate list of `pref.name:value` items to set e.g.
[filename.html]
prefs: [dom.serviceWorkers.enabled:true,
dom.serviceWorkers.exemptFromPerDomainMax:true,
dom.caches.enabled:true]
Setting per-Directory Metadata
------------------------------
Occasionally it is useful to set metadata for an entire directory of
tests e.g. to disable then all, or to enable prefs for every test. In
that case it is possible to create a `__dir__.ini` file in the
metadata directory corresponding to the tests for which you want to
set this metadata e.g. to disable all the tests in
`tests/feature/unsupported/`, one might create
`meta/feature/unsupported/__dir__.ini` with the contents:
disabled: Feature is unsupported
Settings set in this way are inherited into subdirectories. It is
possible to unset a value that has been set in a parent using the
special token `@Reset` (usually used with prefs), or to force a value
to true or false using `@True` and `@False`. For example to enable
the tests in `meta/feature/unsupported/subfeature-supported` one might
create an ini file
`meta/feature/unsupported/subfeature-supported/__dir__.ini` like:
disabled: @False
Test Format
-----------
Javascript tests are written using
[testharness.js](http://github.com/w3c/testharness.js/). Reftests are
similar to standard Gecko reftests without an explicit manifest file,
but with in-test or filename conventions for identifying the
reference.
Full documentation on test authoring and submission can be found on
[testthewebforward.org](http://testthewebforward.org/docs).
Test Manifest
-------------
web-platform-tests use a large auto-generated JSON file as their
manifest. This stores data about the type of tests, their references,
if any, and their timeout, gathered by inspecting the filenames and
the contents of the test files.
In order to update the manifest it is recommended that you run `mach
web-platform-tests --manifest-update`. This rescans the test directory
looking for new, removed, or altered tests.
Running Tests In Other Browsers
-------------------------------
web-platform-tests is cross browser, and the runner is compatible with
multiple browsers. Therefore it's possible to check the behaviour of
tests in other browsers. This is somewhat more involved than running
them in Firefox since extra dependencies may be required. For example
to test in Chrome:
1. Download the chromedriver binary and place it somewhere sensible
e.g. `~/bin`
2. In your gecko source tree activate the virtualenv created by mach,
since this has most dependencies already installed. This is typically
in objdir/_virtualenv and is activated via e.g.
source objdir/_virtualenv/bin/activate
3. Install the extra requirements:
cd testing/web-platform/harness
pip install -r requirements_chrome.txt
4. Edit the config file `testing/web-platform/wptrunner.ini` so that
Chrome support is enabled by changing the section that reads:
[products]
firefox =
to read
[products]
firefox =
chrome =
(alternatively create a new config file elsewhere and use the
`--config` option to `runtests.py` to point wptrunner at this config
file).
5. Run `runtests.py` using the location of chromedriver as
the binary:
cd testing/web-platform
python runtests.py --product=chrome --binary=~/bin/chromedriver --log-mach=-
By default this will use the same test checkout and metadata as are in
the Gecko tree, so it's easy to compare behaviour relative to Firefox.

83
testing/web-platform/certs/cacert.pem
View file

@ -1,83 +0,0 @@
Certificate:
Data:
Version: 3 (0x2)
Serial Number: 1 (0x1)
Signature Algorithm: sha256WithRSAEncryption
Issuer: CN=web-platform-tests
Validity
Not Before: Dec 22 12:09:15 2014 GMT
Not After : Dec 21 12:09:15 2024 GMT
Subject: CN=web-platform-tests
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (2048 bit)
Modulus:
00:c0:3e:c3:bd:b7:9e:2e:3e:ec:1f:a5:ae:9f:85:
8d:c0:59:a4:40:a8:fe:66:cf:ef:7d:1f:4a:91:52:
e6:55:0f:e5:69:b8:44:da:62:17:c7:88:28:ad:ec:
6d:b0:00:cd:a1:69:0e:e8:19:84:58:ad:e9:ce:31:
50:6c:a4:82:14:91:08:5a:a3:ae:c8:49:13:19:18:
7b:5b:2b:44:30:eb:bf:c7:7c:bb:d4:32:17:6a:4d:
eb:84:f1:65:9b:9d:21:3e:91:ae:74:75:b3:7c:b4:
cd:fa:98:73:d4:7c:d1:5d:1b:89:72:f7:d4:52:bd:
05:a3:c1:cf:b6:58:e7:51:ec:c2:71:c7:f6:0b:00:
97:58:f9:cb:c3:18:46:7b:55:0f:90:bf:da:a3:7d:
d6:c5:48:ea:b3:b5:a8:12:96:ac:38:65:10:b9:b1:
69:cb:4e:3b:4c:c3:83:74:33:63:b4:cc:fe:65:c1:
ad:12:51:f1:02:72:50:49:03:ab:a6:28:20:41:15:
ca:77:15:d9:85:55:69:9d:31:c1:db:12:be:46:db:
e6:d3:8f:2f:66:2d:9b:61:08:30:57:6c:d9:4f:b1:
6a:a8:e5:90:e3:e2:68:96:45:6e:3f:de:5e:13:fe:
8a:bd:f1:3f:7f:26:ec:3c:1a:80:b0:a8:ec:52:c5:
11:03
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints:
CA:TRUE
X509v3 Subject Key Identifier:
6A:AB:53:64:92:36:87:23:34:B3:1D:6F:85:4B:F5:DF:5A:5C:74:8F
X509v3 Authority Key Identifier:
keyid:6A:AB:53:64:92:36:87:23:34:B3:1D:6F:85:4B:F5:DF:5A:5C:74:8F
DirName:/CN=web-platform-tests
serial:01
X509v3 Key Usage:
Certificate Sign
X509v3 Extended Key Usage:
TLS Web Server Authentication
Signature Algorithm: sha256WithRSAEncryption
46:af:02:04:97:2f:5b:00:11:c5:8f:c4:e1:2b:23:62:d1:61:
4a:28:ed:82:82:57:d7:28:65:88:a5:61:16:20:37:02:90:16:
0b:05:43:46:db:bd:3d:b4:4d:1c:6e:85:ff:5d:dc:0f:a4:a4:
29:98:24:ae:39:ab:e4:97:a9:10:bc:a5:b9:4b:c1:2e:5a:ce:
89:32:00:f1:fc:94:35:a6:71:c8:9c:d9:05:43:44:6c:68:62:
ae:b1:71:20:17:5f:c4:fb:ae:05:e6:26:e5:41:88:cc:db:51:
55:ed:85:a0:c9:e5:68:65:a7:fa:7a:db:8f:81:61:60:50:0b:
16:b0:10:49:19:bb:70:0e:37:09:03:20:e8:a2:b9:e5:eb:c2:
6a:7b:4f:60:cd:fb:22:0b:27:c6:0d:2d:e2:32:cc:28:de:c6:
e2:14:6a:ad:3f:c4:6e:78:9d:71:24:9b:56:c4:54:28:e9:ec:
09:6e:34:cc:6d:5c:bc:e6:68:96:44:ff:62:d0:54:a0:a4:37:
d8:f7:9f:bc:bb:dc:ad:2c:49:e2:64:b9:4d:aa:e4:22:e6:df:
3a:17:23:13:c1:aa:0e:94:27:46:5d:11:b9:0b:dc:3d:cf:93:
20:eb:18:56:c5:ac:e3:92:eb:55:d3:cb:ce:e4:9c:21:85:d6:
21:93:92:4f
-----BEGIN CERTIFICATE-----
MIIDTzCCAjegAwIBAgIBATANBgkqhkiG9w0BAQsFADAdMRswGQYDVQQDDBJ3ZWIt
cGxhdGZvcm0tdGVzdHMwHhcNMTQxMjIyMTIwOTE1WhcNMjQxMjIxMTIwOTE1WjAd
MRswGQYDVQQDDBJ3ZWItcGxhdGZvcm0tdGVzdHMwggEiMA0GCSqGSIb3DQEBAQUA
A4IBDwAwggEKAoIBAQDAPsO9t54uPuwfpa6fhY3AWaRAqP5mz+99H0qRUuZVD+Vp
uETaYhfHiCit7G2wAM2haQ7oGYRYrenOMVBspIIUkQhao67ISRMZGHtbK0Qw67/H
fLvUMhdqTeuE8WWbnSE+ka50dbN8tM36mHPUfNFdG4ly99RSvQWjwc+2WOdR7MJx
x/YLAJdY+cvDGEZ7VQ+Qv9qjfdbFSOqztagSlqw4ZRC5sWnLTjtMw4N0M2O0zP5l
wa0SUfECclBJA6umKCBBFcp3FdmFVWmdMcHbEr5G2+bTjy9mLZthCDBXbNlPsWqo
5ZDj4miWRW4/3l4T/oq98T9/Juw8GoCwqOxSxREDAgMBAAGjgZkwgZYwDAYDVR0T
BAUwAwEB/zAdBgNVHQ4EFgQUaqtTZJI2hyM0sx1vhUv131pcdI8wRQYDVR0jBD4w
PIAUaqtTZJI2hyM0sx1vhUv131pcdI+hIaQfMB0xGzAZBgNVBAMMEndlYi1wbGF0
Zm9ybS10ZXN0c4IBATALBgNVHQ8EBAMCAgQwEwYDVR0lBAwwCgYIKwYBBQUHAwEw
DQYJKoZIhvcNAQELBQADggEBAEavAgSXL1sAEcWPxOErI2LRYUoo7YKCV9coZYil
YRYgNwKQFgsFQ0bbvT20TRxuhf9d3A+kpCmYJK45q+SXqRC8pblLwS5azokyAPH8
lDWmccic2QVDRGxoYq6xcSAXX8T7rgXmJuVBiMzbUVXthaDJ5Whlp/p624+BYWBQ
CxawEEkZu3AONwkDIOiiueXrwmp7T2DN+yILJ8YNLeIyzCjexuIUaq0/xG54nXEk
m1bEVCjp7AluNMxtXLzmaJZE/2LQVKCkN9j3n7y73K0sSeJkuU2q5CLm3zoXIxPB
qg6UJ0ZdEbkL3D3PkyDrGFbFrOOS61XTy87knCGF1iGTkk8=
-----END CERTIFICATE-----

30
testing/web-platform/certs/cakey.pem
View file

@ -1,30 +0,0 @@
-----BEGIN ENCRYPTED PRIVATE KEY-----
MIIFDjBABgkqhkiG9w0BBQ0wMzAbBgkqhkiG9w0BBQwwDgQIb9ES7h6YGBcCAggA
MBQGCCqGSIb3DQMHBAi2ZVLgq1XvagSCBMjAmGEVvjwXvF00veuQmJsmcjVHB/qP
VqSXjCQ94orZb89UfFnPO9zXdvLWxrwb5WP6bbQv+Sh4htExXCD5XZi1AzWNCybe
da0vvQGgjdzUh2fCrG4K7J0w20lrYgw3HVSj/WtmdbdZFdoX+BgXrxcqkE3M5opZ
3UD3yIQeXSxUkh3iv6zzZaWujxjDI2JpwxRmMVbrr8OeBrKJsqB2DnKmq+emmvEF
iXTN3Ww/Aj6GIqfPZ8jpVdwcVN5QpeHAh7b2lszt7GEOGcBhutPq4Aqy8PIiDR80
sUYI7V8OXm+Y45DnfkvsogZEifOiUrQ2U+aGDu+Zt88661wVzjq+voJlz8EaIPCE
B/NS2SgNqI2/DrjEEecn6hjgHWIUBwOfeNoSi1Tri6KZFyxG26LE/V8Cd50yodx9
pBgFxdCbmYLeRcVeXW2bu0ZMjPddRlR5MHfrkM5ZAze7nRxoiyWnB/U8pPf+bQvx
K4P9KcwCOeHigkaCYZKq7nmZyEy4km89zIugT/YWhMWyVwylTpagaiiJwYLjug8n
CbFZWAkORBIl2g/YCuTBUJtC2IWX8kw+nYVwqBszpZyC6nbha2UmhQDfMAowQA5v
n1LnV8I6f7u6HidbB8WX2UZoh03A4beCBz+dq2VaUquLTL4KQTIz+6rw7nEysrnH
TIb8SlwsYAlzzwyyM9dSWt7iQeNjmH7zL0MozMs3LKHIrsWi7ZZh8BUYnT2vKdNV
2ZLOMcR0tYVmVZ8uYkR9kny/fbZcKN54xScohA2UX261W+sWiEgN+RaBsQ79pFgi
vYldfjaGNSvftXa590xn2tlS6/suB5MxiW5g3PuBg5XtVZ95l0f1n376Xh41sJv8
YHrCtFHOlSpDJULGiXVh/wXBmS7qJ8DhnUUG699EdlsFf6Qg22WB3AZRvEJdYC4z
P8W+jZ15NTDbHg3Hv7/CFYVzbXv2w0jkiqQgDF/wc6t/EdLD+2hzcN+nJGjtxZbn
xjbXcg98CUMU+dc/aD4N45K9e9rPg3+iZLwvsRvwx+MszmgxxPv05pNyRO7RVk8r
gkyyp9/CJFme+4nFKUc0dUy2yNXZtklTX0XKm/YNKin6uUMlIArIa54Cfvt9QslV
iD+SxU1ZHmzwKT82+5ZeIRLNWvFV/9E4nD+BTagK2Fdwnsu1S2k7ItD9lK/cBPGS
0tz1HWv4Auj3wMPZklp3SQluOl6eAIVqqI9GaX/d42DctBQWLTa27YibWyNIcw7o
3N8GDREMawTBdDRwlZ3oT+yiGLX1c8ds2o0/4IcJlOkDoxXErmdlZo9oVe6z4R7g
62yR53atVTLoUnAjxHXx0bJiyayv9Y3wjOEvuhuqdd9F+HOhTtAHr/BJQNhEk+z8
531CZTJjb1p11PbOtHGV2IeB0S82mxkkXRykEXOb89ZpDHNRiMinThRkoCmuRI9r
dTiES9B02yMPxJ3sLQyDxCoS5mwfcAqKTeK+yCvTvBy+t5tw63DbWlMp/7Ahy65K
rWMHdwqwfoB+ZYw5sYZdPvuBVAT01I2JbOqX36RacQultFns2OinxOJHa1HjtXyS
cPVEkMa7ci3Ym9j5RQNLVsgJe7YK9HixX5HjQFAowAH2pXZ5pKJIJYxPIUKtZlsz
qbM=
-----END ENCRYPTED PRIVATE KEY-----

28
testing/web-platform/certs/web-platform.test.key
View file

@ -1,28 +0,0 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCzhNaLAVkYhdHc
Mt8495CFGz6lXoE+L/w6X3937yO7OognD74lRs1jfcuV2KVQENKi0reX0Q1s+/kF
6G+oS72VZ557lFipbZP94BLFzbSKZFIxXw7jiYRx2pjdS+wCJaV9Nf5j2rOs7KVG
Dw1kI1xt8+zMKGMjwEua7I/B7rGiPnJNcLUJweu0EFU8i+oblH5LdOb0n0+mRTC1
8Li00VlQZQqGU+pMn570WGwx9Rc6b1eLy1/wKAtFko0wIEn/UuYsyxia1+buPk80
NRUTxQLaxV+++1vOjb+1NXY8fOacOyaHTY2A5hbGJ/JQSbZydENJSUQ4u3hDI+4W
Ptli5qXXAgMBAAECggEBAIcwDQSnIjo2ZECHytQykpG6X6XXEksLhc1Lp0lhPC49
uNR5pX6a4AcBb3PLr0opMQZO2tUoKA0ff3t0e8loKD+/xXhY0Z/dlioEOP7elwv0
2nS1mhe9spCuxpk4GGXRhdtR8t2tj8s0do3YvgPgITXoEDX6YBZHNGhZpzSrFPgQ
/c3eGCVmzWYuLFfdj5OPQ9bwTaY4JSvDLZT0/WTgiica7VySwfz3HP1fFqNykTiK
ACQREvtxfk5Ym2nT6oni7CM2zOEJL9SXicXI5HO4bERH0ZYh//F3g6mwGiFXUJPd
NKgaTM1oT9kRGkUaEYsRWrddwR8d5mXLvBuTJbgIsSECgYEA1+2uJSYRW1OqbhYP
ms59YQHSs3VjpJpnCV2zNa2Wixs57KS2cOH7B6KrQCogJFLtgCDVLtyoErfVkD7E
FivTgYr1pVCRppJddQzXik31uOINOBVffr7/09g3GcRN+ubHPZPq3K+dD6gHa3Aj
0nH1EjEEV0QpSTQFn87OF2mc9wcCgYEA1NVqMbbzd+9Xft5FXuSbX6E+S02dOGat
SgpnkTM80rjqa6eHdQzqk3JqyteHPgdi1vdYRlSPOj/X+6tySY0Ej9sRnYOfddA2
kpiDiVkmiqVolyJPY69Utj+E3TzJ1vhCQuYknJmB7zP9tDcTxMeq0l/NaWvGshEK
yC4UTQog1rECgYASOFILfGzWgfbNlzr12xqlRtwanHst9oFfPvLSQrWDQ2bd2wAy
Aj+GY2mD3oobxouX1i1m6OOdwLlalJFDNauBMNKNgoDnx03vhIfjebSURy7KXrNS
JJe9rm7n07KoyzRgs8yLlp3wJkOKA0pihY8iW9R78JpzPNqEo5SsURMXnQKBgBlV
gfuC9H4tPjP6zzUZbyk1701VYsaI6k2q6WMOP0ox+q1v1p7nN7DvaKjWeOG4TVqb
PKW6gQYE/XeWk9cPcyCQigs+1KdYbnaKsvWRaBYO1GFREzQhdarv6qfPCZOOH40J
Cgid+Sp4/NULzU2aGspJ3xCSZKdjge4MFhyJfRkxAoGBAJlwqY4nue0MBLGNpqcs
WwDtSasHvegKAcxGBKL5oWPbLBk7hk+hdqc8f6YqCkCNqv/ooBspL15ESItL+6yT
zt0YkK4oH9tmLDb+rvqZ7ZdXbWSwKITMoCyyHUtT6OKt/RtA0Vdy9LPnP27oSO/C
dk8Qf7KgKZLWo0ZNkvw38tEC
-----END PRIVATE KEY-----

86
testing/web-platform/certs/web-platform.test.pem
View file

@ -1,86 +0,0 @@
Certificate:
Data:
Version: 3 (0x2)
Serial Number: 2 (0x2)
Signature Algorithm: sha256WithRSAEncryption
Issuer: CN=web-platform-tests
Validity
Not Before: Dec 22 12:09:16 2014 GMT
Not After : Dec 21 12:09:16 2024 GMT
Subject: CN=web-platform.test
Subject Public Key Info:
Public Key Algorithm: rsaEncryption
Public-Key: (2048 bit)
Modulus:
00:b3:84:d6:8b:01:59:18:85:d1:dc:32:df:38:f7:
90:85:1b:3e:a5:5e:81:3e:2f:fc:3a:5f:7f:77:ef:
23:bb:3a:88:27:0f:be:25:46:cd:63:7d:cb:95:d8:
a5:50:10:d2:a2:d2:b7:97:d1:0d:6c:fb:f9:05:e8:
6f:a8:4b:bd:95:67:9e:7b:94:58:a9:6d:93:fd:e0:
12:c5:cd:b4:8a:64:52:31:5f:0e:e3:89:84:71:da:
98:dd:4b:ec:02:25:a5:7d:35:fe:63:da:b3:ac:ec:
a5:46:0f:0d:64:23:5c:6d:f3:ec:cc:28:63:23:c0:
4b:9a:ec:8f:c1:ee:b1:a2:3e:72:4d:70:b5:09:c1:
eb:b4:10:55:3c:8b:ea:1b:94:7e:4b:74:e6:f4:9f:
4f:a6:45:30:b5:f0:b8:b4:d1:59:50:65:0a:86:53:
ea:4c:9f:9e:f4:58:6c:31:f5:17:3a:6f:57:8b:cb:
5f:f0:28:0b:45:92:8d:30:20:49:ff:52:e6:2c:cb:
18:9a:d7:e6:ee:3e:4f:34:35:15:13:c5:02:da:c5:
5f:be:fb:5b:ce:8d:bf:b5:35:76:3c:7c:e6:9c:3b:
26:87:4d:8d:80:e6:16:c6:27:f2:50:49:b6:72:74:
43:49:49:44:38:bb:78:43:23:ee:16:3e:d9:62:e6:
a5:d7
Exponent: 65537 (0x10001)
X509v3 extensions:
X509v3 Basic Constraints:
CA:FALSE
X509v3 Subject Key Identifier:
2D:98:A3:99:39:1C:FE:E9:9A:6D:17:94:D2:3A:96:EE:C8:9E:04:22
X509v3 Authority Key Identifier:
keyid:6A:AB:53:64:92:36:87:23:34:B3:1D:6F:85:4B:F5:DF:5A:5C:74:8F
X509v3 Key Usage:
Digital Signature, Non Repudiation, Key Encipherment
X509v3 Extended Key Usage:
TLS Web Server Authentication
X509v3 Subject Alternative Name:
DNS:web-platform.test, DNS:www.web-platform.test, DNS:xn--n8j6ds53lwwkrqhv28a.web-platform.test, DNS:xn--lve-6lad.web-platform.test, DNS:www2.web-platform.test, DNS:www1.web-platform.test
Signature Algorithm: sha256WithRSAEncryption
33:db:f7:f0:f6:92:16:4f:2d:42:bc:b8:aa:e6:ab:5e:f9:b9:
b0:48:ae:b5:8d:cc:02:7b:e9:6f:4e:75:f7:17:a0:5e:7b:87:
06:49:48:83:c5:bb:ca:95:07:37:0e:5d:e3:97:de:9e:0c:a4:
82:30:11:81:49:5d:50:29:72:92:a5:ca:17:b1:7c:f1:32:11:
17:57:e6:59:c1:ac:e3:3b:26:d2:94:97:50:6a:b9:54:88:84:
9b:6f:b1:06:f5:80:04:22:10:14:b1:f5:97:25:fc:66:d6:69:
a3:36:08:85:23:ff:8e:3c:2b:e0:6d:e7:61:f1:00:8f:61:3d:
b0:87:ad:72:21:f6:f0:cc:4f:c9:20:bf:83:11:0f:21:f4:b8:
c0:dd:9c:51:d7:bb:27:32:ec:ab:a4:62:14:28:32:da:f2:87:
80:68:9c:ea:ac:eb:f5:7f:f5:de:f4:c0:39:91:c8:76:a4:ee:
d0:a8:50:db:c1:4b:f9:c4:3d:d9:e8:8e:b6:3f:c0:96:79:12:
d8:fa:4d:0a:b3:36:76:aa:4e:b2:82:2f:a2:d4:0d:db:fd:64:
77:6f:6e:e9:94:7f:0f:c8:3a:3c:96:3d:cd:4d:6c:ba:66:95:
f7:b4:9d:a4:94:9f:97:b3:9a:0d:dc:18:8c:11:0b:56:65:8e:
46:4c:e6:5e
-----BEGIN CERTIFICATE-----
MIID2jCCAsKgAwIBAgIBAjANBgkqhkiG9w0BAQsFADAdMRswGQYDVQQDDBJ3ZWIt
cGxhdGZvcm0tdGVzdHMwHhcNMTQxMjIyMTIwOTE2WhcNMjQxMjIxMTIwOTE2WjAc
MRowGAYDVQQDExF3ZWItcGxhdGZvcm0udGVzdDCCASIwDQYJKoZIhvcNAQEBBQAD
ggEPADCCAQoCggEBALOE1osBWRiF0dwy3zj3kIUbPqVegT4v/Dpff3fvI7s6iCcP
viVGzWN9y5XYpVAQ0qLSt5fRDWz7+QXob6hLvZVnnnuUWKltk/3gEsXNtIpkUjFf
DuOJhHHamN1L7AIlpX01/mPas6zspUYPDWQjXG3z7MwoYyPAS5rsj8HusaI+ck1w
tQnB67QQVTyL6huUfkt05vSfT6ZFMLXwuLTRWVBlCoZT6kyfnvRYbDH1FzpvV4vL
X/AoC0WSjTAgSf9S5izLGJrX5u4+TzQ1FRPFAtrFX777W86Nv7U1djx85pw7JodN
jYDmFsYn8lBJtnJ0Q0lJRDi7eEMj7hY+2WLmpdcCAwEAAaOCASQwggEgMAkGA1Ud
EwQCMAAwHQYDVR0OBBYEFC2Yo5k5HP7pmm0XlNI6lu7IngQiMB8GA1UdIwQYMBaA
FGqrU2SSNocjNLMdb4VL9d9aXHSPMAsGA1UdDwQEAwIF4DATBgNVHSUEDDAKBggr
BgEFBQcDATCBsAYDVR0RBIGoMIGlghF3ZWItcGxhdGZvcm0udGVzdIIVd3d3Lndl
Yi1wbGF0Zm9ybS50ZXN0gil4bi0tbjhqNmRzNTNsd3drcnFodjI4YS53ZWItcGxh
dGZvcm0udGVzdIIeeG4tLWx2ZS02bGFkLndlYi1wbGF0Zm9ybS50ZXN0ghZ3d3cy
LndlYi1wbGF0Zm9ybS50ZXN0ghZ3d3cxLndlYi1wbGF0Zm9ybS50ZXN0MA0GCSqG
SIb3DQEBCwUAA4IBAQAz2/fw9pIWTy1CvLiq5qte+bmwSK61jcwCe+lvTnX3F6Be
e4cGSUiDxbvKlQc3Dl3jl96eDKSCMBGBSV1QKXKSpcoXsXzxMhEXV+ZZwazjOybS
lJdQarlUiISbb7EG9YAEIhAUsfWXJfxm1mmjNgiFI/+OPCvgbedh8QCPYT2wh61y
IfbwzE/JIL+DEQ8h9LjA3ZxR17snMuyrpGIUKDLa8oeAaJzqrOv1f/Xe9MA5kch2
pO7QqFDbwUv5xD3Z6I62P8CWeRLY+k0KszZ2qk6ygi+i1A3b/WR3b27plH8PyDo8
lj3NTWy6ZpX3tJ2klJ+Xs5oN3BiMEQtWZY5GTOZe
-----END CERTIFICATE-----

7
testing/web-platform/harness/.gitignore vendored
View file

@ -1,7 +0,0 @@
*.py[co]
*~
*#
\#*
_virtualenv
test/test.cfg
test/metadata/MANIFEST.json

13
testing/web-platform/harness/MANIFEST.in
View file

@ -1,13 +0,0 @@
exclude MANIFEST.in
include requirements.txt
include wptrunner/browsers/b2g_setup/*
include wptrunner.default.ini
include wptrunner/testharness_runner.html
include wptrunner/testharnessreport.js
include wptrunner/testharnessreport-servo.js
include wptrunner/executors/testharness_marionette.js
include wptrunner/executors/testharness_webdriver.js
include wptrunner/executors/reftest.js
include wptrunner/executors/reftest-wait.js
include wptrunner/config.json
include wptrunner/browsers/server-locations.txt

242
testing/web-platform/harness/README.rst
View file

@ -1,242 +0,0 @@
wptrunner: A web-platform-tests harness
=======================================
wptrunner is a harness for running the W3C `web-platform-tests testsuite`_.
.. contents::
Installation
~~~~~~~~~~~~
wptrunner is expected to be installed into a virtualenv using pip. For
development, it can be installed using the `-e` option::
pip install -e ./
Running the Tests
~~~~~~~~~~~~~~~~~
After installation, the command ``wptrunner`` should be available to run
the tests.
The ``wptrunner`` command takes multiple options, of which the
following are most significant:
``--product`` (defaults to `firefox`)
The product to test against: `b2g`, `chrome`, `firefox`, or `servo`.
``--binary`` (required if product is `firefox` or `servo`)
The path to a binary file for the product (browser) to test against.
``--webdriver-binary`` (required if product is `chrome`)
The path to a `driver` binary; e.g., a `chromedriver` binary.
``--certutil-binary`` (required if product is `firefox` [#]_)
The path to a `certutil` binary (for tests that must be run over https).
``--metadata`` (required)
The path to a directory containing test metadata. [#]_
``--tests`` (required)
The path to a directory containing a web-platform-tests checkout.
``--prefs-root`` (required only when testing a Firefox binary)
The path to a directory containing Firefox test-harness preferences. [#]_
``--config`` (should default to `wptrunner.default.ini`)
The path to the config (ini) file.
.. [#] The ``--certutil-binary`` option is required when the product is
``firefox`` unless ``--ssl-type=none`` is specified.
.. [#] The ``--metadata`` path is to a directory that contains:
* a ``MANIFEST.json`` file (instructions on generating this file are
available in the `detailed documentation
<http://wptrunner.readthedocs.org/en/latest/usage.html#installing-wptrunner>`_);
and
* (optionally) any expectation files (see below)
.. [#] Example ``--prefs-root`` value: ``~/mozilla-central/testing/profiles``.
There are also a variety of other options available; use ``--help`` to
list them.
-------------------------------
Example: How to start wptrunner
-------------------------------
To test a Firefox Nightly build in an OS X environment, you might start
wptrunner using something similar to the following example::
wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \
--binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/dist/Nightly.app/Contents/MacOS/firefox \
--certutil-binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/security/nss/cmd/certutil/certutil \
--prefs-root=~/mozilla-central/testing/profiles
And to test a Chromium build in an OS X environment, you might start
wptrunner using something similar to the following example::
wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \
--binary=~/chromium/src/out/Release/Chromium.app/Contents/MacOS/Chromium \
--webdriver-binary=/usr/local/bin/chromedriver --product=chrome
-------------------------------------
Example: How to run a subset of tests
-------------------------------------
To restrict a test run just to tests in a particular web-platform-tests
subdirectory, specify the directory name in the positional arguments after
the options; for example, run just the tests in the `dom` subdirectory::
wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \
--binary=/path/to/firefox --certutil-binary=/path/to/certutil \
--prefs-root=/path/to/testing/profiles \
dom
Output
~~~~~~
By default wptrunner just dumps its entire output as raw JSON messages
to stdout. This is convenient for piping into other tools, but not ideal
for humans reading the output.
As an alternative, you can use the ``--log-mach`` option, which provides
output in a reasonable format for humans. The option requires a value:
either the path for a file to write the `mach`-formatted output to, or
"`-`" (a hyphen) to write the `mach`-formatted output to stdout.
When using ``--log-mach``, output of the full raw JSON log is still
available, from the ``--log-raw`` option. So to output the full raw JSON
log to a file and a human-readable summary to stdout, you might start
wptrunner using something similar to the following example::
wptrunner --metadata=~/web-platform-tests/ --tests=~/web-platform-tests/ \
--binary=/path/to/firefox --certutil-binary=/path/to/certutil \
--prefs-root=/path/to/testing/profiles \
--log-raw=output.log --log-mach=-
Expectation Data
~~~~~~~~~~~~~~~~
wptrunner is designed to be used in an environment where it is not
just necessary to know which tests passed, but to compare the results
between runs. For this reason it is possible to store the results of a
previous run in a set of ini-like "expectation files". This format is
documented below. To generate the expectation files use `wptrunner` with
the `--log-raw=/path/to/log/file` option. This can then be used as
input to the `wptupdate` tool.
Expectation File Format
~~~~~~~~~~~~~~~~~~~~~~~
Metadata about tests, notably including their expected results, is
stored in a modified ini-like format that is designed to be human
editable, but also to be machine updatable.
Each test file that requires metadata to be specified (because it has
a non-default expectation or because it is disabled, for example) has
a corresponding expectation file in the `metadata` directory. For
example a test file `html/test1.html` containing a failing test would
have an expectation file called `html/test1.html.ini` in the
`metadata` directory.
An example of an expectation file is::
example_default_key: example_value
[filename.html]
type: testharness
[subtest1]
expected: FAIL
[subtest2]
expected:
if platform == 'win': TIMEOUT
if platform == 'osx': ERROR
FAIL
[filename.html?query=something]
type: testharness
disabled: bug12345
The file consists of two elements, key-value pairs and
sections.
Sections are delimited by headings enclosed in square brackets. Any
closing square bracket in the heading itself my be escaped with a
backslash. Each section may then contain any number of key-value pairs
followed by any number of subsections. So that it is clear which data
belongs to each section without the use of end-section markers, the
data for each section (i.e. the key-value pairs and subsections) must
be indented using spaces. Indentation need only be consistent, but
using two spaces per level is recommended.
In a test expectation file, each resource provided by the file has a
single section, with the section heading being the part after the last
`/` in the test url. Tests that have subsections may have subsections
for those subtests in which the heading is the name of the subtest.
Simple key-value pairs are of the form::
key: value
Note that unlike ini files, only `:` is a valid seperator; `=` will
not work as expected. Key-value pairs may also have conditional
values of the form::
key:
if condition1: value1
if condition2: value2
default
In this case each conditional is evaluated in turn and the value is
that on the right hand side of the first matching conditional. In the
case that no condition matches, the unconditional default is used. If
no condition matches and no default is provided it is equivalent to
the key not being present. Conditionals use a simple python-like expression
language e.g.::
if debug and (platform == "linux" or platform == "osx"): FAIL
For test expectations the avaliable variables are those in the
`run_info` which for desktop are `version`, `os`, `bits`, `processor`,
`debug` and `product`.
Key-value pairs specified at the top level of the file before any
sections are special as they provide defaults for the rest of the file
e.g.::
key1: value1
[section 1]
key2: value2
[section 2]
key1: value3
In this case, inside section 1, `key1` would have the value `value1`
and `key2` the value `value2` whereas in section 2 `key1` would have
the value `value3` and `key2` would be undefined.
The web-platform-test harness knows about several keys:
`expected`
Must evaluate to a possible test status indicating the expected
result of the test. The implicit default is PASS or OK when the
field isn't present.
`disabled`
Any value indicates that the test is disabled.
`type`
The test type e.g. `testharness`, `reftest`, or `wdspec`.
`reftype`
The type of comparison for reftests; either `==` or `!=`.
`refurl`
The reference url for reftests.
.. _`web-platform-tests testsuite`: https://github.com/w3c/web-platform-tests

177
testing/web-platform/harness/docs/Makefile
View file

@ -1,177 +0,0 @@
# Makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
PAPER =
BUILDDIR = _build
# User-friendly check for sphinx-build
ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1)
$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/)
endif
# Internal variables.
PAPEROPT_a4 = -D latex_paper_size=a4
PAPEROPT_letter = -D latex_paper_size=letter
ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
# the i18n builder cannot share the environment and doctrees with the others
I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) .
.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo " html to make standalone HTML files"
@echo " dirhtml to make HTML files named index.html in directories"
@echo " singlehtml to make a single large HTML file"
@echo " pickle to make pickle files"
@echo " json to make JSON files"
@echo " htmlhelp to make HTML files and a HTML help project"
@echo " qthelp to make HTML files and a qthelp project"
@echo " devhelp to make HTML files and a Devhelp project"
@echo " epub to make an epub"
@echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
@echo " latexpdf to make LaTeX files and run them through pdflatex"
@echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
@echo " text to make text files"
@echo " man to make manual pages"
@echo " texinfo to make Texinfo files"
@echo " info to make Texinfo files and run them through makeinfo"
@echo " gettext to make PO message catalogs"
@echo " changes to make an overview of all changed/added/deprecated items"
@echo " xml to make Docutils-native XML files"
@echo " pseudoxml to make pseudoxml-XML files for display purposes"
@echo " linkcheck to check all external links for integrity"
@echo " doctest to run all doctests embedded in the documentation (if enabled)"
clean:
rm -rf $(BUILDDIR)/*
html:
$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/html."
dirhtml:
$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
@echo
@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
singlehtml:
$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
@echo
@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
pickle:
$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
@echo
@echo "Build finished; now you can process the pickle files."
json:
$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
@echo
@echo "Build finished; now you can process the JSON files."
htmlhelp:
$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
@echo
@echo "Build finished; now you can run HTML Help Workshop with the" \
".hhp project file in $(BUILDDIR)/htmlhelp."
qthelp:
$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
@echo
@echo "Build finished; now you can run "qcollectiongenerator" with the" \
".qhcp project file in $(BUILDDIR)/qthelp, like this:"
@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/wptrunner.qhcp"
@echo "To view the help file:"
@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/wptrunner.qhc"
devhelp:
$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
@echo
@echo "Build finished."
@echo "To view the help file:"
@echo "# mkdir -p $$HOME/.local/share/devhelp/wptrunner"
@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/wptrunner"
@echo "# devhelp"
epub:
$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
@echo
@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
latex:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo
@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
@echo "Run \`make' in that directory to run these through (pdf)latex" \
"(use \`make latexpdf' here to do that automatically)."
latexpdf:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through pdflatex..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
latexpdfja:
$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
@echo "Running LaTeX files through platex and dvipdfmx..."
$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
text:
$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
@echo
@echo "Build finished. The text files are in $(BUILDDIR)/text."
man:
$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
@echo
@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
texinfo:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo
@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
@echo "Run \`make' in that directory to run these through makeinfo" \
"(use \`make info' here to do that automatically)."
info:
$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
@echo "Running Texinfo files through makeinfo..."
make -C $(BUILDDIR)/texinfo info
@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
gettext:
$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
@echo
@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
changes:
$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
@echo
@echo "The overview file is in $(BUILDDIR)/changes."
linkcheck:
$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
@echo
@echo "Link check complete; look for any errors in the above output " \
"or in $(BUILDDIR)/linkcheck/output.txt."
doctest:
$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
@echo "Testing of doctests in the sources finished, look at the " \
"results in $(BUILDDIR)/doctest/output.txt."
xml:
$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
@echo
@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
pseudoxml:
$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
@echo
@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

1
testing/web-platform/harness/docs/architecture.svg
View file

File diff suppressed because one or more lines are too long
Side by side

Before

Width:  |  Height:  |  Size: 19 KiB

267
testing/web-platform/harness/docs/conf.py
View file

@ -1,267 +0,0 @@
# -*- coding: utf-8 -*-
#
# wptrunner documentation build configuration file, created by
# sphinx-quickstart on Mon May 19 18:14:20 2014.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys
import os
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
'sphinx.ext.autodoc',
'sphinx.ext.intersphinx',
'sphinx.ext.viewcode',
]
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'wptrunner'
copyright = u''
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.3'
# The full version, including alpha/beta/rc tags.
release = '0.3'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']
# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'wptrunnerdoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#'preamble': '',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
('index', 'wptrunner.tex', u'wptrunner Documentation',
u'James Graham', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'wptrunner', u'wptrunner Documentation',
[u'James Graham'], 1)
]
# If true, show URL addresses after external links.
#man_show_urls = False
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
('index', 'wptrunner', u'wptrunner Documentation',
u'James Graham', 'wptrunner', 'One line description of project.',
'Miscellaneous'),
]
# Documents to append as an appendix to all manuals.
#texinfo_appendices = []
# If false, no module index is generated.
#texinfo_domain_indices = True
# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'
# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False
# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {'python': ('http://docs.python.org/', None),
'mozlog': ('http://mozbase.readthedocs.org/en/latest/', None)}

106
testing/web-platform/harness/docs/design.rst
View file

@ -1,106 +0,0 @@
wptrunner Design
================
The design of wptrunner is intended to meet the following
requirements:
* Possible to run tests from W3C web-platform-tests.
* Tests should be run as fast as possible. In particular it should
not be necessary to restart the browser between tests, or similar.
* As far as possible, the tests should run in a "normal" browser and
browsing context. In particular many tests assume that they are
running in a top-level browsing context, so we must avoid the use
of an ``iframe`` test container.
* It must be possible to deal with all kinds of behaviour of the
browser runder test, for example, crashing, hanging, etc.
* It should be possible to add support for new platforms and browsers
with minimal code changes.
* It must be possible to run tests in parallel to further improve
performance.
* Test output must be in a machine readable form.
Architecture
------------
In order to meet the above requirements, wptrunner is designed to
push as much of the test scheduling as possible into the harness. This
allows the harness to monitor the state of the browser and perform
appropriate action if it gets into an unwanted state e.g. kill the
browser if it appears to be hung.
The harness will typically communicate with the browser via some remote
control protocol such as WebDriver. However for browsers where no such
protocol is supported, other implementation strategies are possible,
typically at the expense of speed.
The overall architecture of wptrunner is shown in the diagram below:
.. image:: architecture.svg
The main entry point to the code is :py:func:`run_tests` in
``wptrunner.py``. This is responsible for setting up the test
environment, loading the list of tests to be executed, and invoking
the remainder of the code to actually execute some tests.
The test environment is encapsulated in the
:py:class:`TestEnvironment` class. This defers to code in
``web-platform-tests`` which actually starts the required servers to
run the tests.
The set of tests to run is defined by the
:py:class:`TestLoader`. This is constructed with a
:py:class:`TestFilter` (not shown), which takes any filter arguments
from the command line to restrict the set of tests that will be
run. The :py:class:`TestLoader` reads both the ``web-platform-tests``
JSON manifest and the expectation data stored in ini files and
produces a :py:class:`multiprocessing.Queue` of tests to run, and
their expected results.
Actually running the tests happens through the
:py:class:`ManagerGroup` object. This takes the :py:class:`Queue` of
tests to be run and starts a :py:class:`testrunner.TestRunnerManager` for each
instance of the browser under test that will be started. These
:py:class:`TestRunnerManager` instances are each started in their own
thread.
A :py:class:`TestRunnerManager` coordinates starting the product under
test, and outputting results from the test. In the case that the test
has timed out or the browser has crashed, it has to restart the
browser to ensure the test run can continue. The functionality for
initialising the browser under test, and probing its state
(e.g. whether the process is still alive) is implemented through a
:py:class:`Browser` object. An implementation of this class must be
provided for each product that is supported.
The functionality for actually running the tests is provided by a
:py:class:`TestRunner` object. :py:class:`TestRunner` instances are
run in their own child process created with the
:py:mod:`multiprocessing` module. This allows them to run concurrently
and to be killed and restarted as required. Communication between the
:py:class:`TestRunnerManager` and the :py:class:`TestRunner` is
provided by a pair of queues, one for sending messages in each
direction. In particular test results are sent from the
:py:class:`TestRunner` to the :py:class:`TestRunnerManager` using one
of these queues.
The :py:class:`TestRunner` object is generic in that the same
:py:class:`TestRunner` is used regardless of the product under
test. However the details of how to run the test may vary greatly with
the product since different products support different remote control
protocols (or none at all). These protocol-specific parts are placed
in the :py:class:`Executor` object. There is typically a different
:py:class:`Executor` class for each combination of control protocol
and test type. The :py:class:`TestRunner` is responsible for pulling
each test off the :py:class:`Queue` of tests and passing it down to
the :py:class:`Executor`.
The executor often requires access to details of the particular
browser instance that it is testing so that it knows e.g. which port
to connect to to send commands to the browser. These details are
encapsulated in the :py:class:`ExecutorBrowser` class.

248
testing/web-platform/harness/docs/expectation.rst
View file

@ -1,248 +0,0 @@
Expectation Data
================
Introduction
------------
For use in continuous integration systems, and other scenarios where
regression tracking is required, wptrunner supports storing and
loading the expected result of each test in a test run. Typically
these expected results will initially be generated by running the
testsuite in a baseline build. They may then be edited by humans as
new features are added to the product that change the expected
results. The expected results may also vary for a single product
depending on the platform on which it is run. Therefore, the raw
structured log data is not a suitable format for storing these
files. Instead something is required that is:
* Human readable
* Human editable
* Machine readable / writable
* Capable of storing test id / result pairs
* Suitable for storing in a version control system (i.e. text-based)
The need for different results per platform means either having
multiple expectation files for each platform, or having a way to
express conditional values within a certain file. The former would be
rather cumbersome for humans updating the expectation files, so the
latter approach has been adopted, leading to the requirement:
* Capable of storing result values that are conditional on the platform.
There are few extant formats that meet these requirements, so
wptrunner uses a bespoke ``expectation manifest`` format, which is
closely based on the standard ``ini`` format.
Directory Layout
----------------
Expectation manifest files must be stored under the ``metadata``
directory passed to the test runner. The directory layout follows that
of web-platform-tests with each test path having a corresponding
manifest file. Tests that differ only by query string, or reftests
with the same test path but different ref paths share the same
reference file. The file name is taken from the last /-separated part
of the path, suffixed with ``.ini``.
As an optimisation, files which produce only default results
(i.e. ``PASS`` or ``OK``) don't require a corresponding manifest file.
For example a test with url::
/spec/section/file.html?query=param
would have an expectation file ::
metadata/spec/section/file.html.ini
.. _wptupdate-label:
Generating Expectation Files
----------------------------
wptrunner provides the tool ``wptupdate`` to generate expectation
files from the results of a set of baseline test runs. The basic
syntax for this is::
wptupdate [options] [logfile]...
Each ``logfile`` is a structured log file from a previous run. These
can be generated from wptrunner using the ``--log-raw`` option
e.g. ``--log-raw=structured.log``. The default behaviour is to update
all the test data for the particular combination of hardware and OS
used in the run corresponding to the log data, whilst leaving any
other expectations untouched.
wptupdate takes several useful options:
``--sync``
Pull the latest version of web-platform-tests from the
upstream specified in the config file. If this is specified in
combination with logfiles, it is assumed that the results in the log
files apply to the post-update tests.
``--no-check-clean``
Don't attempt to check if the working directory is clean before
doing the update (assuming that the working directory is a git or
mercurial tree).
``--patch``
Create a a git commit, or a mq patch, with the changes made by wptupdate.
``--ignore-existing``
Overwrite all the expectation data for any tests that have a result
in the passed log files, not just data for the same platform.
Examples
~~~~~~~~
Update the local copy of web-platform-tests without changing the
expectation data and commit (or create a mq patch for) the result::
wptupdate --patch --sync
Update all the expectations from a set of cross-platform test runs::
wptupdate --no-check-clean --patch osx.log linux.log windows.log
Add expectation data for some new tests that are expected to be
platform-independent::
wptupdate --no-check-clean --patch --ignore-existing tests.log
Manifest Format
---------------
The format of the manifest files is based on the ini format. Files are
divided into sections, each (apart from the root section) having a
heading enclosed in square braces. Within each section are key-value
pairs. There are several notable differences from standard .ini files,
however:
* Sections may be hierarchically nested, with significant whitespace
indicating nesting depth.
* Only ``:`` is valid as a key/value separator
A simple example of a manifest file is::
root_key: root_value
[section]
section_key: section_value
[subsection]
subsection_key: subsection_value
[another_section]
another_key: another_value
Conditional Values
~~~~~~~~~~~~~~~~~~
In order to support values that depend on some external data, the
right hand side of a key/value pair can take a set of conditionals
rather than a plain value. These values are placed on a new line
following the key, with significant indentation. Conditional values
are prefixed with ``if`` and terminated with a colon, for example::
key:
if cond1: value1
if cond2: value2
value3
In this example, the value associated with ``key`` is determined by
first evaluating ``cond1`` against external data. If that is true,
``key`` is assigned the value ``value1``, otherwise ``cond2`` is
evaluated in the same way. If both ``cond1`` and ``cond2`` are false,
the unconditional ``value3`` is used.
Conditions themselves use a Python-like expression syntax. Operands
can either be variables, corresponding to data passed in, numbers
(integer or floating point; exponential notation is not supported) or
quote-delimited strings. Equality is tested using ``==`` and
inequality by ``!=``. The operators ``and``, ``or`` and ``not`` are
used in the expected way. Parentheses can also be used for
grouping. For example::
key:
if (a == 2 or a == 3) and b == "abc": value1
if a == 1 or b != "abc": value2
value3
Here ``a`` and ``b`` are variables, the value of which will be
supplied when the manifest is used.
Expectation Manifests
---------------------
When used for expectation data, manifests have the following format:
* A section per test URL described by the manifest, with the section
heading being the part of the test URL following the last ``/`` in
the path (this allows multiple tests in a single manifest file with
the same path part of the URL, but different query parts).
* A subsection per subtest, with the heading being the title of the
subtest.
* A key ``type`` indicating the test type. This takes the values
``testharness`` and ``reftest``.
* For reftests, keys ``reftype`` indicating the reference type
(``==`` or ``!=``) and ``refurl`` indicating the URL of the
reference.
* A key ``expected`` giving the expectation value of each (sub)test.
* A key ``disabled`` which can be set to any value to indicate that
the (sub)test is disabled and should either not be run (for tests)
or that its results should be ignored (subtests).
* A key ``restart-after`` which can be set to any value to indicate that
the runner should restart the browser after running this test (e.g. to
clear out unwanted state).
* Variables ``debug``, ``os``, ``version``, ``processor`` and
``bits`` that describe the configuration of the browser under
test. ``debug`` is a boolean indicating whether a build is a debug
build. ``os`` is a string indicating the operating system, and
``version`` a string indicating the particular version of that
operating system. ``processor`` is a string indicating the
processor architecture and ``bits`` an integer indicating the
number of bits. This information is typically provided by
:py:mod:`mozinfo`.
* Top level keys are taken as defaults for the whole file. So, for
example, a top level key with ``expected: FAIL`` would indicate
that all tests and subtests in the file are expected to fail,
unless they have an ``expected`` key of their own.
An simple example manifest might look like::
[test.html?variant=basic]
type: testharness
[Test something unsupported]
expected: FAIL
[test.html?variant=broken]
expected: ERROR
[test.html?variant=unstable]
disabled: http://test.bugs.example.org/bugs/12345
A more complex manifest with conditional properties might be::
[canvas_test.html]
expected:
if os == "osx": FAIL
if os == "windows" and version == "XP": FAIL
PASS
Note that ``PASS`` in the above works, but is unnecessary; ``PASS``
(or ``OK``) is always the default expectation for (sub)tests.

24
testing/web-platform/harness/docs/index.rst
View file

@ -1,24 +0,0 @@
.. wptrunner documentation master file, created by
sphinx-quickstart on Mon May 19 18:14:20 2014.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to wptrunner's documentation!
=====================================
Contents:
.. toctree::
:maxdepth: 2
usage
expectation
design
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

242
testing/web-platform/harness/docs/make.bat
View file

@ -1,242 +0,0 @@
@ECHO OFF
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set BUILDDIR=_build
set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% .
set I18NSPHINXOPTS=%SPHINXOPTS% .
if NOT "%PAPER%" == "" (
set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS%
set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS%
)
if "%1" == "" goto help
if "%1" == "help" (
:help
echo.Please use `make ^<target^>` where ^<target^> is one of
echo. html to make standalone HTML files
echo. dirhtml to make HTML files named index.html in directories
echo. singlehtml to make a single large HTML file
echo. pickle to make pickle files
echo. json to make JSON files
echo. htmlhelp to make HTML files and a HTML help project
echo. qthelp to make HTML files and a qthelp project
echo. devhelp to make HTML files and a Devhelp project
echo. epub to make an epub
echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter
echo. text to make text files
echo. man to make manual pages
echo. texinfo to make Texinfo files
echo. gettext to make PO message catalogs
echo. changes to make an overview over all changed/added/deprecated items
echo. xml to make Docutils-native XML files
echo. pseudoxml to make pseudoxml-XML files for display purposes
echo. linkcheck to check all external links for integrity
echo. doctest to run all doctests embedded in the documentation if enabled
goto end
)
if "%1" == "clean" (
for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i
del /q /s %BUILDDIR%\*
goto end
)
%SPHINXBUILD% 2> nul
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
if "%1" == "html" (
%SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/html.
goto end
)
if "%1" == "dirhtml" (
%SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml.
goto end
)
if "%1" == "singlehtml" (
%SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml.
goto end
)
if "%1" == "pickle" (
%SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the pickle files.
goto end
)
if "%1" == "json" (
%SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can process the JSON files.
goto end
)
if "%1" == "htmlhelp" (
%SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run HTML Help Workshop with the ^
.hhp project file in %BUILDDIR%/htmlhelp.
goto end
)
if "%1" == "qthelp" (
%SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished; now you can run "qcollectiongenerator" with the ^
.qhcp project file in %BUILDDIR%/qthelp, like this:
echo.^> qcollectiongenerator %BUILDDIR%\qthelp\wptrunner.qhcp
echo.To view the help file:
echo.^> assistant -collectionFile %BUILDDIR%\qthelp\wptrunner.ghc
goto end
)
if "%1" == "devhelp" (
%SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp
if errorlevel 1 exit /b 1
echo.
echo.Build finished.
goto end
)
if "%1" == "epub" (
%SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The epub file is in %BUILDDIR%/epub.
goto end
)
if "%1" == "latex" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
if errorlevel 1 exit /b 1
echo.
echo.Build finished; the LaTeX files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdf" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf
cd %BUILDDIR%/..
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "latexpdfja" (
%SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex
cd %BUILDDIR%/latex
make all-pdf-ja
cd %BUILDDIR%/..
echo.
echo.Build finished; the PDF files are in %BUILDDIR%/latex.
goto end
)
if "%1" == "text" (
%SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The text files are in %BUILDDIR%/text.
goto end
)
if "%1" == "man" (
%SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The manual pages are in %BUILDDIR%/man.
goto end
)
if "%1" == "texinfo" (
%SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo.
goto end
)
if "%1" == "gettext" (
%SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The message catalogs are in %BUILDDIR%/locale.
goto end
)
if "%1" == "changes" (
%SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes
if errorlevel 1 exit /b 1
echo.
echo.The overview file is in %BUILDDIR%/changes.
goto end
)
if "%1" == "linkcheck" (
%SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck
if errorlevel 1 exit /b 1
echo.
echo.Link check complete; look for any errors in the above output ^
or in %BUILDDIR%/linkcheck/output.txt.
goto end
)
if "%1" == "doctest" (
%SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest
if errorlevel 1 exit /b 1
echo.
echo.Testing of doctests in the sources finished, look at the ^
results in %BUILDDIR%/doctest/output.txt.
goto end
)
if "%1" == "xml" (
%SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The XML files are in %BUILDDIR%/xml.
goto end
)
if "%1" == "pseudoxml" (
%SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml
if errorlevel 1 exit /b 1
echo.
echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml.
goto end
)
:end

238
testing/web-platform/harness/docs/usage.rst
View file

@ -1,238 +0,0 @@
Getting Started
===============
Installing wptrunner
--------------------
The easiest way to install wptrunner is into a virtualenv, using pip::
virtualenv wptrunner
cd wptrunner
source bin/activate
pip install wptrunner
This will install the base dependencies for wptrunner, but not any
extra dependencies required to test against specific browsers. In
order to do this you must use use the extra requirements files in
``$VIRTUAL_ENV/requirements/requirements_browser.txt``. For example,
in order to test against Firefox you would have to run::
pip install -r requirements/requirements_firefox.txt
If you intend to work on the code, the ``-e`` option to pip should be
used in combination with a source checkout i.e. inside a virtual
environment created as above::
git clone https://github.com/w3c/wptrunner.git
cd wptrunner
pip install -e ./
In addition to the dependencies installed by pip, wptrunner requires
a copy of the web-platform-tests repository. This can be located
anywhere on the filesystem, but the easiest option is to put it
under the same parent directory as the wptrunner checkout::
git clone https://github.com/w3c/web-platform-tests.git
It is also necessary to generate a web-platform-tests ``MANIFEST.json``
file. It's recommended to also put that under the same parent directory as
the wptrunner checkout, in a directory named ``meta``::
mkdir meta
cd web-platform-tests
python manifest --path ../meta/MANIFEST.json
The ``MANIFEST.json`` file needs to be regenerated each time the
web-platform-tests checkout is updated. To aid with the update process
there is a tool called ``wptupdate``, which is described in
:ref:`wptupdate-label`.
Running the Tests
-----------------
A test run is started using the ``wptrunner`` command. The command
takes multiple options, of which the following are most significant:
``--product`` (defaults to `firefox`)
The product to test against: `b2g`, `chrome`, `firefox`, or `servo`.
``--binary`` (required if product is `firefox` or `servo`)
The path to a binary file for the product (browser) to test against.
``--webdriver-binary`` (required if product is `chrome`)
The path to a `*driver` binary; e.g., a `chromedriver` binary.
``--certutil-binary`` (required if product is `firefox` [#]_)
The path to a `certutil` binary (for tests that must be run over https).
``--metadata`` (required only when not `using default paths`_)
The path to a directory containing test metadata. [#]_
``--tests`` (required only when not `using default paths`_)
The path to a directory containing a web-platform-tests checkout.
``--prefs-root`` (required only when testing a Firefox binary)
The path to a directory containing Firefox test-harness preferences. [#]_
``--config`` (should default to `wptrunner.default.ini`)
The path to the config (ini) file.
.. [#] The ``--certutil-binary`` option is required when the product is
``firefox`` unless ``--ssl-type=none`` is specified.
.. [#] The ``--metadata`` path is to a directory that contains:
* a ``MANIFEST.json`` file (the web-platform-tests documentation has
instructions on generating this file)
* (optionally) any expectation files (see :ref:`wptupdate-label`)
.. [#] Example ``--prefs-root`` value: ``~/mozilla-central/testing/profiles``.
There are also a variety of other command-line options available; use
``--help`` to list them.
The following examples show how to start wptrunner with various options.
------------------
Starting wptrunner
------------------
The examples below assume the following directory layout,
though no specific folder structure is required::
~/testtwf/wptrunner # wptrunner checkout
~/testtwf/web-platform-tests # web-platform-tests checkout
~/testtwf/meta # metadata
To test a Firefox Nightly build in an OS X environment, you might start
wptrunner using something similar to the following example::
wptrunner --metadata=~/testtwf/meta/ --tests=~/testtwf/web-platform-tests/ \
--binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/dist/Nightly.app/Contents/MacOS/firefox \
--certutil-binary=~/mozilla-central/obj-x86_64-apple-darwin14.3.0/security/nss/cmd/certutil/certutil \
--prefs-root=~/mozilla-central/testing/profiles
And to test a Chromium build in an OS X environment, you might start
wptrunner using something similar to the following example::
wptrunner --metadata=~/testtwf/meta/ --tests=~/testtwf/web-platform-tests/ \
--binary=~/chromium/src/out/Release/Chromium.app/Contents/MacOS/Chromium \
--webdriver-binary=/usr/local/bin/chromedriver --product=chrome
--------------------
Running test subsets
--------------------
To restrict a test run just to tests in a particular web-platform-tests
subdirectory, specify the directory name in the positional arguments after
the options; for example, run just the tests in the `dom` subdirectory::
wptrunner --metadata=~/testtwf/meta --tests=~/testtwf/web-platform-tests/ \
--binary=/path/to/firefox --certutil-binary=/path/to/certutil \
--prefs-root=/path/to/testing/profiles \
dom
-------------------
Running in parallel
-------------------
To speed up the testing process, use the ``--processes`` option to have
wptrunner run multiple browser instances in parallel. For example, to
have wptrunner attempt to run tests against with six browser instances
in parallel, specify ``--processes=6``. But note that behaviour in this
mode is necessarily less deterministic than with ``--processes=1`` (the
default), so there may be more noise in the test results.
-------------------
Using default paths
-------------------
The (otherwise-required) ``--tests`` and ``--metadata`` command-line
options/flags be omitted if any configuration file is found that
contains a section specifying the ``tests`` and ``metadata`` keys.
See the `Configuration File`_ section for more information about
configuration files, including information about their expected
locations.
The content of the ``wptrunner.default.ini`` default configuration file
makes wptrunner look for tests (that is, a web-platform-tests checkout)
as a subdirectory of the current directory named ``tests``, and for
metadata files in a subdirectory of the current directory named ``meta``.
Output
------
wptrunner uses the :py:mod:`mozlog` package for output. This
structures events such as test results or log messages as JSON objects
that can then be fed to other tools for interpretation. More details
about the message format are given in the
:py:mod:`mozlog` documentation.
By default the raw JSON messages are dumped to stdout. This is
convenient for piping into other tools, but not ideal for humans
reading the output. :py:mod:`mozlog` comes with several other
formatters, which are accessible through command line options. The
general format of these options is ``--log-name=dest``, where ``name``
is the name of the format and ``dest`` is a path to a destination
file, or ``-`` for stdout. The raw JSON data is written by the ``raw``
formatter so, the default setup corresponds to ``--log-raw=-``.
A reasonable output format for humans is provided as ``mach``. So in
order to output the full raw log to a file and a human-readable
summary to stdout, one might pass the options::
--log-raw=output.log --log-mach=-
Configuration File
------------------
wptrunner uses a ``.ini`` file to control some configuration
sections. The file has three sections; ``[products]``,
``[manifest:default]`` and ``[web-platform-tests]``.
``[products]`` is used to
define the set of available products. By default this section is empty
which means that all the products distributed with wptrunner are
enabled (although their dependencies may not be installed). The set
of enabled products can be set by using the product name as the
key. For built in products the value is empty. It is also possible to
provide the path to a script implementing the browser functionality
e.g.::
[products]
chrome =
netscape4 = path/to/netscape.py
``[manifest:default]`` specifies the default paths for the tests and metadata,
relative to the config file. For example::
[manifest:default]
tests = ~/testtwf/web-platform-tests
metadata = ~/testtwf/meta
``[web-platform-tests]`` is used to set the properties of the upstream
repository when updating the paths. ``remote_url`` specifies the git
url to pull from; ``branch`` the branch to sync against and
``sync_path`` the local path, relative to the configuration file, to
use when checking out the tests e.g.::
[web-platform-tests]
remote_url = https://github.com/w3c/web-platform-tests.git
branch = master
sync_path = sync
A configuration file must contain all the above fields; falling back
to the default values for unspecified fields is not yet supported.
The ``wptrunner`` and ``wptupdate`` commands will use configuration
files in the following order:
* Any path supplied with a ``--config`` flag to the command.
* A file called ``wptrunner.ini`` in the current directory
* The default configuration file (``wptrunner.default.ini`` in the
source directory)

4
testing/web-platform/harness/requirements.txt
View file

@ -1,4 +0,0 @@
html5lib >= 0.99
mozinfo >= 0.7
mozlog >= 3.0
mozdebug >= 0.1

7
testing/web-platform/harness/requirements_b2g.txt
View file

@ -1,7 +0,0 @@
fxos_appgen >= 0.5
mozdevice >= 0.41
gaiatest >= 0.26
marionette_client >= 0.7.10
moznetwork >= 0.24
mozprofile >= 0.21
mozrunner >= 6.1

2
testing/web-platform/harness/requirements_chrome.txt
View file

@ -1,2 +0,0 @@
mozprocess >= 0.19
selenium >= 2.41.0

5
testing/web-platform/harness/requirements_firefox.txt
View file

@ -1,5 +0,0 @@
marionette_driver >= 0.4
mozprofile >= 0.21
mozprocess >= 0.19
mozcrash >= 0.13
mozrunner >= 6.7

1
testing/web-platform/harness/requirements_servo.txt
View file

@ -1 +0,0 @@
mozprocess >= 0.19

73
testing/web-platform/harness/setup.py
View file

@ -1,73 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import glob
import os
import sys
import textwrap
from setuptools import setup, find_packages
here = os.path.split(__file__)[0]
PACKAGE_NAME = 'wptrunner'
PACKAGE_VERSION = '1.14'
# Dependencies
with open(os.path.join(here, "requirements.txt")) as f:
deps = f.read().splitlines()
# Browser-specific requirements
requirements_files = glob.glob(os.path.join(here, "requirements_*.txt"))
profile_dest = None
dest_exists = False
setup(name=PACKAGE_NAME,
version=PACKAGE_VERSION,
description="Harness for running the W3C web-platform-tests against various products",
author='Mozilla Automation and Testing Team',
author_email='tools@lists.mozilla.org',
license='MPL 2.0',
packages=find_packages(exclude=["tests", "metadata", "prefs"]),
entry_points={
'console_scripts': [
'wptrunner = wptrunner.wptrunner:main',
'wptupdate = wptrunner.update:main',
]
},
zip_safe=False,
platforms=['Any'],
classifiers=['Development Status :: 4 - Beta',
'Environment :: Console',
'Intended Audience :: Developers',
'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
'Operating System :: OS Independent'],
package_data={"wptrunner": ["executors/testharness_marionette.js",
"executors/testharness_webdriver.js",
"executors/reftest.js",
"executors/reftest-wait.js",
"testharnessreport.js",
"testharness_runner.html",
"config.json",
"wptrunner.default.ini",
"browsers/server-locations.txt",
"browsers/b2g_setup/*",
"prefs/*"]},
include_package_data=True,
data_files=[("requirements", requirements_files)],
install_requires=deps
)
if "install" in sys.argv:
path = os.path.relpath(os.path.join(sys.prefix, "requirements"), os.curdir)
print textwrap.fill("""In order to use with one of the built-in browser
products, you will need to install the extra dependencies. These are provided
as requirements_[name].txt in the %s directory and can be installed using
e.g.""" % path, 80)
print """
pip install -r %s/requirements_firefox.txt
""" % path

3
testing/web-platform/harness/test/metadata/reftest/reftest_and_fail.html.ini
View file

@ -1,3 +0,0 @@
[reftest_and_fail.html]
type: reftest
expected: FAIL

3
testing/web-platform/harness/test/metadata/reftest/reftest_cycle_fail.html.ini
View file

@ -1,3 +0,0 @@
[reftest_cycle_fail.html]
type: reftest
expected: FAIL

3
testing/web-platform/harness/test/metadata/reftest/reftest_match_fail.html.ini
View file

@ -1,3 +0,0 @@
[reftest_match_fail.html]
type: reftest
expected: FAIL

3
testing/web-platform/harness/test/metadata/reftest/reftest_mismatch_fail.html.ini
View file

@ -1,3 +0,0 @@
[reftest_mismatch_fail.html]
type: reftest
expected: FAIL

3
testing/web-platform/harness/test/metadata/reftest/reftest_ref_timeout.html.ini
View file

@ -1,3 +0,0 @@
[reftest_ref_timeout.html]
type: reftest
expected: TIMEOUT

3
testing/web-platform/harness/test/metadata/reftest/reftest_timeout.html.ini
View file

@ -1,3 +0,0 @@
[reftest_timeout.html]
type: reftest
expected: TIMEOUT

2
testing/web-platform/harness/test/metadata/testharness/firefox/__dir__.ini
View file

@ -1,2 +0,0 @@
prefs: ["browser.display.foreground_color:#FF0000",
"browser.display.background_color:#000000"]

2
testing/web-platform/harness/test/metadata/testharness/firefox/subdir/test_pref_reset.html.ini
View file

@ -1,2 +0,0 @@
[test_pref_reset.html]
prefs: [@Reset]

3
testing/web-platform/harness/test/metadata/testharness/firefox/test_pref_set.html.ini
View file

@ -1,3 +0,0 @@
[test_pref_set.html]
prefs: ["browser.display.foreground_color:#00FF00",
"browser.display.background_color:#000000"]

1
testing/web-platform/harness/test/metadata/testharness/subdir/__dir__.ini
View file

@ -1 +0,0 @@
disabled: true

2
testing/web-platform/harness/test/metadata/testharness/subdir/testharness_1.html.ini
View file

@ -1,2 +0,0 @@
[testharness_1.html]
disabled: @False

4
testing/web-platform/harness/test/metadata/testharness/testharness_0.html.ini
View file

@ -1,4 +0,0 @@
[testharness_0.html]
type: testharness
[Test that should fail]
expected: FAIL

3
testing/web-platform/harness/test/metadata/testharness/testharness_error.html.ini
View file

@ -1,3 +0,0 @@
[testharness_error.html]
type: testharness
expected: ERROR

3
testing/web-platform/harness/test/metadata/testharness/testharness_timeout.html.ini
View file

@ -1,3 +0,0 @@
[testharness_timeout.html]
type: testharness
expected: TIMEOUT

20
testing/web-platform/harness/test/test.cfg.example
View file

@ -1,20 +0,0 @@
[general]
tests=/path/to/web-platform-tests/
metadata=/path/to/web-platform-tests/
ssl-type=none
# [firefox]
# binary=/path/to/firefox
# prefs-root=/path/to/gecko-src/testing/profiles/
# [servo]
# binary=/path/to/servo-src/target/release/servo
# exclude=testharness # Because it needs a special testharness.js
# [servodriver]
# binary=/path/to/servo-src/target/release/servo
# exclude=testharness # Because it needs a special testharness.js
# [chrome]
# binary=/path/to/chrome
# webdriver-binary=/path/to/chromedriver

166
testing/web-platform/harness/test/test.py
View file

@ -1,166 +0,0 @@
import ConfigParser
import argparse
import json
import os
import sys
import tempfile
import threading
import time
from StringIO import StringIO
from mozlog import structuredlog, reader
from mozlog.handlers import BaseHandler, StreamHandler, StatusHandler
from mozlog.formatters import MachFormatter
from wptrunner import wptcommandline, wptrunner
here = os.path.abspath(os.path.dirname(__file__))
def setup_wptrunner_logging(logger):
structuredlog.set_default_logger(logger)
wptrunner.logger = logger
wptrunner.wptlogging.setup_stdlib_logger()
class ResultHandler(BaseHandler):
def __init__(self, verbose=False, logger=None):
self.inner = StreamHandler(sys.stdout, MachFormatter())
BaseHandler.__init__(self, self.inner)
self.product = None
self.verbose = verbose
self.logger = logger
self.register_message_handlers("wptrunner-test", {"set-product": self.set_product})
def set_product(self, product):
self.product = product
def __call__(self, data):
if self.product is not None and data["action"] in ["suite_start", "suite_end"]:
# Hack: mozlog sets some internal state to prevent multiple suite_start or
# suite_end messages. We actually want that here (one from the metaharness
# and one from the individual test type harness), so override that internal
# state (a better solution might be to not share loggers, but this works well
# enough)
self.logger._state.suite_started = True
return
if (not self.verbose and
(data["action"] == "process_output" or
data["action"] == "log" and data["level"] not in ["error", "critical"])):
return
if "test" in data:
data = data.copy()
data["test"] = "%s: %s" % (self.product, data["test"])
return self.inner(data)
def test_settings():
return {
"include": "_test",
"manifest-update": "",
"no-capture-stdio": ""
}
def read_config():
parser = ConfigParser.ConfigParser()
parser.read("test.cfg")
rv = {"general":{},
"products":{}}
rv["general"].update(dict(parser.items("general")))
# This only allows one product per whatever for now
for product in parser.sections():
if product != "general":
dest = rv["products"][product] = {}
for key, value in parser.items(product):
rv["products"][product][key] = value
return rv
def run_tests(product, kwargs):
kwargs["test_paths"]["/_test/"] = {"tests_path": os.path.join(here, "testdata"),
"metadata_path": os.path.join(here, "metadata")}
wptrunner.run_tests(**kwargs)
def settings_to_argv(settings):
rv = []
for name, value in settings.iteritems():
key = "--%s" % name
if not value:
rv.append(key)
elif isinstance(value, list):
for item in value:
rv.extend([key, item])
else:
rv.extend([key, value])
return rv
def set_from_args(settings, args):
if args.test:
settings["include"] = args.test
if args.tags:
settings["tags"] = args.tags
def run(config, args):
logger = structuredlog.StructuredLogger("web-platform-tests")
logger.add_handler(ResultHandler(logger=logger, verbose=args.verbose))
setup_wptrunner_logging(logger)
parser = wptcommandline.create_parser()
logger.suite_start(tests=[])
for product, product_settings in config["products"].iteritems():
if args.product and product not in args.product:
continue
settings = test_settings()
settings.update(config["general"])
settings.update(product_settings)
settings["product"] = product
set_from_args(settings, args)
kwargs = vars(parser.parse_args(settings_to_argv(settings)))
wptcommandline.check_args(kwargs)
logger.send_message("wptrunner-test", "set-product", product)
run_tests(product, kwargs)
logger.send_message("wptrunner-test", "set-product", None)
logger.suite_end()
def get_parser():
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbose", action="store_true", default=False,
help="verbose log output")
parser.add_argument("--product", action="append",
help="Specific product to include in test run")
parser.add_argument("--pdb", action="store_true",
help="Invoke pdb on uncaught exception")
parser.add_argument("--tag", action="append", dest="tags",
help="tags to select tests")
parser.add_argument("test", nargs="*",
help="Specific tests to include in test run")
return parser
def main():
config = read_config()
args = get_parser().parse_args()
try:
run(config, args)
except Exception:
if args.pdb:
import pdb, traceback
print traceback.format_exc()
pdb.post_mortem()
else:
raise
if __name__ == "__main__":
main()

4
testing/web-platform/harness/test/testdata/reftest/green-ref.html vendored
View file

@ -1,4 +0,0 @@
<link rel=match href=green.html>
<style>
:root {background-color:green}
</style>

3
testing/web-platform/harness/test/testdata/reftest/green.html vendored
View file

@ -1,3 +0,0 @@
<style>
:root {background-color:green}
</style>

3
testing/web-platform/harness/test/testdata/reftest/red.html vendored
View file

@ -1,3 +0,0 @@
<style>
:root {background-color:red}
</style>

9
testing/web-platform/harness/test/testdata/reftest/reftest.https.html vendored
View file

@ -1,9 +0,0 @@
<link rel=match href=green.html>
<style>
:root {background-color:red}
</style>
<script>
if (window.location.protocol === "https:") {
document.documentElement.style.backgroundColor = "green";
}
</script>

5
testing/web-platform/harness/test/testdata/reftest/reftest_and_fail.html vendored
View file

@ -1,5 +0,0 @@
<title>Reftest chain that should fail</title>
<link rel=match href=reftest_and_fail_0-ref.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_and_fail_0-ref.html vendored
View file

@ -1,5 +0,0 @@
<title>Reftest chain that should fail</title>
<link rel=match href=red.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_cycle.html vendored
View file

@ -1,5 +0,0 @@
<title>Reftest with cycle, all match</title>
<link rel=match href=reftest_cycle_0-ref.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_cycle_0-ref.html vendored
View file

@ -1,5 +0,0 @@
<title>OR match that should pass</title>
<link rel=match href=reftest_cycle_1-ref.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_cycle_1-ref.html vendored
View file

@ -1,5 +0,0 @@
<title>Reftest with cycle, all match</title>
<link rel=match href=reftest_cycle.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_cycle_fail.html vendored
View file

@ -1,5 +0,0 @@
<title>Reftest with cycle, fails</title>
<link rel=match href=reftest_cycle_fail_0-ref.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_cycle_fail_0-ref.html vendored
View file

@ -1,5 +0,0 @@
<title>Reftest with cycle, fails</title>
<link rel=mismatch href=reftest_cycle_fail.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_match.html vendored
View file

@ -1,5 +0,0 @@
<title>rel=match that should pass</title>
<link rel=match href=green.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_match_fail.html vendored
View file

@ -1,5 +0,0 @@
<title>rel=match that should fail</title>
<link rel=match href=red.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_mismatch.html vendored
View file

@ -1,5 +0,0 @@
<title>rel=mismatch that should pass</title>
<link rel=mismatch href=red.html>
<style>
:root {background-color:green}
</style>

5
testing/web-platform/harness/test/testdata/reftest/reftest_mismatch_fail.html vendored
View file

@ -1,5 +0,0 @@
<title>rel=mismatch that should fail</title>
<link rel=mismatch href=green.html>
<style>
:root {background-color:green}
</style>

6
testing/web-platform/harness/test/testdata/reftest/reftest_or_0.html vendored
View file

@ -1,6 +0,0 @@
<title>OR match that should pass</title>
<link rel=match href=red.html>
<link rel=match href=green.html>
<style>
:root {background-color:green}
</style>

6
testing/web-platform/harness/test/testdata/reftest/reftest_ref_timeout-ref.html vendored
View file

@ -1,6 +0,0 @@
<html class="reftest-wait">
<title>rel=match that should time out in the ref</title>
<link rel=match href=reftest_ref_timeout-ref.html>
<style>
:root {background-color:green}
</style>

6
testing/web-platform/harness/test/testdata/reftest/reftest_ref_timeout.html vendored
View file

@ -1,6 +0,0 @@
<html>
<title>rel=match that should time out in the ref</title>
<link rel=match href=reftest_ref_timeout-ref.html>
<style>
:root {background-color:green}
</style>

6
testing/web-platform/harness/test/testdata/reftest/reftest_timeout.html vendored
View file

@ -1,6 +0,0 @@
<html class="reftest-wait">
<title>rel=match that should timeout</title>
<link rel=match href=green.html>
<style>
:root {background-color:green}
</style>

11
testing/web-platform/harness/test/testdata/reftest/reftest_wait_0.html vendored
View file

@ -1,11 +0,0 @@
<title>rel=match that should fail</title>
<link rel=match href=red.html>
<style>
:root {background-color:red}
</style>
<body class="reftest-wait">
<script>
setTimeout(function() {
document.documentElement.style.backgroundColor = "green";
body.className = "";
}, 2000);

10
testing/web-platform/harness/test/testdata/testharness/firefox/subdir/test_pref_inherit.html vendored
View file

@ -1,10 +0,0 @@
<!doctype html>
<title>Example pref test</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
<script>
test(function() {
assert_equals(getComputedStyle(document.body).color, "rgb(255, 0, 0)");
}, "Test that pref was set");
</script>

10
testing/web-platform/harness/test/testdata/testharness/firefox/subdir/test_pref_reset.html vendored
View file

@ -1,10 +0,0 @@
<!doctype html>
<title>Example pref test</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
<script>
test(function() {
assert_equals(getComputedStyle(document.body).color, "rgb(0, 0, 0)");
}, "Test that pref was reset");
</script>

10
testing/web-platform/harness/test/testdata/testharness/firefox/test_pref_dir.html vendored
View file

@ -1,10 +0,0 @@
<!doctype html>
<title>Example pref test</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<p>Test requires the pref browser.display.foreground_color to be set to #FF0000</p>
<script>
test(function() {
assert_equals(getComputedStyle(document.body).color, "rgb(255, 0, 0)");
}, "Test that pref was set");
</script>

10
testing/web-platform/harness/test/testdata/testharness/firefox/test_pref_set.html vendored
View file

@ -1,10 +0,0 @@
<!doctype html>
<title>Example pref test</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<p>Test requires the pref browser.display.foreground_color to be set to #00FF00</p>
<script>
test(function() {
assert_equals(getComputedStyle(document.body).color, "rgb(0, 255, 0)");
}, "Test that pref was set");
</script>

9
testing/web-platform/harness/test/testdata/testharness/subdir/testharness_1.html vendored
View file

@ -1,9 +0,0 @@
<!doctype html>
<title>Test should be enabled</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>
test(function() {
assert_true(true);
}, "Test that should pass");
</script>

10
testing/web-platform/harness/test/testdata/testharness/testharness.https.html vendored
View file

@ -1,10 +0,0 @@
<!doctype html>
<title>Example https test</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>
test(function() {
assert_equals(window.location.protocol, "https:");
}, "Test that file was loaded with the correct protocol");
</script>

9
testing/web-platform/harness/test/testdata/testharness/testharness_0.html vendored
View file

@ -1,9 +0,0 @@
<!doctype html>
<title>Test should be disabled</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>
test(function() {
assert_true(false);
}, "Test that should fail");
</script>

7
testing/web-platform/harness/test/testdata/testharness/testharness_error.html vendored
View file

@ -1,7 +0,0 @@
<!doctype html>
<title>testharness.js test that should error</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>
undefined_function()
</script>

9
testing/web-platform/harness/test/testdata/testharness/testharness_long_timeout.html vendored
View file

@ -1,9 +0,0 @@
<!doctype html>
<title>testharness.js test with long timeout</title>
<meta name=timeout content=long>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
<script>
var t = async_test("Long timeout test");
setTimeout(t.step_func_done(function() {assert_true(true)}), 15*1000);
</script>

6
testing/web-platform/harness/test/testdata/testharness/testharness_timeout.html vendored
View file

@ -1,6 +0,0 @@
<!doctype html>
<title>Simple testharness.js usage</title>
<script src="/resources/testharness.js"></script>
<script src="/resources/testharnessreport.js"></script>
// This file should time out, obviously

11
testing/web-platform/harness/wptrunner.default.ini
View file

@ -1,11 +0,0 @@
[products]
[web-platform-tests]
remote_url = https://github.com/w3c/web-platform-tests.git
branch = master
sync_path = %(pwd)s/sync
[manifest:default]
tests = %(pwd)s/tests
metadata = %(pwd)s/meta
url_base = /

3
testing/web-platform/harness/wptrunner/__init__.py
View file

@ -1,3 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

33
testing/web-platform/harness/wptrunner/browsers/__init__.py
View file

@ -1,33 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
"""Subpackage where each product is defined. Each product is created by adding a
a .py file containing a __wptrunner__ variable in the global scope. This must be
a dictionary with the fields
"product": Name of the product, assumed to be unique.
"browser": String indicating the Browser implementation used to launch that
product.
"executor": Dictionary with keys as supported test types and values as the name
of the Executor implemantation that will be used to run that test
type.
"browser_kwargs": String naming function that takes product, binary,
prefs_root and the wptrunner.run_tests kwargs dict as arguments
and returns a dictionary of kwargs to use when creating the
Browser class.
"executor_kwargs": String naming a function that takes http server url and
timeout multiplier and returns kwargs to use when creating
the executor class.
"env_options": String naming a funtion of no arguments that returns the
arguments passed to the TestEnvironment.
All classes and functions named in the above dict must be imported into the
module global scope.
"""
product_list = ["b2g",
"chrome",
"firefox",
"servo",
"servodriver"]

243
testing/web-platform/harness/wptrunner/browsers/b2g.py
View file

@ -1,243 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import tempfile
import shutil
import subprocess
import fxos_appgen
import gaiatest
import mozdevice
import moznetwork
import mozrunner
from marionette import expected
from marionette.by import By
from marionette.wait import Wait
from mozprofile import FirefoxProfile, Preferences
from .base import get_free_port, BrowserError, Browser, ExecutorBrowser
from ..executors.executormarionette import MarionetteTestharnessExecutor
from ..hosts import HostsFile, HostsLine
from ..environment import hostnames
here = os.path.split(__file__)[0]
__wptrunner__ = {"product": "b2g",
"check_args": "check_args",
"browser": "B2GBrowser",
"executor": {"testharness": "B2GMarionetteTestharnessExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options"}
def check_args(**kwargs):
pass
def browser_kwargs(test_environment, **kwargs):
return {"prefs_root": kwargs["prefs_root"],
"no_backup": kwargs.get("b2g_no_backup", False)}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
timeout_multiplier = kwargs["timeout_multiplier"]
if timeout_multiplier is None:
timeout_multiplier = 2
executor_kwargs = {"server_config": server_config,
"timeout_multiplier": timeout_multiplier,
"close_after_done": False}
if test_type == "reftest":
executor_kwargs["cache_manager"] = cache_manager
return executor_kwargs
def env_options():
return {"host": "web-platform.test",
"bind_hostname": "false",
"test_server_port": False}
class B2GBrowser(Browser):
used_ports = set()
init_timeout = 180
def __init__(self, logger, prefs_root, no_backup=False):
Browser.__init__(self, logger)
logger.info("Waiting for device")
subprocess.call(["adb", "wait-for-device"])
self.device = mozdevice.DeviceManagerADB()
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
self.cert_test_app = None
self.runner = None
self.prefs_root = prefs_root
self.no_backup = no_backup
self.backup_path = None
self.backup_paths = []
self.backup_dirs = []
def setup(self):
self.logger.info("Running B2G setup")
self.backup_path = tempfile.mkdtemp()
self.logger.debug("Backing up device to %s" % (self.backup_path,))
if not self.no_backup:
self.backup_dirs = [("/data/local", os.path.join(self.backup_path, "local")),
("/data/b2g/mozilla", os.path.join(self.backup_path, "profile"))]
self.backup_paths = [("/system/etc/hosts", os.path.join(self.backup_path, "hosts"))]
for remote, local in self.backup_dirs:
self.device.getDirectory(remote, local)
for remote, local in self.backup_paths:
self.device.getFile(remote, local)
self.setup_hosts()
def start(self):
profile = FirefoxProfile()
profile.set_preferences({"dom.disable_open_during_load": False,
"marionette.defaultPrefs.enabled": True})
self.logger.debug("Creating device runner")
self.runner = mozrunner.B2GDeviceRunner(profile=profile)
self.logger.debug("Starting device runner")
self.runner.start()
self.logger.debug("Device runner started")
def setup_hosts(self):
host_ip = moznetwork.get_ip()
temp_dir = tempfile.mkdtemp()
hosts_path = os.path.join(temp_dir, "hosts")
remote_path = "/system/etc/hosts"
try:
self.device.getFile("/system/etc/hosts", hosts_path)
with open(hosts_path) as f:
hosts_file = HostsFile.from_file(f)
for canonical_hostname in hostnames:
hosts_file.set_host(HostsLine(host_ip, canonical_hostname))
with open(hosts_path, "w") as f:
hosts_file.to_file(f)
self.logger.info("Installing hosts file")
self.device.remount()
self.device.removeFile(remote_path)
self.device.pushFile(hosts_path, remote_path)
finally:
os.unlink(hosts_path)
os.rmdir(temp_dir)
def load_prefs(self):
prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
if os.path.exists(prefs_path):
preferences = Preferences.read_prefs(prefs_path)
else:
self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
preferences = []
return preferences
def stop(self):
pass
def on_output(self):
raise NotImplementedError
def cleanup(self):
self.logger.debug("Running browser cleanup steps")
self.device.remount()
for remote, local in self.backup_dirs:
self.device.removeDir(remote)
self.device.pushDir(local, remote)
for remote, local in self.backup_paths:
self.device.removeFile(remote)
self.device.pushFile(local, remote)
shutil.rmtree(self.backup_path)
self.device.reboot(wait=True)
def pid(self):
return None
def is_alive(self):
return True
def executor_browser(self):
return B2GExecutorBrowser, {"marionette_port": self.marionette_port}
class B2GExecutorBrowser(ExecutorBrowser):
# The following methods are called from a different process
def __init__(self, *args, **kwargs):
ExecutorBrowser.__init__(self, *args, **kwargs)
import sys, subprocess
self.device = mozdevice.ADBB2G()
self.device.forward("tcp:%s" % self.marionette_port,
"tcp:2828")
self.executor = None
self.marionette = None
self.gaia_device = None
self.gaia_apps = None
def after_connect(self, executor):
self.executor = executor
self.marionette = executor.marionette
self.executor.logger.debug("Running browser.after_connect steps")
self.gaia_apps = gaiatest.GaiaApps(marionette=executor.marionette)
self.executor.logger.debug("Waiting for homescreen to load")
# Moved out of gaia_test temporarily
self.executor.logger.info("Waiting for B2G to be ready")
self.wait_for_homescreen(timeout=60)
self.install_cert_app()
self.use_cert_app()
def install_cert_app(self):
"""Install the container app used to run the tests"""
if fxos_appgen.is_installed("CertTest App"):
self.executor.logger.info("CertTest App is already installed")
return
self.executor.logger.info("Installing CertTest App")
app_path = os.path.join(here, "b2g_setup", "certtest_app.zip")
fxos_appgen.install_app("CertTest App", app_path, marionette=self.marionette)
self.executor.logger.debug("Install complete")
def use_cert_app(self):
"""Start the app used to run the tests"""
self.executor.logger.info("Homescreen loaded")
self.gaia_apps.launch("CertTest App")
def wait_for_homescreen(self, timeout):
self.executor.logger.info("Waiting for home screen to load")
Wait(self.marionette, timeout).until(expected.element_present(
By.CSS_SELECTOR, '#homescreen[loading-state=false]'))
class B2GMarionetteTestharnessExecutor(MarionetteTestharnessExecutor):
def after_connect(self):
self.browser.after_connect(self)
MarionetteTestharnessExecutor.after_connect(self)

BIN
testing/web-platform/harness/wptrunner/browsers/b2g_setup/certtest_app.zip
View file

Binary file not shown.

160
testing/web-platform/harness/wptrunner/browsers/base.py
View file

@ -1,160 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import platform
import socket
from abc import ABCMeta, abstractmethod
from ..wptcommandline import require_arg
here = os.path.split(__file__)[0]
def cmd_arg(name, value=None):
prefix = "-" if platform.system() == "Windows" else "--"
rv = prefix + name
if value is not None:
rv += "=" + value
return rv
def get_free_port(start_port, exclude=None):
"""Get the first port number after start_port (inclusive) that is
not currently bound.
:param start_port: Integer port number at which to start testing.
:param exclude: Set of port numbers to skip"""
port = start_port
while True:
if exclude and port in exclude:
port += 1
continue
s = socket.socket()
try:
s.bind(("127.0.0.1", port))
except socket.error:
port += 1
else:
return port
finally:
s.close()
def browser_command(binary, args, debug_info):
if debug_info:
if debug_info.requiresEscapedArgs:
args = [item.replace("&", "\\&") for item in args]
debug_args = [debug_info.path] + debug_info.args
else:
debug_args = []
command = [binary] + args
return debug_args, command
class BrowserError(Exception):
pass
class Browser(object):
__metaclass__ = ABCMeta
process_cls = None
init_timeout = 30
def __init__(self, logger):
"""Abstract class serving as the basis for Browser implementations.
The Browser is used in the TestRunnerManager to start and stop the browser
process, and to check the state of that process. This class also acts as a
context manager, enabling it to do browser-specific setup at the start of
the testrun and cleanup after the run is complete.
:param logger: Structured logger to use for output.
"""
self.logger = logger
def __enter__(self):
self.setup()
return self
def __exit__(self, *args, **kwargs):
self.cleanup()
def setup(self):
"""Used for browser-specific setup that happens at the start of a test run"""
pass
@abstractmethod
def start(self):
"""Launch the browser object and get it into a state where is is ready to run tests"""
pass
@abstractmethod
def stop(self):
"""Stop the running browser process."""
pass
@abstractmethod
def pid(self):
"""pid of the browser process or None if there is no pid"""
pass
@abstractmethod
def is_alive(self):
"""Boolean indicating whether the browser process is still running"""
pass
def setup_ssl(self, hosts):
"""Return a certificate to use for tests requiring ssl that will be trusted by the browser"""
raise NotImplementedError("ssl testing not supported")
def cleanup(self):
"""Browser-specific cleanup that is run after the testrun is finished"""
pass
def executor_browser(self):
"""Returns the ExecutorBrowser subclass for this Browser subclass and the keyword arguments
with which it should be instantiated"""
return ExecutorBrowser, {}
def log_crash(self, process, test):
"""Return a list of dictionaries containing information about crashes that happend
in the browser, or an empty list if no crashes occurred"""
self.logger.crash(process, test)
class NullBrowser(Browser):
def start(self):
"""No-op browser to use in scenarios where the TestRunnerManager shouldn't
actually own the browser process (e.g. Servo where we start one browser
per test)"""
pass
def stop(self):
pass
def pid(self):
return None
def is_alive(self):
return True
def on_output(self, line):
raise NotImplementedError
class ExecutorBrowser(object):
def __init__(self, **kwargs):
"""View of the Browser used by the Executor object.
This is needed because the Executor runs in a child process and
we can't ship Browser instances between processes on Windows.
Typically this will have a few product-specific properties set,
but in some cases it may have more elaborate methods for setting
up the browser from the runner process.
"""
for k, v in kwargs.iteritems():
setattr(self, k, v)

81
testing/web-platform/harness/wptrunner/browsers/chrome.py
View file

@ -1,81 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from .base import Browser, ExecutorBrowser, require_arg
from ..webdriver_server import ChromeDriverServer
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executorselenium import (SeleniumTestharnessExecutor,
SeleniumRefTestExecutor)
__wptrunner__ = {"product": "chrome",
"check_args": "check_args",
"browser": "ChromeBrowser",
"executor": {"testharness": "SeleniumTestharnessExecutor",
"reftest": "SeleniumRefTestExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options"}
def check_args(**kwargs):
require_arg(kwargs, "webdriver_binary")
def browser_kwargs(**kwargs):
return {"binary": kwargs["binary"],
"webdriver_binary": kwargs["webdriver_binary"]}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
from selenium.webdriver import DesiredCapabilities
executor_kwargs = base_executor_kwargs(test_type, server_config,
cache_manager, **kwargs)
executor_kwargs["close_after_done"] = True
executor_kwargs["capabilities"] = dict(DesiredCapabilities.CHROME.items())
if kwargs["binary"] is not None:
executor_kwargs["capabilities"]["chromeOptions"] = {"binary": kwargs["binary"]}
return executor_kwargs
def env_options():
return {"host": "web-platform.test",
"bind_hostname": "true"}
class ChromeBrowser(Browser):
"""Chrome is backed by chromedriver, which is supplied through
``wptrunner.webdriver.ChromeDriverServer``.
"""
def __init__(self, logger, binary, webdriver_binary="chromedriver"):
"""Creates a new representation of Chrome. The `binary` argument gives
the browser binary to use for testing."""
Browser.__init__(self, logger)
self.binary = binary
self.server = ChromeDriverServer(self.logger, binary=webdriver_binary)
def start(self):
self.server.start(block=False)
def stop(self):
self.server.stop()
def pid(self):
return self.server.pid
def is_alive(self):
# TODO(ato): This only indicates the driver is alive,
# and doesn't say anything about whether a browser session
# is active.
return self.server.is_alive()
def cleanup(self):
self.stop()
def executor_browser(self):
return ExecutorBrowser, {"webdriver_url": self.server.url}

269
testing/web-platform/harness/wptrunner/browsers/firefox.py
View file

@ -1,269 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import platform
import subprocess
import sys
import mozinfo
from mozprocess import ProcessHandler
from mozprofile import FirefoxProfile, Preferences
from mozprofile.permissions import ServerLocations
from mozrunner import FirefoxRunner
from mozrunner.utils import get_stack_fixer_function
from mozcrash import mozcrash
from .base import (get_free_port,
Browser,
ExecutorBrowser,
require_arg,
cmd_arg,
browser_command)
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executormarionette import (MarionetteTestharnessExecutor,
MarionetteRefTestExecutor,
MarionetteWdspecExecutor)
from ..environment import hostnames
here = os.path.join(os.path.split(__file__)[0])
__wptrunner__ = {"product": "firefox",
"check_args": "check_args",
"browser": "FirefoxBrowser",
"executor": {"testharness": "MarionetteTestharnessExecutor",
"reftest": "MarionetteRefTestExecutor",
"wdspec": "MarionetteWdspecExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options",
"run_info_extras": "run_info_extras",
"update_properties": "update_properties"}
def check_args(**kwargs):
require_arg(kwargs, "binary")
if kwargs["ssl_type"] != "none":
require_arg(kwargs, "certutil_binary")
def browser_kwargs(**kwargs):
return {"binary": kwargs["binary"],
"prefs_root": kwargs["prefs_root"],
"debug_info": kwargs["debug_info"],
"symbols_path": kwargs["symbols_path"],
"stackwalk_binary": kwargs["stackwalk_binary"],
"certutil_binary": kwargs["certutil_binary"],
"ca_certificate_path": kwargs["ssl_env"].ca_cert_path(),
"e10s": kwargs["gecko_e10s"],
"stackfix_dir": kwargs["stackfix_dir"]}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
executor_kwargs = base_executor_kwargs(test_type, server_config,
cache_manager, **kwargs)
executor_kwargs["close_after_done"] = test_type != "reftest"
if kwargs["timeout_multiplier"] is None:
if test_type == "reftest":
if run_info_data["debug"] or run_info_data.get("asan"):
executor_kwargs["timeout_multiplier"] = 4
else:
executor_kwargs["timeout_multiplier"] = 2
elif run_info_data["debug"] or run_info_data.get("asan"):
executor_kwargs["timeout_multiplier"] = 3
if test_type == "wdspec":
executor_kwargs["webdriver_binary"] = kwargs.get("webdriver_binary")
return executor_kwargs
def env_options():
return {"host": "127.0.0.1",
"external_host": "web-platform.test",
"bind_hostname": "false",
"certificate_domain": "web-platform.test",
"supports_debugger": True}
def run_info_extras(**kwargs):
return {"e10s": kwargs["gecko_e10s"]}
def update_properties():
return ["debug", "e10s", "os", "version", "processor", "bits"], {"debug", "e10s"}
class FirefoxBrowser(Browser):
used_ports = set()
init_timeout = 60
def __init__(self, logger, binary, prefs_root, debug_info=None,
symbols_path=None, stackwalk_binary=None, certutil_binary=None,
ca_certificate_path=None, e10s=False, stackfix_dir=None):
Browser.__init__(self, logger)
self.binary = binary
self.prefs_root = prefs_root
self.marionette_port = None
self.runner = None
self.debug_info = debug_info
self.profile = None
self.symbols_path = symbols_path
self.stackwalk_binary = stackwalk_binary
self.ca_certificate_path = ca_certificate_path
self.certutil_binary = certutil_binary
self.e10s = e10s
if self.symbols_path and stackfix_dir:
self.stack_fixer = get_stack_fixer_function(stackfix_dir,
self.symbols_path)
else:
self.stack_fixer = None
def start(self):
self.marionette_port = get_free_port(2828, exclude=self.used_ports)
self.used_ports.add(self.marionette_port)
env = os.environ.copy()
env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
locations = ServerLocations(filename=os.path.join(here, "server-locations.txt"))
preferences = self.load_prefs()
self.profile = FirefoxProfile(locations=locations,
preferences=preferences)
self.profile.set_preferences({"marionette.defaultPrefs.enabled": True,
"marionette.defaultPrefs.port": self.marionette_port,
"dom.disable_open_during_load": False,
"network.dns.localDomains": ",".join(hostnames),
"network.proxy.type": 0,
"places.history.enabled": False})
if self.e10s:
self.profile.set_preferences({"browser.tabs.remote.autostart": True})
if self.ca_certificate_path is not None:
self.setup_ssl()
debug_args, cmd = browser_command(self.binary, [cmd_arg("marionette"), "about:blank"],
self.debug_info)
self.runner = FirefoxRunner(profile=self.profile,
binary=cmd[0],
cmdargs=cmd[1:],
env=env,
process_class=ProcessHandler,
process_args={"processOutputLine": [self.on_output]})
self.logger.debug("Starting Firefox")
self.runner.start(debug_args=debug_args, interactive=self.debug_info and self.debug_info.interactive)
self.logger.debug("Firefox Started")
def load_prefs(self):
prefs_path = os.path.join(self.prefs_root, "prefs_general.js")
if os.path.exists(prefs_path):
preferences = Preferences.read_prefs(prefs_path)
else:
self.logger.warning("Failed to find base prefs file in %s" % prefs_path)
preferences = []
return preferences
def stop(self):
self.logger.debug("Stopping browser")
if self.runner is not None:
try:
self.runner.stop()
except OSError:
# This can happen on Windows if the process is already dead
pass
def pid(self):
if self.runner.process_handler is None:
return None
try:
return self.runner.process_handler.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the firefox process to the log"""
data = line.decode("utf8", "replace")
if self.stack_fixer:
data = self.stack_fixer(data)
self.logger.process_output(self.pid(),
data,
command=" ".join(self.runner.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
def executor_browser(self):
assert self.marionette_port is not None
return ExecutorBrowser, {"marionette_port": self.marionette_port}
def log_crash(self, process, test):
dump_dir = os.path.join(self.profile.profile, "minidumps")
mozcrash.log_crashes(self.logger,
dump_dir,
symbols_path=self.symbols_path,
stackwalk_binary=self.stackwalk_binary,
process=process,
test=test)
def setup_ssl(self):
"""Create a certificate database to use in the test profile. This is configured
to trust the CA Certificate that has signed the web-platform.test server
certificate."""
self.logger.info("Setting up ssl")
# Make sure the certutil libraries from the source tree are loaded when using a
# local copy of certutil
# TODO: Maybe only set this if certutil won't launch?
env = os.environ.copy()
certutil_dir = os.path.dirname(self.binary)
if mozinfo.isMac:
env_var = "DYLD_LIBRARY_PATH"
elif mozinfo.isUnix:
env_var = "LD_LIBRARY_PATH"
else:
env_var = "PATH"
env[env_var] = (os.path.pathsep.join([certutil_dir, env[env_var]])
if env_var in env else certutil_dir).encode(
sys.getfilesystemencoding() or 'utf-8', 'replace')
def certutil(*args):
cmd = [self.certutil_binary] + list(args)
self.logger.process_output("certutil",
subprocess.check_output(cmd,
env=env,
stderr=subprocess.STDOUT),
" ".join(cmd))
pw_path = os.path.join(self.profile.profile, ".crtdbpw")
with open(pw_path, "w") as f:
# Use empty password for certificate db
f.write("\n")
cert_db_path = self.profile.profile
# Create a new certificate db
certutil("-N", "-d", cert_db_path, "-f", pw_path)
# Add the CA certificate to the database and mark as trusted to issue server certs
certutil("-A", "-d", cert_db_path, "-f", pw_path, "-t", "CT,,",
"-n", "web-platform-tests", "-i", self.ca_certificate_path)
# List all certs in the database
certutil("-L", "-d", cert_db_path)

38
testing/web-platform/harness/wptrunner/browsers/server-locations.txt
View file

@ -1,38 +0,0 @@
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# See /build/pgo/server-locations.txt for documentation on the format
http://localhost:8000 primary
http://web-platform.test:8000
http://www.web-platform.test:8000
http://www1.web-platform.test:8000
http://www2.web-platform.test:8000
http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8000
http://xn--lve-6lad.web-platform.test:8000
http://web-platform.test:8001
http://www.web-platform.test:8001
http://www1.web-platform.test:8001
http://www2.web-platform.test:8001
http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8001
http://xn--lve-6lad.web-platform.test:8001
https://web-platform.test:8443
https://www.web-platform.test:8443
https://www1.web-platform.test:8443
https://www2.web-platform.test:8443
https://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8443
https://xn--lve-6lad.web-platform.test:8443
# These are actually ws servers, but until mozprofile is
# fixed we have to pretend that they are http servers
http://web-platform.test:8888
http://www.web-platform.test:8888
http://www1.web-platform.test:8888
http://www2.web-platform.test:8888
http://xn--n8j6ds53lwwkrqhv28a.web-platform.test:8888
http://xn--lve-6lad.web-platform.test:8888

80
testing/web-platform/harness/wptrunner/browsers/servo.py
View file

@ -1,80 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
from .base import NullBrowser, ExecutorBrowser, require_arg
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executorservo import ServoTestharnessExecutor, ServoRefTestExecutor
here = os.path.join(os.path.split(__file__)[0])
__wptrunner__ = {"product": "servo",
"check_args": "check_args",
"browser": "ServoBrowser",
"executor": {"testharness": "ServoTestharnessExecutor",
"reftest": "ServoRefTestExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options",
"run_info_extras": "run_info_extras",
"update_properties": "update_properties"}
def check_args(**kwargs):
require_arg(kwargs, "binary")
def browser_kwargs(**kwargs):
return {"binary": kwargs["binary"],
"debug_info": kwargs["debug_info"],
"binary_args": kwargs["binary_args"],
"user_stylesheets": kwargs.get("user_stylesheets"),
"render_backend": kwargs.get("servo_backend")}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data,
**kwargs):
rv = base_executor_kwargs(test_type, server_config,
cache_manager, **kwargs)
rv["pause_after_test"] = kwargs["pause_after_test"]
return rv
def env_options():
return {"host": "127.0.0.1",
"external_host": "web-platform.test",
"bind_hostname": "true",
"testharnessreport": "testharnessreport-servo.js",
"supports_debugger": True}
def run_info_extras(**kwargs):
return {"backend": kwargs["servo_backend"]}
def update_properties():
return ["debug", "os", "version", "processor", "bits", "backend"], None
def render_arg(render_backend):
return {"cpu": "--cpu", "webrender": "-w"}[render_backend]
class ServoBrowser(NullBrowser):
def __init__(self, logger, binary, debug_info=None, binary_args=None,
user_stylesheets=None, render_backend="cpu"):
NullBrowser.__init__(self, logger)
self.binary = binary
self.debug_info = debug_info
self.binary_args = binary_args or []
self.user_stylesheets = user_stylesheets or []
self.render_backend = render_backend
def executor_browser(self):
return ExecutorBrowser, {"binary": self.binary,
"debug_info": self.debug_info,
"binary_args": self.binary_args,
"user_stylesheets": self.user_stylesheets,
"render_backend": self.render_backend}

162
testing/web-platform/harness/wptrunner/browsers/servodriver.py
View file

@ -1,162 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import subprocess
import tempfile
from mozprocess import ProcessHandler
from .base import Browser, require_arg, get_free_port, browser_command, ExecutorBrowser
from .servo import render_arg
from ..executors import executor_kwargs as base_executor_kwargs
from ..executors.executorservodriver import (ServoWebDriverTestharnessExecutor,
ServoWebDriverRefTestExecutor)
here = os.path.join(os.path.split(__file__)[0])
__wptrunner__ = {"product": "servodriver",
"check_args": "check_args",
"browser": "ServoWebDriverBrowser",
"executor": {"testharness": "ServoWebDriverTestharnessExecutor",
"reftest": "ServoWebDriverRefTestExecutor"},
"browser_kwargs": "browser_kwargs",
"executor_kwargs": "executor_kwargs",
"env_options": "env_options",
"run_info_extras": "run_info_extras",
"update_properties": "update_properties"}
hosts_text = """127.0.0.1 web-platform.test
127.0.0.1 www.web-platform.test
127.0.0.1 www1.web-platform.test
127.0.0.1 www2.web-platform.test
127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test
127.0.0.1 xn--lve-6lad.web-platform.test
"""
def check_args(**kwargs):
require_arg(kwargs, "binary")
def browser_kwargs(**kwargs):
return {"binary": kwargs["binary"],
"debug_info": kwargs["debug_info"],
"user_stylesheets": kwargs.get("user_stylesheets"),
"render_backend": kwargs.get("servo_backend")}
def executor_kwargs(test_type, server_config, cache_manager, run_info_data, **kwargs):
rv = base_executor_kwargs(test_type, server_config,
cache_manager, **kwargs)
return rv
def env_options():
return {"host": "127.0.0.1",
"external_host": "web-platform.test",
"bind_hostname": "true",
"testharnessreport": "testharnessreport-servodriver.js",
"supports_debugger": True}
def run_info_extras(**kwargs):
return {"backend": kwargs["servo_backend"]}
def update_properties():
return ["debug", "os", "version", "processor", "bits", "backend"], None
def make_hosts_file():
hosts_fd, hosts_path = tempfile.mkstemp()
with os.fdopen(hosts_fd, "w") as f:
f.write(hosts_text)
return hosts_path
class ServoWebDriverBrowser(Browser):
used_ports = set()
def __init__(self, logger, binary, debug_info=None, webdriver_host="127.0.0.1",
user_stylesheets=None, render_backend="cpu"):
Browser.__init__(self, logger)
self.binary = binary
self.webdriver_host = webdriver_host
self.webdriver_port = None
self.proc = None
self.debug_info = debug_info
self.hosts_path = make_hosts_file()
self.command = None
self.user_stylesheets = user_stylesheets if user_stylesheets else []
self.render_backend = render_backend
def start(self):
self.webdriver_port = get_free_port(4444, exclude=self.used_ports)
self.used_ports.add(self.webdriver_port)
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
debug_args, command = browser_command(self.binary,
[render_arg(self.render_backend), "--hard-fail",
"--webdriver", str(self.webdriver_port),
"about:blank"],
self.debug_info)
for stylesheet in self.user_stylesheets:
command += ["--user-stylesheet", stylesheet]
self.command = command
self.command = debug_args + self.command
if not self.debug_info or not self.debug_info.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
env=env,
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
self.logger.debug("Servo Started")
def stop(self):
self.logger.debug("Stopping browser")
if self.proc is not None:
try:
self.proc.kill()
except OSError:
# This can happen on Windows if the process is already dead
pass
def pid(self):
if self.proc is None:
return None
try:
return self.proc.pid
except AttributeError:
return None
def on_output(self, line):
"""Write a line of output from the process to the log"""
self.logger.process_output(self.pid(),
line.decode("utf8", "replace"),
command=" ".join(self.command))
def is_alive(self):
if self.runner:
return self.runner.is_running()
return False
def cleanup(self):
self.stop()
def executor_browser(self):
assert self.webdriver_port is not None
return ExecutorBrowser, {"webdriver_host": self.webdriver_host,
"webdriver_port": self.webdriver_port}

7
testing/web-platform/harness/wptrunner/config.json
View file

@ -1,7 +0,0 @@
{"host": "%(host)s",
"ports":{"http":[8000, 8001],
"https":[8443],
"ws":[8888]},
"check_subdomains":false,
"bind_hostname":%(bind_hostname)s,
"ssl":{}}

64
testing/web-platform/harness/wptrunner/config.py
View file

@ -1,64 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import ConfigParser
import os
import sys
from collections import OrderedDict
here = os.path.split(__file__)[0]
class ConfigDict(dict):
def __init__(self, base_path, *args, **kwargs):
self.base_path = base_path
dict.__init__(self, *args, **kwargs)
def get_path(self, key, default=None):
if key not in self:
return default
path = self[key]
os.path.expanduser(path)
return os.path.abspath(os.path.join(self.base_path, path))
def read(config_path):
config_path = os.path.abspath(config_path)
config_root = os.path.split(config_path)[0]
parser = ConfigParser.SafeConfigParser()
success = parser.read(config_path)
assert config_path in success, success
subns = {"pwd": os.path.abspath(os.path.curdir)}
rv = OrderedDict()
for section in parser.sections():
rv[section] = ConfigDict(config_root)
for key in parser.options(section):
rv[section][key] = parser.get(section, key, False, subns)
return rv
def path(argv=None):
if argv is None:
argv = []
path = None
for i, arg in enumerate(argv):
if arg == "--config":
if i + 1 < len(argv):
path = argv[i + 1]
elif arg.startswith("--config="):
path = arg.split("=", 1)[1]
if path is not None:
break
if path is None:
if os.path.exists("wptrunner.ini"):
path = os.path.abspath("wptrunner.ini")
else:
path = os.path.join(here, "..", "wptrunner.default.ini")
return os.path.abspath(path)
def load():
return read(path(sys.argv))

212
testing/web-platform/harness/wptrunner/environment.py
View file

@ -1,212 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import multiprocessing
import signal
import socket
import sys
import time
from mozlog import get_default_logger, handlers
from wptlogging import LogLevelRewriter
here = os.path.split(__file__)[0]
serve = None
sslutils = None
hostnames = ["web-platform.test",
"www.web-platform.test",
"www1.web-platform.test",
"www2.web-platform.test",
"xn--n8j6ds53lwwkrqhv28a.web-platform.test",
"xn--lve-6lad.web-platform.test"]
def do_delayed_imports(logger, test_paths):
global serve, sslutils
serve_root = serve_path(test_paths)
sys.path.insert(0, serve_root)
failed = []
try:
from tools.serve import serve
except ImportError:
failed.append("serve")
try:
import sslutils
except ImportError:
failed.append("sslutils")
if failed:
logger.critical(
"Failed to import %s. Ensure that tests path %s contains web-platform-tests" %
(", ".join(failed), serve_root))
sys.exit(1)
def serve_path(test_paths):
return test_paths["/"]["tests_path"]
def get_ssl_kwargs(**kwargs):
if kwargs["ssl_type"] == "openssl":
args = {"openssl_binary": kwargs["openssl_binary"]}
elif kwargs["ssl_type"] == "pregenerated":
args = {"host_key_path": kwargs["host_key_path"],
"host_cert_path": kwargs["host_cert_path"],
"ca_cert_path": kwargs["ca_cert_path"]}
else:
args = {}
return args
def ssl_env(logger, **kwargs):
ssl_env_cls = sslutils.environments[kwargs["ssl_type"]]
return ssl_env_cls(logger, **get_ssl_kwargs(**kwargs))
class TestEnvironmentError(Exception):
pass
class TestEnvironment(object):
def __init__(self, test_paths, ssl_env, pause_after_test, debug_info, options):
"""Context manager that owns the test environment i.e. the http and
websockets servers"""
self.test_paths = test_paths
self.ssl_env = ssl_env
self.server = None
self.config = None
self.external_config = None
self.pause_after_test = pause_after_test
self.test_server_port = options.pop("test_server_port", True)
self.debug_info = debug_info
self.options = options if options is not None else {}
self.cache_manager = multiprocessing.Manager()
self.stash = serve.stash.StashServer()
def __enter__(self):
self.stash.__enter__()
self.ssl_env.__enter__()
self.cache_manager.__enter__()
self.setup_server_logging()
self.config = self.load_config()
serve.set_computed_defaults(self.config)
self.external_config, self.servers = serve.start(self.config, self.ssl_env,
self.get_routes())
if self.options.get("supports_debugger") and self.debug_info and self.debug_info.interactive:
self.ignore_interrupts()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.process_interrupts()
for scheme, servers in self.servers.iteritems():
for port, server in servers:
server.kill()
self.cache_manager.__exit__(exc_type, exc_val, exc_tb)
self.ssl_env.__exit__(exc_type, exc_val, exc_tb)
self.stash.__exit__()
def ignore_interrupts(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
def process_interrupts(self):
signal.signal(signal.SIGINT, signal.SIG_DFL)
def load_config(self):
default_config_path = os.path.join(serve_path(self.test_paths), "config.default.json")
local_config_path = os.path.join(here, "config.json")
with open(default_config_path) as f:
default_config = json.load(f)
with open(local_config_path) as f:
data = f.read()
local_config = json.loads(data % self.options)
#TODO: allow non-default configuration for ssl
local_config["external_host"] = self.options.get("external_host", None)
local_config["ssl"]["encrypt_after_connect"] = self.options.get("encrypt_after_connect", False)
config = serve.merge_json(default_config, local_config)
config["doc_root"] = serve_path(self.test_paths)
if not self.ssl_env.ssl_enabled:
config["ports"]["https"] = [None]
host = self.options.get("certificate_domain", config["host"])
hosts = [host]
hosts.extend("%s.%s" % (item[0], host) for item in serve.get_subdomains(host).values())
key_file, certificate = self.ssl_env.host_cert_path(hosts)
config["key_file"] = key_file
config["certificate"] = certificate
return config
def setup_server_logging(self):
server_logger = get_default_logger(component="wptserve")
assert server_logger is not None
log_filter = handlers.LogLevelFilter(lambda x:x, "info")
# Downgrade errors to warnings for the server
log_filter = LogLevelRewriter(log_filter, ["error"], "warning")
server_logger.component_filter = log_filter
try:
#Set as the default logger for wptserve
serve.set_logger(server_logger)
serve.logger = server_logger
except Exception:
# This happens if logging has already been set up for wptserve
pass
def get_routes(self):
route_builder = serve.RoutesBuilder()
for path, format_args, content_type, route in [
("testharness_runner.html", {}, "text/html", "/testharness_runner.html"),
(self.options.get("testharnessreport", "testharnessreport.js"),
{"output": self.pause_after_test}, "text/javascript",
"/resources/testharnessreport.js")]:
path = os.path.normpath(os.path.join(here, path))
route_builder.add_static(path, format_args, content_type, route)
for url_base, paths in self.test_paths.iteritems():
if url_base == "/":
continue
route_builder.add_mount_point(url_base, paths["tests_path"])
if "/" not in self.test_paths:
del route_builder.mountpoint_routes["/"]
return route_builder.get_routes()
def ensure_started(self):
# Pause for a while to ensure that the server has a chance to start
time.sleep(2)
for scheme, servers in self.servers.iteritems():
for port, server in servers:
if self.test_server_port:
s = socket.socket()
try:
s.connect((self.config["host"], port))
except socket.error:
raise EnvironmentError(
"%s server on port %d failed to start" % (scheme, port))
finally:
s.close()
if not server.is_alive():
raise EnvironmentError("%s server on port %d failed to start" % (scheme, port))

8
testing/web-platform/harness/wptrunner/executors/__init__.py
View file

@ -1,8 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from base import (executor_kwargs,
testharness_result_converter,
reftest_result_converter,
TestExecutor)

329
testing/web-platform/harness/wptrunner/executors/base.py
View file

@ -1,329 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import hashlib
import json
import os
import traceback
import urlparse
from abc import ABCMeta, abstractmethod
from ..testrunner import Stop
here = os.path.split(__file__)[0]
def executor_kwargs(test_type, server_config, cache_manager, **kwargs):
timeout_multiplier = kwargs["timeout_multiplier"]
if timeout_multiplier is None:
timeout_multiplier = 1
executor_kwargs = {"server_config": server_config,
"timeout_multiplier": timeout_multiplier,
"debug_info": kwargs["debug_info"]}
if test_type == "reftest":
executor_kwargs["screenshot_cache"] = cache_manager.dict()
return executor_kwargs
def strip_server(url):
"""Remove the scheme and netloc from a url, leaving only the path and any query
or fragment.
url - the url to strip
e.g. http://example.org:8000/tests?id=1#2 becomes /tests?id=1#2"""
url_parts = list(urlparse.urlsplit(url))
url_parts[0] = ""
url_parts[1] = ""
return urlparse.urlunsplit(url_parts)
class TestharnessResultConverter(object):
harness_codes = {0: "OK",
1: "ERROR",
2: "TIMEOUT"}
test_codes = {0: "PASS",
1: "FAIL",
2: "TIMEOUT",
3: "NOTRUN"}
def __call__(self, test, result):
"""Convert a JSON result into a (TestResult, [SubtestResult]) tuple"""
result_url, status, message, stack, subtest_results = result
assert result_url == test.url, ("Got results from %s, expected %s" %
(result_url, test.url))
harness_result = test.result_cls(self.harness_codes[status], message)
return (harness_result,
[test.subtest_result_cls(name, self.test_codes[status], message, stack)
for name, status, message, stack in subtest_results])
testharness_result_converter = TestharnessResultConverter()
def reftest_result_converter(self, test, result):
return (test.result_cls(result["status"], result["message"],
extra=result.get("extra")), [])
def pytest_result_converter(self, test, data):
harness_data, subtest_data = data
if subtest_data is None:
subtest_data = []
harness_result = test.result_cls(*harness_data)
subtest_results = [test.subtest_result_cls(*item) for item in subtest_data]
return (harness_result, subtest_results)
class ExecutorException(Exception):
def __init__(self, status, message):
self.status = status
self.message = message
class TestExecutor(object):
__metaclass__ = ABCMeta
test_type = None
convert_result = None
def __init__(self, browser, server_config, timeout_multiplier=1,
debug_info=None):
"""Abstract Base class for object that actually executes the tests in a
specific browser. Typically there will be a different TestExecutor
subclass for each test type and method of executing tests.
:param browser: ExecutorBrowser instance providing properties of the
browser that will be tested.
:param server_config: Dictionary of wptserve server configuration of the
form stored in TestEnvironment.external_config
:param timeout_multiplier: Multiplier relative to base timeout to use
when setting test timeout.
"""
self.runner = None
self.browser = browser
self.server_config = server_config
self.timeout_multiplier = timeout_multiplier
self.debug_info = debug_info
self.last_environment = {"protocol": "http",
"prefs": {}}
self.protocol = None # This must be set in subclasses
@property
def logger(self):
"""StructuredLogger for this executor"""
if self.runner is not None:
return self.runner.logger
def setup(self, runner):
"""Run steps needed before tests can be started e.g. connecting to
browser instance
:param runner: TestRunner instance that is going to run the tests"""
self.runner = runner
if self.protocol is not None:
self.protocol.setup(runner)
def teardown(self):
"""Run cleanup steps after tests have finished"""
if self.protocol is not None:
self.protocol.teardown()
def run_test(self, test):
"""Run a particular test.
:param test: The test to run"""
if test.environment != self.last_environment:
self.on_environment_change(test.environment)
try:
result = self.do_test(test)
except Exception as e:
result = self.result_from_exception(test, e)
if result is Stop:
return result
# log result of parent test
if result[0].status == "ERROR":
self.logger.debug(result[0].message)
self.last_environment = test.environment
self.runner.send_message("test_ended", test, result)
def server_url(self, protocol):
return "%s://%s:%s" % (protocol,
self.server_config["host"],
self.server_config["ports"][protocol][0])
def test_url(self, test):
return urlparse.urljoin(self.server_url(test.environment["protocol"]), test.url)
@abstractmethod
def do_test(self, test):
"""Test-type and protocol specific implementation of running a
specific test.
:param test: The test to run."""
pass
def on_environment_change(self, new_environment):
pass
def result_from_exception(self, test, e):
if hasattr(e, "status") and e.status in test.result_cls.statuses:
status = e.status
else:
status = "ERROR"
message = unicode(getattr(e, "message", ""))
if message:
message += "\n"
message += traceback.format_exc(e)
return test.result_cls(status, message), []
class TestharnessExecutor(TestExecutor):
convert_result = testharness_result_converter
class RefTestExecutor(TestExecutor):
convert_result = reftest_result_converter
def __init__(self, browser, server_config, timeout_multiplier=1, screenshot_cache=None,
debug_info=None):
TestExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.screenshot_cache = screenshot_cache
class RefTestImplementation(object):
def __init__(self, executor):
self.timeout_multiplier = executor.timeout_multiplier
self.executor = executor
# Cache of url:(screenshot hash, screenshot). Typically the
# screenshot is None, but we set this value if a test fails
# and the screenshot was taken from the cache so that we may
# retrieve the screenshot from the cache directly in the future
self.screenshot_cache = self.executor.screenshot_cache
self.message = None
@property
def logger(self):
return self.executor.logger
def get_hash(self, test, viewport_size, dpi):
timeout = test.timeout * self.timeout_multiplier
key = (test.url, viewport_size, dpi)
if key not in self.screenshot_cache:
success, data = self.executor.screenshot(test, viewport_size, dpi)
if not success:
return False, data
screenshot = data
hash_value = hashlib.sha1(screenshot).hexdigest()
self.screenshot_cache[key] = (hash_value, None)
rv = (hash_value, screenshot)
else:
rv = self.screenshot_cache[key]
self.message.append("%s %s" % (test.url, rv[0]))
return True, rv
def is_pass(self, lhs_hash, rhs_hash, relation):
assert relation in ("==", "!=")
self.message.append("Testing %s %s %s" % (lhs_hash, relation, rhs_hash))
return ((relation == "==" and lhs_hash == rhs_hash) or
(relation == "!=" and lhs_hash != rhs_hash))
def run_test(self, test):
viewport_size = test.viewport_size
dpi = test.dpi
self.message = []
# Depth-first search of reference tree, with the goal
# of reachings a leaf node with only pass results
stack = list(((test, item[0]), item[1]) for item in reversed(test.references))
while stack:
hashes = [None, None]
screenshots = [None, None]
nodes, relation = stack.pop()
for i, node in enumerate(nodes):
success, data = self.get_hash(node, viewport_size, dpi)
if success is False:
return {"status": data[0], "message": data[1]}
hashes[i], screenshots[i] = data
if self.is_pass(hashes[0], hashes[1], relation):
if nodes[1].references:
stack.extend(list(((nodes[1], item[0]), item[1]) for item in reversed(nodes[1].references)))
else:
# We passed
return {"status":"PASS", "message": None}
# We failed, so construct a failure message
for i, (node, screenshot) in enumerate(zip(nodes, screenshots)):
if screenshot is None:
success, screenshot = self.retake_screenshot(node, viewport_size, dpi)
if success:
screenshots[i] = screenshot
log_data = [{"url": nodes[0].url, "screenshot": screenshots[0]}, relation,
{"url": nodes[1].url, "screenshot": screenshots[1]}]
return {"status": "FAIL",
"message": "\n".join(self.message),
"extra": {"reftest_screenshots": log_data}}
def retake_screenshot(self, node, viewport_size, dpi):
success, data = self.executor.screenshot(node, viewport_size, dpi)
if not success:
return False, data
key = (node.url, viewport_size, dpi)
hash_val, _ = self.screenshot_cache[key]
self.screenshot_cache[key] = hash_val, data
return True, data
class WdspecExecutor(TestExecutor):
convert_result = pytest_result_converter
class Protocol(object):
def __init__(self, executor, browser):
self.executor = executor
self.browser = browser
@property
def logger(self):
return self.executor.logger
def setup(self, runner):
pass
def teardown(self):
pass
def wait(self):
pass

595
testing/web-platform/harness/wptrunner/executors/executormarionette.py
View file

@ -1,595 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import hashlib
import httplib
import os
import socket
import threading
import time
import traceback
import urlparse
import uuid
from collections import defaultdict
from ..wpttest import WdspecResult, WdspecSubtestResult
errors = None
marionette = None
webdriver = None
here = os.path.join(os.path.split(__file__)[0])
from . import pytestrunner
from .base import (ExecutorException,
Protocol,
RefTestExecutor,
RefTestImplementation,
TestExecutor,
TestharnessExecutor,
testharness_result_converter,
reftest_result_converter,
strip_server,
WdspecExecutor)
from ..testrunner import Stop
from ..webdriver_server import GeckoDriverServer
# Extra timeout to use after internal test timeout at which the harness
# should force a timeout
extra_timeout = 5 # seconds
def do_delayed_imports():
global errors, marionette, webdriver
# Marionette client used to be called marionette, recently it changed
# to marionette_driver for unfathomable reasons
try:
import marionette
from marionette import errors
except ImportError:
from marionette_driver import marionette, errors
import webdriver
class MarionetteProtocol(Protocol):
def __init__(self, executor, browser):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.marionette = None
self.marionette_port = browser.marionette_port
self.timeout = None
self.runner_handle = None
def setup(self, runner):
"""Connect to browser via Marionette."""
Protocol.setup(self, runner)
self.logger.debug("Connecting to Marionette on port %i" % self.marionette_port)
self.marionette = marionette.Marionette(host='localhost',
port=self.marionette_port,
socket_timeout=None)
# XXX Move this timeout somewhere
self.logger.debug("Waiting for Marionette connection")
while True:
success = self.marionette.wait_for_port(60)
#When running in a debugger wait indefinitely for firefox to start
if success or self.executor.debug_info is None:
break
session_started = False
if success:
try:
self.logger.debug("Starting Marionette session")
self.marionette.start_session()
except Exception as e:
self.logger.warning("Starting marionette session failed: %s" % e)
else:
self.logger.debug("Marionette session started")
session_started = True
if not success or not session_started:
self.logger.warning("Failed to connect to Marionette")
self.executor.runner.send_message("init_failed")
else:
try:
self.after_connect()
except Exception:
self.logger.warning("Post-connection steps failed")
self.logger.error(traceback.format_exc())
self.executor.runner.send_message("init_failed")
else:
self.executor.runner.send_message("init_succeeded")
def teardown(self):
try:
self.marionette.delete_session()
except Exception:
# This is typically because the session never started
pass
del self.marionette
@property
def is_alive(self):
"""Check if the Marionette connection is still active."""
try:
self.marionette.current_window_handle
except Exception:
return False
return True
def after_connect(self):
self.load_runner("http")
def set_timeout(self, timeout):
"""Set the Marionette script timeout.
:param timeout: Script timeout in seconds
"""
self.marionette.timeout.script = timeout
self.timeout = timeout
def load_runner(self, protocol):
# Check if we previously had a test window open, and if we did make sure it's closed
self.marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
url = urlparse.urljoin(self.executor.server_url(protocol), "/testharness_runner.html")
self.logger.debug("Loading %s" % url)
self.runner_handle = self.marionette.current_window_handle
try:
self.marionette.navigate(url)
except Exception as e:
self.logger.critical(
"Loading initial page %s failed. Ensure that the "
"there are no other programs bound to this port and "
"that your firewall rules or network setup does not "
"prevent access.\e%s" % (url, traceback.format_exc(e)))
self.marionette.execute_script(
"document.title = '%s'" % threading.current_thread().name.replace("'", '"'))
def close_old_windows(self, protocol):
handles = self.marionette.window_handles
runner_handle = None
try:
handles.remove(self.runner_handle)
runner_handle = self.runner_handle
except ValueError:
# The runner window probably changed id but we can restore it
# This isn't supposed to happen, but marionette ids are not yet stable
# We assume that the first handle returned corresponds to the runner,
# but it hopefully doesn't matter too much if that assumption is
# wrong since we reload the runner in that tab anyway.
runner_handle = handles.pop(0)
for handle in handles:
self.marionette.switch_to_window(handle)
self.marionette.close()
self.marionette.switch_to_window(runner_handle)
if runner_handle != self.runner_handle:
self.load_runner(protocol)
def wait(self):
socket_timeout = self.marionette.client.sock.gettimeout()
if socket_timeout:
self.marionette.timeout.script = socket_timeout / 2
while True:
try:
self.marionette.execute_async_script("")
except errors.ScriptTimeoutException:
self.logger.debug("Script timed out")
pass
except (socket.timeout, IOError):
self.logger.debug("Socket closed")
break
except Exception as e:
self.logger.error(traceback.format_exc(e))
break
def on_environment_change(self, old_environment, new_environment):
#Unset all the old prefs
for name in old_environment.get("prefs", {}).iterkeys():
value = self.executor.original_pref_values[name]
if value is None:
self.clear_user_pref(name)
else:
self.set_pref(name, value)
for name, value in new_environment.get("prefs", {}).iteritems():
self.executor.original_pref_values[name] = self.get_pref(name)
self.set_pref(name, value)
def set_pref(self, name, value):
if value.lower() not in ("true", "false"):
try:
int(value)
except ValueError:
value = "'%s'" % value
else:
value = value.lower()
self.logger.info("Setting pref %s (%s)" % (name, value))
script = """
let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
.getService(Components.interfaces.nsIPrefBranch);
let pref = '%s';
let type = prefInterface.getPrefType(pref);
let value = %s;
switch(type) {
case prefInterface.PREF_STRING:
prefInterface.setCharPref(pref, value);
break;
case prefInterface.PREF_BOOL:
prefInterface.setBoolPref(pref, value);
break;
case prefInterface.PREF_INT:
prefInterface.setIntPref(pref, value);
break;
}
""" % (name, value)
with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
self.marionette.execute_script(script)
def clear_user_pref(self, name):
self.logger.info("Clearing pref %s" % (name))
script = """
let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
.getService(Components.interfaces.nsIPrefBranch);
let pref = '%s';
prefInterface.clearUserPref(pref);
""" % name
with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
self.marionette.execute_script(script)
def get_pref(self, name):
script = """
let prefInterface = Components.classes["@mozilla.org/preferences-service;1"]
.getService(Components.interfaces.nsIPrefBranch);
let pref = '%s';
let type = prefInterface.getPrefType(pref);
switch(type) {
case prefInterface.PREF_STRING:
return prefInterface.getCharPref(pref);
case prefInterface.PREF_BOOL:
return prefInterface.getBoolPref(pref);
case prefInterface.PREF_INT:
return prefInterface.getIntPref(pref);
case prefInterface.PREF_INVALID:
return null;
}
""" % name
with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
self.marionette.execute_script(script)
class RemoteMarionetteProtocol(Protocol):
def __init__(self, executor, browser):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.session = None
self.webdriver_binary = executor.webdriver_binary
self.marionette_port = browser.marionette_port
self.server = None
def setup(self, runner):
"""Connect to browser via the Marionette HTTP server."""
try:
self.server = GeckoDriverServer(
self.logger, self.marionette_port, binary=self.webdriver_binary)
self.server.start(block=False)
self.logger.info(
"WebDriver HTTP server listening at %s" % self.server.url)
self.logger.info(
"Establishing new WebDriver session with %s" % self.server.url)
self.session = webdriver.Session(
self.server.host, self.server.port, self.server.base_path)
except Exception:
self.logger.error(traceback.format_exc())
self.executor.runner.send_message("init_failed")
else:
self.executor.runner.send_message("init_succeeded")
def teardown(self):
try:
if self.session.session_id is not None:
self.session.end()
except Exception:
pass
if self.server is not None and self.server.is_alive:
self.server.stop()
@property
def is_alive(self):
"""Test that the Marionette connection is still alive.
Because the remote communication happens over HTTP we need to
make an explicit request to the remote. It is allowed for
WebDriver spec tests to not have a WebDriver session, since this
may be what is tested.
An HTTP request to an invalid path that results in a 404 is
proof enough to us that the server is alive and kicking.
"""
conn = httplib.HTTPConnection(self.server.host, self.server.port)
conn.request("HEAD", self.server.base_path + "invalid")
res = conn.getresponse()
return res.status == 404
class ExecuteAsyncScriptRun(object):
def __init__(self, logger, func, protocol, url, timeout):
self.logger = logger
self.result = (None, None)
self.protocol = protocol
self.marionette = protocol.marionette
self.func = func
self.url = url
self.timeout = timeout
self.result_flag = threading.Event()
def run(self):
timeout = self.timeout
try:
if timeout is not None:
if timeout + extra_timeout != self.protocol.timeout:
self.protocol.set_timeout(timeout + extra_timeout)
else:
# We just want it to never time out, really, but marionette doesn't
# make that possible. It also seems to time out immediately if the
# timeout is set too high. This works at least.
self.protocol.set_timeout(2**28 - 1)
except IOError:
self.logger.error("Lost marionette connection before starting test")
return Stop
executor = threading.Thread(target = self._run)
executor.start()
if timeout is not None:
wait_timeout = timeout + 2 * extra_timeout
else:
wait_timeout = None
flag = self.result_flag.wait(wait_timeout)
if self.result[1] is None:
self.logger.debug("Timed out waiting for a result")
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
def _run(self):
try:
self.result = True, self.func(self.marionette, self.url, self.timeout)
except errors.ScriptTimeoutException:
self.logger.debug("Got a marionette timeout")
self.result = False, ("EXTERNAL-TIMEOUT", None)
except (socket.timeout, IOError):
# This can happen on a crash
# Also, should check after the test if the firefox process is still running
# and otherwise ignore any other result and set it to crash
self.result = False, ("CRASH", None)
except Exception as e:
message = getattr(e, "message", "")
if message:
message += "\n"
message += traceback.format_exc(e)
self.result = False, ("ERROR", e)
finally:
self.result_flag.set()
class MarionetteTestharnessExecutor(TestharnessExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
close_after_done=True, debug_info=None, **kwargs):
"""Marionette-based executor for testharness.js tests"""
TestharnessExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = MarionetteProtocol(self, browser)
self.script = open(os.path.join(here, "testharness_marionette.js")).read()
self.close_after_done = close_after_done
self.window_id = str(uuid.uuid4())
self.original_pref_values = {}
if marionette is None:
do_delayed_imports()
def is_alive(self):
return self.protocol.is_alive
def on_environment_change(self, new_environment):
self.protocol.on_environment_change(self.last_environment, new_environment)
if new_environment["protocol"] != self.last_environment["protocol"]:
self.protocol.load_runner(new_environment["protocol"])
def do_test(self, test):
timeout = (test.timeout * self.timeout_multiplier if self.debug_info is None
else None)
success, data = ExecuteAsyncScriptRun(self.logger,
self.do_testharness,
self.protocol,
self.test_url(test),
timeout).run()
if success:
return self.convert_result(test, data)
return (test.result_cls(*data), [])
def do_testharness(self, marionette, url, timeout):
if self.close_after_done:
marionette.execute_script("if (window.wrappedJSObject.win) {window.wrappedJSObject.win.close()}")
self.protocol.close_old_windows(self.protocol)
if timeout is not None:
timeout_ms = str(timeout * 1000)
else:
timeout_ms = "null"
script = self.script % {"abs_url": url,
"url": strip_server(url),
"window_id": self.window_id,
"timeout_multiplier": self.timeout_multiplier,
"timeout": timeout_ms,
"explicit_timeout": timeout is None}
rv = marionette.execute_async_script(script, new_sandbox=False)
return rv
class MarionetteRefTestExecutor(RefTestExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
screenshot_cache=None, close_after_done=True,
debug_info=None, **kwargs):
"""Marionette-based executor for reftests"""
RefTestExecutor.__init__(self,
browser,
server_config,
screenshot_cache=screenshot_cache,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = MarionetteProtocol(self, browser)
self.implementation = RefTestImplementation(self)
self.close_after_done = close_after_done
self.has_window = False
self.original_pref_values = {}
with open(os.path.join(here, "reftest.js")) as f:
self.script = f.read()
with open(os.path.join(here, "reftest-wait.js")) as f:
self.wait_script = f.read()
def is_alive(self):
return self.protocol.is_alive
def on_environment_change(self, new_environment):
self.protocol.on_environment_change(self.last_environment, new_environment)
def do_test(self, test):
if self.close_after_done and self.has_window:
self.protocol.marionette.close()
self.protocol.marionette.switch_to_window(
self.protocol.marionette.window_handles[-1])
self.has_window = False
if not self.has_window:
self.protocol.marionette.execute_script(self.script)
self.protocol.marionette.switch_to_window(self.protocol.marionette.window_handles[-1])
self.has_window = True
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def screenshot(self, test, viewport_size, dpi):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
assert dpi is None
timeout = self.timeout_multiplier * test.timeout if self.debug_info is None else None
test_url = self.test_url(test)
return ExecuteAsyncScriptRun(self.logger,
self._screenshot,
self.protocol,
test_url,
timeout).run()
def _screenshot(self, marionette, url, timeout):
marionette.navigate(url)
marionette.execute_async_script(self.wait_script)
screenshot = marionette.screenshot()
# strip off the data:img/png, part of the url
if screenshot.startswith("data:image/png;base64,"):
screenshot = screenshot.split(",", 1)[1]
return screenshot
class WdspecRun(object):
def __init__(self, func, session, path, timeout):
self.func = func
self.result = (None, None)
self.session = session
self.path = path
self.timeout = timeout
self.result_flag = threading.Event()
def run(self):
"""Runs function in a thread and interrupts it if it exceeds the
given timeout. Returns (True, (Result, [SubtestResult ...])) in
case of success, or (False, (status, extra information)) in the
event of failure.
"""
executor = threading.Thread(target=self._run)
executor.start()
flag = self.result_flag.wait(self.timeout)
if self.result[1] is None:
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
def _run(self):
try:
self.result = True, self.func(self.session, self.path, self.timeout)
except (socket.timeout, IOError):
self.result = False, ("CRASH", None)
except Exception as e:
message = getattr(e, "message")
if message:
message += "\n"
message += traceback.format_exc(e)
self.result = False, ("ERROR", message)
finally:
self.result_flag.set()
class MarionetteWdspecExecutor(WdspecExecutor):
def __init__(self, browser, server_config, webdriver_binary,
timeout_multiplier=1, close_after_done=True, debug_info=None):
WdspecExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.webdriver_binary = webdriver_binary
self.protocol = RemoteMarionetteProtocol(self, browser)
def is_alive(self):
return self.protocol.is_alive
def on_environment_change(self, new_environment):
pass
def do_test(self, test):
timeout = test.timeout * self.timeout_multiplier + extra_timeout
success, data = WdspecRun(self.do_wdspec,
self.protocol.session,
test.abs_path,
timeout).run()
if success:
return self.convert_result(test, data)
return (test.result_cls(*data), [])
def do_wdspec(self, session, path, timeout):
harness_result = ("OK", None)
subtest_results = pytestrunner.run(
path, session, self.server_url, timeout=timeout)
return (harness_result, subtest_results)

271
testing/web-platform/harness/wptrunner/executors/executorselenium.py
View file

@ -1,271 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import socket
import sys
import threading
import time
import traceback
import urlparse
import uuid
from .base import (ExecutorException,
Protocol,
RefTestExecutor,
RefTestImplementation,
TestExecutor,
TestharnessExecutor,
testharness_result_converter,
reftest_result_converter,
strip_server)
from ..testrunner import Stop
here = os.path.join(os.path.split(__file__)[0])
webdriver = None
exceptions = None
extra_timeout = 5
def do_delayed_imports():
global webdriver
global exceptions
from selenium import webdriver
from selenium.common import exceptions
class SeleniumProtocol(Protocol):
def __init__(self, executor, browser, capabilities, **kwargs):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.capabilities = capabilities
self.url = browser.webdriver_url
self.webdriver = None
def setup(self, runner):
"""Connect to browser via Selenium's WebDriver implementation."""
self.runner = runner
self.logger.debug("Connecting to Selenium on URL: %s" % self.url)
session_started = False
try:
self.webdriver = webdriver.Remote(
self.url, desired_capabilities=self.capabilities)
except:
self.logger.warning(
"Connecting to Selenium failed:\n%s" % traceback.format_exc())
else:
self.logger.debug("Selenium session started")
session_started = True
if not session_started:
self.logger.warning("Failed to connect to Selenium")
self.executor.runner.send_message("init_failed")
else:
try:
self.after_connect()
except:
print >> sys.stderr, traceback.format_exc()
self.logger.warning(
"Failed to connect to navigate initial page")
self.executor.runner.send_message("init_failed")
else:
self.executor.runner.send_message("init_succeeded")
def teardown(self):
self.logger.debug("Hanging up on Selenium session")
try:
self.webdriver.quit()
except:
pass
del self.webdriver
def is_alive(self):
try:
# Get a simple property over the connection
self.webdriver.current_window_handle
# TODO what exception?
except (socket.timeout, exceptions.ErrorInResponseException):
return False
return True
def after_connect(self):
self.load_runner("http")
def load_runner(self, protocol):
url = urlparse.urljoin(self.executor.server_url(protocol),
"/testharness_runner.html")
self.logger.debug("Loading %s" % url)
self.webdriver.get(url)
self.webdriver.execute_script("document.title = '%s'" %
threading.current_thread().name.replace("'", '"'))
def wait(self):
while True:
try:
self.webdriver.execute_async_script("");
except exceptions.TimeoutException:
pass
except (socket.timeout, exceptions.NoSuchWindowException,
exceptions.ErrorInResponseException, IOError):
break
except Exception as e:
self.logger.error(traceback.format_exc(e))
break
class SeleniumRun(object):
def __init__(self, func, webdriver, url, timeout):
self.func = func
self.result = None
self.webdriver = webdriver
self.url = url
self.timeout = timeout
self.result_flag = threading.Event()
def run(self):
timeout = self.timeout
try:
self.webdriver.set_script_timeout((timeout + extra_timeout) * 1000)
except exceptions.ErrorInResponseException:
self.logger.error("Lost WebDriver connection")
return Stop
executor = threading.Thread(target=self._run)
executor.start()
flag = self.result_flag.wait(timeout + 2 * extra_timeout)
if self.result is None:
assert not flag
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
def _run(self):
try:
self.result = True, self.func(self.webdriver, self.url, self.timeout)
except exceptions.TimeoutException:
self.result = False, ("EXTERNAL-TIMEOUT", None)
except (socket.timeout, exceptions.ErrorInResponseException):
self.result = False, ("CRASH", None)
except Exception as e:
message = getattr(e, "message", "")
if message:
message += "\n"
message += traceback.format_exc(e)
self.result = False, ("ERROR", e)
finally:
self.result_flag.set()
class SeleniumTestharnessExecutor(TestharnessExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
close_after_done=True, capabilities=None, debug_info=None):
"""Selenium-based executor for testharness.js tests"""
TestharnessExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = SeleniumProtocol(self, browser, capabilities)
with open(os.path.join(here, "testharness_webdriver.js")) as f:
self.script = f.read()
self.close_after_done = close_after_done
self.window_id = str(uuid.uuid4())
def is_alive(self):
return self.protocol.is_alive()
def on_protocol_change(self, new_protocol):
self.protocol.load_runner(new_protocol)
def do_test(self, test):
url = self.test_url(test)
success, data = SeleniumRun(self.do_testharness,
self.protocol.webdriver,
url,
test.timeout * self.timeout_multiplier).run()
if success:
return self.convert_result(test, data)
return (test.result_cls(*data), [])
def do_testharness(self, webdriver, url, timeout):
return webdriver.execute_async_script(
self.script % {"abs_url": url,
"url": strip_server(url),
"window_id": self.window_id,
"timeout_multiplier": self.timeout_multiplier,
"timeout": timeout * 1000})
class SeleniumRefTestExecutor(RefTestExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
screenshot_cache=None, close_after_done=True,
debug_info=None, capabilities=None):
"""Selenium WebDriver-based executor for reftests"""
RefTestExecutor.__init__(self,
browser,
server_config,
screenshot_cache=screenshot_cache,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = SeleniumProtocol(self, browser,
capabilities=capabilities)
self.implementation = RefTestImplementation(self)
self.close_after_done = close_after_done
self.has_window = False
with open(os.path.join(here, "reftest.js")) as f:
self.script = f.read()
with open(os.path.join(here, "reftest-wait_webdriver.js")) as f:
self.wait_script = f.read()
def is_alive(self):
return self.protocol.is_alive()
def do_test(self, test):
self.logger.info("Test requires OS-level window focus")
if self.close_after_done and self.has_window:
self.protocol.webdriver.close()
self.protocol.webdriver.switch_to_window(
self.protocol.webdriver.window_handles[-1])
self.has_window = False
if not self.has_window:
self.protocol.webdriver.execute_script(self.script)
self.protocol.webdriver.switch_to_window(
self.protocol.webdriver.window_handles[-1])
self.has_window = True
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def screenshot(self, test, viewport_size, dpi):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
assert dpi is None
return SeleniumRun(self._screenshot,
self.protocol.webdriver,
self.test_url(test),
test.timeout).run()
def _screenshot(self, webdriver, url, timeout):
webdriver.get(url)
webdriver.execute_async_script(self.wait_script)
screenshot = webdriver.get_screenshot_as_base64()
# strip off the data:img/png, part of the url
if screenshot.startswith("data:image/png;base64,"):
screenshot = screenshot.split(",", 1)[1]
return screenshot

275
testing/web-platform/harness/wptrunner/executors/executorservo.py
View file

@ -1,275 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import base64
import hashlib
import json
import os
import subprocess
import tempfile
import threading
import urlparse
import uuid
from collections import defaultdict
from mozprocess import ProcessHandler
from .base import (ExecutorException,
Protocol,
RefTestImplementation,
testharness_result_converter,
reftest_result_converter)
from .process import ProcessTestExecutor
from ..browsers.base import browser_command
render_arg = None
def do_delayed_imports():
global render_arg
from ..browsers.servo import render_arg
hosts_text = """127.0.0.1 web-platform.test
127.0.0.1 www.web-platform.test
127.0.0.1 www1.web-platform.test
127.0.0.1 www2.web-platform.test
127.0.0.1 xn--n8j6ds53lwwkrqhv28a.web-platform.test
127.0.0.1 xn--lve-6lad.web-platform.test
"""
def make_hosts_file():
hosts_fd, hosts_path = tempfile.mkstemp()
with os.fdopen(hosts_fd, "w") as f:
f.write(hosts_text)
return hosts_path
class ServoTestharnessExecutor(ProcessTestExecutor):
convert_result = testharness_result_converter
def __init__(self, browser, server_config, timeout_multiplier=1, debug_info=None,
pause_after_test=False):
do_delayed_imports()
ProcessTestExecutor.__init__(self, browser, server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.pause_after_test = pause_after_test
self.result_data = None
self.result_flag = None
self.protocol = Protocol(self, browser)
self.hosts_path = make_hosts_file()
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
ProcessTestExecutor.teardown(self)
def do_test(self, test):
self.result_data = None
self.result_flag = threading.Event()
args = [render_arg(self.browser.render_backend), "--hard-fail", "-u", "Servo/wptrunner",
"-Z", "replace-surrogates", "-z", self.test_url(test)]
for stylesheet in self.browser.user_stylesheets:
args += ["--user-stylesheet", stylesheet]
for pref, value in test.environment.get('prefs', {}).iteritems():
args += ["--pref", "%s=%s" % (pref, value)]
args += self.browser.binary_args
debug_args, command = browser_command(self.binary, args, self.debug_info)
self.command = command
if self.pause_after_test:
self.command.remove("-z")
self.command = debug_args + self.command
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
onFinish=self.on_finish,
env=env,
storeOutput=False)
self.proc.run()
else:
self.proc = subprocess.Popen(self.command, env=env)
try:
timeout = test.timeout * self.timeout_multiplier
# Now wait to get the output we expect, or until we reach the timeout
if not self.interactive and not self.pause_after_test:
wait_timeout = timeout + 5
self.result_flag.wait(wait_timeout)
else:
wait_timeout = None
self.proc.wait()
proc_is_running = True
if self.result_flag.is_set():
if self.result_data is not None:
result = self.convert_result(test, self.result_data)
else:
self.proc.wait()
result = (test.result_cls("CRASH", None), [])
proc_is_running = False
else:
result = (test.result_cls("TIMEOUT", None), [])
if proc_is_running:
if self.pause_after_test:
self.logger.info("Pausing until the browser exits")
self.proc.wait()
else:
self.proc.kill()
except KeyboardInterrupt:
self.proc.kill()
raise
return result
def on_output(self, line):
prefix = "ALERT: RESULT: "
line = line.decode("utf8", "replace")
if line.startswith(prefix):
self.result_data = json.loads(line[len(prefix):])
self.result_flag.set()
else:
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))
def on_finish(self):
self.result_flag.set()
class TempFilename(object):
def __init__(self, directory):
self.directory = directory
self.path = None
def __enter__(self):
self.path = os.path.join(self.directory, str(uuid.uuid4()))
return self.path
def __exit__(self, *args, **kwargs):
try:
os.unlink(self.path)
except OSError:
pass
class ServoRefTestExecutor(ProcessTestExecutor):
convert_result = reftest_result_converter
def __init__(self, browser, server_config, binary=None, timeout_multiplier=1,
screenshot_cache=None, debug_info=None, pause_after_test=False):
do_delayed_imports()
ProcessTestExecutor.__init__(self,
browser,
server_config,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = Protocol(self, browser)
self.screenshot_cache = screenshot_cache
self.implementation = RefTestImplementation(self)
self.tempdir = tempfile.mkdtemp()
self.hosts_path = make_hosts_file()
def teardown(self):
try:
os.unlink(self.hosts_path)
except OSError:
pass
os.rmdir(self.tempdir)
ProcessTestExecutor.teardown(self)
def screenshot(self, test, viewport_size, dpi):
full_url = self.test_url(test)
with TempFilename(self.tempdir) as output_path:
debug_args, command = browser_command(
self.binary,
[render_arg(self.browser.render_backend), "--hard-fail", "--exit",
"-u", "Servo/wptrunner", "-Z", "disable-text-aa,load-webfonts-synchronously,replace-surrogates",
"--output=%s" % output_path, full_url],
self.debug_info)
for stylesheet in self.browser.user_stylesheets:
command += ["--user-stylesheet", stylesheet]
for pref in test.environment.get('prefs', {}):
command += ["--pref", pref]
if viewport_size:
command += ["--resolution", viewport_size]
if dpi:
command += ["--device-pixel-ratio", dpi]
self.command = debug_args + command
env = os.environ.copy()
env["HOST_FILE"] = self.hosts_path
env["RUST_BACKTRACE"] = "1"
if not self.interactive:
self.proc = ProcessHandler(self.command,
processOutputLine=[self.on_output],
env=env)
try:
self.proc.run()
timeout = test.timeout * self.timeout_multiplier + 5
rv = self.proc.wait(timeout=timeout)
except KeyboardInterrupt:
self.proc.kill()
raise
else:
self.proc = subprocess.Popen(self.command,
env=env)
try:
rv = self.proc.wait()
except KeyboardInterrupt:
self.proc.kill()
raise
if rv is None:
self.proc.kill()
return False, ("EXTERNAL-TIMEOUT", None)
if rv != 0 or not os.path.exists(output_path):
return False, ("CRASH", None)
with open(output_path) as f:
# Might need to strip variable headers or something here
data = f.read()
return True, base64.b64encode(data)
def do_test(self, test):
result = self.implementation.run_test(test)
return self.convert_result(test, result)
def on_output(self, line):
line = line.decode("utf8", "replace")
if self.interactive:
print line
else:
self.logger.process_output(self.proc.pid,
line,
" ".join(self.command))

262
testing/web-platform/harness/wptrunner/executors/executorservodriver.py
View file

@ -1,262 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import os
import socket
import threading
import time
import traceback
from .base import (Protocol,
RefTestExecutor,
RefTestImplementation,
TestharnessExecutor,
strip_server)
from .. import webdriver
from ..testrunner import Stop
webdriver = None
here = os.path.join(os.path.split(__file__)[0])
extra_timeout = 5
def do_delayed_imports():
global webdriver
import webdriver
class ServoWebDriverProtocol(Protocol):
def __init__(self, executor, browser, capabilities, **kwargs):
do_delayed_imports()
Protocol.__init__(self, executor, browser)
self.capabilities = capabilities
self.host = browser.webdriver_host
self.port = browser.webdriver_port
self.session = None
def setup(self, runner):
"""Connect to browser via WebDriver."""
self.runner = runner
url = "http://%s:%d" % (self.host, self.port)
session_started = False
try:
self.session = webdriver.Session(self.host, self.port,
extension=webdriver.servo.ServoCommandExtensions)
self.session.start()
except:
self.logger.warning(
"Connecting with WebDriver failed:\n%s" % traceback.format_exc())
else:
self.logger.debug("session started")
session_started = True
if not session_started:
self.logger.warning("Failed to connect via WebDriver")
self.executor.runner.send_message("init_failed")
else:
self.executor.runner.send_message("init_succeeded")
def teardown(self):
self.logger.debug("Hanging up on WebDriver session")
try:
self.session.end()
except:
pass
def is_alive(self):
try:
# Get a simple property over the connection
self.session.window_handle
# TODO what exception?
except Exception:
return False
return True
def after_connect(self):
pass
def wait(self):
while True:
try:
self.session.execute_async_script("")
except webdriver.TimeoutException:
pass
except (socket.timeout, IOError):
break
except Exception as e:
self.logger.error(traceback.format_exc(e))
break
def on_environment_change(self, old_environment, new_environment):
#Unset all the old prefs
self.session.extension.reset_prefs(*old_environment.get("prefs", {}).keys())
self.session.extension.set_prefs(new_environment.get("prefs", {}))
class ServoWebDriverRun(object):
def __init__(self, func, session, url, timeout, current_timeout=None):
self.func = func
self.result = None
self.session = session
self.url = url
self.timeout = timeout
self.result_flag = threading.Event()
def run(self):
executor = threading.Thread(target=self._run)
executor.start()
flag = self.result_flag.wait(self.timeout + extra_timeout)
if self.result is None:
assert not flag
self.result = False, ("EXTERNAL-TIMEOUT", None)
return self.result
def _run(self):
try:
self.result = True, self.func(self.session, self.url, self.timeout)
except webdriver.TimeoutException:
self.result = False, ("EXTERNAL-TIMEOUT", None)
except (socket.timeout, IOError):
self.result = False, ("CRASH", None)
except Exception as e:
message = getattr(e, "message", "")
if message:
message += "\n"
message += traceback.format_exc(e)
self.result = False, ("ERROR", e)
finally:
self.result_flag.set()
def timeout_func(timeout):
if timeout:
t0 = time.time()
return lambda: time.time() - t0 > timeout + extra_timeout
else:
return lambda: False
class ServoWebDriverTestharnessExecutor(TestharnessExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
close_after_done=True, capabilities=None, debug_info=None):
TestharnessExecutor.__init__(self, browser, server_config, timeout_multiplier=1,
debug_info=None)
self.protocol = ServoWebDriverProtocol(self, browser, capabilities=capabilities)
with open(os.path.join(here, "testharness_servodriver.js")) as f:
self.script = f.read()
self.timeout = None
def on_protocol_change(self, new_protocol):
pass
def is_alive(self):
return self.protocol.is_alive()
def do_test(self, test):
url = self.test_url(test)
timeout = test.timeout * self.timeout_multiplier + extra_timeout
if timeout != self.timeout:
try:
self.protocol.session.timeouts.script = timeout
self.timeout = timeout
except IOError:
self.logger.error("Lost webdriver connection")
return Stop
success, data = ServoWebDriverRun(self.do_testharness,
self.protocol.session,
url,
timeout).run()
if success:
return self.convert_result(test, data)
return (test.result_cls(*data), [])
def do_testharness(self, session, url, timeout):
session.url = url
result = json.loads(
session.execute_async_script(
self.script % {"abs_url": url,
"url": strip_server(url),
"timeout_multiplier": self.timeout_multiplier,
"timeout": timeout * 1000}))
# Prevent leaking every page in history until Servo develops a more sane
# page cache
session.back()
return result
class TimeoutError(Exception):
pass
class ServoWebDriverRefTestExecutor(RefTestExecutor):
def __init__(self, browser, server_config, timeout_multiplier=1,
screenshot_cache=None, capabilities=None, debug_info=None):
"""Selenium WebDriver-based executor for reftests"""
RefTestExecutor.__init__(self,
browser,
server_config,
screenshot_cache=screenshot_cache,
timeout_multiplier=timeout_multiplier,
debug_info=debug_info)
self.protocol = ServoWebDriverProtocol(self, browser,
capabilities=capabilities)
self.implementation = RefTestImplementation(self)
self.timeout = None
with open(os.path.join(here, "reftest-wait_servodriver.js")) as f:
self.wait_script = f.read()
def is_alive(self):
return self.protocol.is_alive()
def do_test(self, test):
try:
result = self.implementation.run_test(test)
return self.convert_result(test, result)
except IOError:
return test.result_cls("CRASH", None), []
except TimeoutError:
return test.result_cls("TIMEOUT", None), []
except Exception as e:
message = getattr(e, "message", "")
if message:
message += "\n"
message += traceback.format_exc(e)
return test.result_cls("ERROR", message), []
def screenshot(self, test, viewport_size, dpi):
# https://github.com/w3c/wptrunner/issues/166
assert viewport_size is None
assert dpi is None
timeout = (test.timeout * self.timeout_multiplier + extra_timeout
if self.debug_info is None else None)
if self.timeout != timeout:
try:
self.protocol.session.timeouts.script = timeout
self.timeout = timeout
except IOError:
self.logger.error("Lost webdriver connection")
return Stop
return ServoWebDriverRun(self._screenshot,
self.protocol.session,
self.test_url(test),
timeout).run()
def _screenshot(self, session, url, timeout):
session.url = url
session.execute_async_script(self.wait_script)
return session.screenshot()

24
testing/web-platform/harness/wptrunner/executors/process.py
View file

@ -1,24 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from .base import TestExecutor
class ProcessTestExecutor(TestExecutor):
def __init__(self, *args, **kwargs):
TestExecutor.__init__(self, *args, **kwargs)
self.binary = self.browser.binary
self.interactive = (False if self.debug_info is None
else self.debug_info.interactive)
def setup(self, runner):
self.runner = runner
self.runner.send_message("init_succeeded")
return True
def is_alive(self):
return True
def do_test(self, test):
raise NotImplementedError

6
testing/web-platform/harness/wptrunner/executors/pytestrunner/__init__.py
View file

@ -1,6 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
from . import fixtures
from .runner import run

Some files were not shown because too many files have changed in this diff Show more