import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

testing/web-platform/harness/wptrunner/wptmanifest/__init__.py

@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from serializer import serialize
+from parser import parse
+from backends.static import compile as compile_static
+from backends.conditional import compile as compile_condition

testing/web-platform/harness/wptrunner/wptmanifest/backends/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.

testing/web-platform/harness/wptrunner/wptmanifest/backends/conditional.py

@@ -0,0 +1,334 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import operator
+
+from ..node import NodeVisitor, DataNode, ConditionalNode, KeyValueNode, ListNode, ValueNode
+from ..parser import parse
+
+
+class ConditionalValue(object):
+    def __init__(self, node, condition_func):
+        self.node = node
+        self.condition_func = condition_func
+        if isinstance(node, ConditionalNode):
+            assert len(node.children) == 2
+            self.condition_node = self.node.children[0]
+            self.value_node = self.node.children[1]
+        else:
+            assert isinstance(node, (ValueNode, ListNode))
+            self.condition_node = None
+            self.value_node = self.node
+
+    @property
+    def value(self):
+        if isinstance(self.value_node, ValueNode):
+            return self.value_node.data
+        else:
+            return [item.data for item in self.value_node.children]
+
+    @value.setter
+    def value(self, value):
+        self.value_node.data = value
+
+    def __call__(self, run_info):
+        return self.condition_func(run_info)
+
+    def set_value(self, value):
+        self.value = value
+
+    def remove(self):
+        if len(self.node.parent.children) == 1:
+            self.node.parent.remove()
+        self.node.remove()
+
+
+class Compiler(NodeVisitor):
+    def compile(self, tree, data_cls_getter=None, **kwargs):
+        """Compile a raw AST into a form where conditional expressions
+        are represented by ConditionalValue objects that can be evaluated
+        at runtime.
+
+        tree - The root node of the wptmanifest AST to compile
+
+        data_cls_getter - A function taking two parameters; the previous
+                          output node and the current ast node and returning
+                          the class of the output node to use for the current
+                          ast node
+        """
+        if data_cls_getter is None:
+            self.data_cls_getter = lambda x, y: ManifestItem
+        else:
+            self.data_cls_getter = data_cls_getter
+
+        self.tree = tree
+        self.output_node = self._initial_output_node(tree, **kwargs)
+        self.visit(tree)
+        assert self.output_node is not None
+        return self.output_node
+
+    def compile_condition(self, condition):
+        """Compile a ConditionalNode into a ConditionalValue.
+
+        condition: A ConditionalNode"""
+        data_node = DataNode()
+        key_value_node = KeyValueNode()
+        key_value_node.append(condition.copy())
+        data_node.append(key_value_node)
+        manifest_item = self.compile(data_node)
+        return manifest_item._data[None][0]
+
+    def _initial_output_node(self, node, **kwargs):
+        return self.data_cls_getter(None, None)(node, **kwargs)
+
+    def visit_DataNode(self, node):
+        if node != self.tree:
+            output_parent = self.output_node
+            self.output_node = self.data_cls_getter(self.output_node, node)(node)
+        else:
+            output_parent = None
+
+        assert self.output_node is not None
+
+        for child in node.children:
+            self.visit(child)
+
+        if output_parent is not None:
+            # Append to the parent *after* processing all the node data
+            output_parent.append(self.output_node)
+            self.output_node = self.output_node.parent
+
+        assert self.output_node is not None
+
+    def visit_KeyValueNode(self, node):
+        key_values = []
+        for child in node.children:
+            condition, value = self.visit(child)
+            key_values.append(ConditionalValue(child, condition))
+
+        self.output_node._add_key_value(node, key_values)
+
+    def visit_ListNode(self, node):
+        return (lambda x:True, [self.visit(child) for child in node.children])
+
+    def visit_ValueNode(self, node):
+        return (lambda x: True, node.data)
+
+    def visit_AtomNode(self, node):
+        return (lambda x: True, node.data)
+
+    def visit_ConditionalNode(self, node):
+        return self.visit(node.children[0]), self.visit(node.children[1])
+
+    def visit_StringNode(self, node):
+        indexes = [self.visit(child) for child in node.children]
+
+        def value(x):
+            rv = node.data
+            for index in indexes:
+                rv = rv[index(x)]
+            return rv
+        return value
+
+    def visit_NumberNode(self, node):
+        if "." in node.data:
+            return lambda x: float(node.data)
+        else:
+            return lambda x: int(node.data)
+
+    def visit_VariableNode(self, node):
+        indexes = [self.visit(child) for child in node.children]
+
+        def value(x):
+            data = x[node.data]
+            for index in indexes:
+                data = data[index(x)]
+            return data
+        return value
+
+    def visit_IndexNode(self, node):
+        assert len(node.children) == 1
+        return self.visit(node.children[0])
+
+    def visit_UnaryExpressionNode(self, node):
+        assert len(node.children) == 2
+        operator = self.visit(node.children[0])
+        operand = self.visit(node.children[1])
+
+        return lambda x: operator(operand(x))
+
+    def visit_BinaryExpressionNode(self, node):
+        assert len(node.children) == 3
+        operator = self.visit(node.children[0])
+        operand_0 = self.visit(node.children[1])
+        operand_1 = self.visit(node.children[2])
+
+        assert operand_0 is not None
+        assert operand_1 is not None
+
+        return lambda x: operator(operand_0(x), operand_1(x))
+
+    def visit_UnaryOperatorNode(self, node):
+        return {"not": operator.not_}[node.data]
+
+    def visit_BinaryOperatorNode(self, node):
+        return {"and": operator.and_,
+                "or": operator.or_,
+                "==": operator.eq,
+                "!=": operator.ne}[node.data]
+
+
+class ManifestItem(object):
+    def __init__(self, node=None, **kwargs):
+        self.node = node
+        self.parent = None
+        self.children = []
+        self._data = {}
+
+    def __repr__(self):
+        return "<ManifestItem %s>" % (self.node.data)
+
+    def __str__(self):
+        rv = [repr(self)]
+        for item in self.children:
+            rv.extend("  %s" % line for line in str(item).split("\n"))
+        return "\n".join(rv)
+
+    def __contains__(self, key):
+        return key in self._data
+
+    @property
+    def is_empty(self):
+        if self._data:
+            return False
+        return all(child.is_empty for child in self.children)
+
+    @property
+    def root(self):
+        node = self
+        while node.parent is not None:
+            node = node.parent
+        return node
+
+    @property
+    def name(self):
+        return self.node.data
+
+    def has_key(self, key):
+        for node in [self, self.root]:
+            if key in node._data:
+                return True
+        return False
+
+    def get(self, key, run_info=None):
+        if run_info is None:
+            run_info = {}
+
+        for node in [self, self.root]:
+            if key in node._data:
+                for cond_value in node._data[key]:
+                    try:
+                        matches = cond_value(run_info)
+                    except KeyError:
+                        matches = False
+                    if matches:
+                        return cond_value.value
+        raise KeyError
+
+    def set(self, key, value, condition=None):
+        # First try to update the existing value
+        if key in self._data:
+            cond_values = self._data[key]
+            for cond_value in cond_values:
+                if cond_value.condition_node == condition:
+                    cond_value.value = value
+                    return
+            # If there isn't a conditional match reuse the existing KeyValueNode as the
+            # parent
+            node = None
+            for child in self.node.children:
+                if child.data == key:
+                    node = child
+                    break
+            assert node is not None
+
+        else:
+            node = KeyValueNode(key)
+            self.node.append(node)
+
+        value_node = ValueNode(value)
+        if condition is not None:
+            conditional_node = ConditionalNode()
+            conditional_node.append(condition)
+            conditional_node.append(value_node)
+            node.append(conditional_node)
+            cond_value = Compiler().compile_condition(conditional_node)
+        else:
+            node.append(value_node)
+            cond_value = ConditionalValue(value_node, lambda x: True)
+
+        # Update the cache of child values. This is pretty annoying and maybe
+        # it should just work directly on the tree
+        if key not in self._data:
+            self._data[key] = []
+        if self._data[key] and self._data[key][-1].condition_node is None:
+            self._data[key].insert(len(self._data[key]) - 1, cond_value)
+        else:
+            self._data[key].append(cond_value)
+
+    def _add_key_value(self, node, values):
+        """Called during construction to set a key-value node"""
+        self._data[node.data] = values
+
+    def append(self, child):
+        self.children.append(child)
+        child.parent = self
+        if child.node.parent != self.node:
+            self.node.append(child.node)
+        return child
+
+    def remove(self):
+        if self.parent:
+            self.parent._remove_child(self)
+
+    def _remove_child(self, child):
+        self.children.remove(child)
+        child.parent = None
+
+    def iterchildren(self, name=None):
+        for item in self.children:
+            if item.name == name or name is None:
+                yield item
+
+    def _flatten(self):
+        rv = {}
+        for node in [self, self.root]:
+            for name, value in node._data.iteritems():
+                if name not in rv:
+                    rv[name] = value
+        return rv
+
+    def iteritems(self):
+        for item in self._flatten().iteritems():
+            yield item
+
+    def iterkeys(self):
+        for item in self._flatten().iterkeys():
+            yield item
+
+    def remove_value(self, key, value):
+        self._data[key].remove(value)
+        if not self._data[key]:
+            del self._data[key]
+        value.remove()
+
+
+def compile_ast(ast, data_cls_getter=None, **kwargs):
+    return Compiler().compile(ast, data_cls_getter=data_cls_getter, **kwargs)
+
+
+def compile(stream, data_cls_getter=None, **kwargs):
+    return compile_ast(parse(stream),
+                       data_cls_getter=data_cls_getter,
+                       **kwargs)

testing/web-platform/harness/wptrunner/wptmanifest/backends/static.py

@@ -0,0 +1,224 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import operator
+
+from ..node import NodeVisitor
+from ..parser import parse
+
+
+class Compiler(NodeVisitor):
+    """Compiler backend that evaluates conditional expressions
+    to give static output"""
+
+    def compile(self, tree, expr_data, data_cls_getter=None, **kwargs):
+        """Compile a raw AST into a form with conditional expressions
+        evaluated.
+
+        tree - The root node of the wptmanifest AST to compile
+
+        expr_data - A dictionary of key / value pairs to use when
+                    evaluating conditional expressions
+
+        data_cls_getter - A function taking two parameters; the previous
+                          output node and the current ast node and returning
+                          the class of the output node to use for the current
+                          ast node
+        """
+
+        self._kwargs = kwargs
+        self.expr_data = expr_data
+
+        if data_cls_getter is None:
+            self.data_cls_getter = lambda x, y: ManifestItem
+        else:
+            self.data_cls_getter = data_cls_getter
+
+        self.output_node = None
+        self.visit(tree)
+        return self.output_node
+
+    def visit_DataNode(self, node):
+        output_parent = self.output_node
+        if self.output_node is None:
+            assert node.parent is None
+            self.output_node = self.data_cls_getter(None, None)(None, **self._kwargs)
+        else:
+            self.output_node = self.data_cls_getter(self.output_node, node)(node.data)
+
+        for child in node.children:
+            self.visit(child)
+
+        if output_parent is not None:
+            output_parent.append(self.output_node)
+            self.output_node = self.output_node.parent
+
+    def visit_KeyValueNode(self, node):
+        key_name = node.data
+        key_value = None
+        for child in node.children:
+            value = self.visit(child)
+            if value is not None:
+                key_value = value
+                break
+        if key_value is not None:
+            self.output_node.set(key_name, key_value)
+
+    def visit_ValueNode(self, node):
+        return node.data
+
+    def visit_AtomNode(self, node):
+        return node.data
+
+    def visit_ListNode(self, node):
+        return [self.visit(child) for child in node.children]
+
+    def visit_ConditionalNode(self, node):
+        assert len(node.children) == 2
+        if self.visit(node.children[0]):
+            return self.visit(node.children[1])
+
+    def visit_StringNode(self, node):
+        value = node.data
+        for child in node.children:
+            value = self.visit(child)(value)
+        return value
+
+    def visit_NumberNode(self, node):
+        if "." in node.data:
+            return float(node.data)
+        else:
+            return int(node.data)
+
+    def visit_VariableNode(self, node):
+        value = self.expr_data[node.data]
+        for child in node.children:
+            value = self.visit(child)(value)
+        return value
+
+    def visit_IndexNode(self, node):
+        assert len(node.children) == 1
+        index = self.visit(node.children[0])
+        return lambda x: x[index]
+
+    def visit_UnaryExpressionNode(self, node):
+        assert len(node.children) == 2
+        operator = self.visit(node.children[0])
+        operand = self.visit(node.children[1])
+
+        return operator(operand)
+
+    def visit_BinaryExpressionNode(self, node):
+        assert len(node.children) == 3
+        operator = self.visit(node.children[0])
+        operand_0 = self.visit(node.children[1])
+        operand_1 = self.visit(node.children[2])
+
+        return operator(operand_0, operand_1)
+
+    def visit_UnaryOperatorNode(self, node):
+        return {"not": operator.not_}[node.data]
+
+    def visit_BinaryOperatorNode(self, node):
+        return {"and": operator.and_,
+                "or": operator.or_,
+                "==": operator.eq,
+                "!=": operator.ne}[node.data]
+
+
+class ManifestItem(object):
+    def __init__(self, name, **kwargs):
+        self.parent = None
+        self.name = name
+        self.children = []
+        self._data = {}
+
+    def __repr__(self):
+        return "<ManifestItem %s>" % (self.name)
+
+    def __str__(self):
+        rv = [repr(self)]
+        for item in self.children:
+            rv.extend("  %s" % line for line in str(item).split("\n"))
+        return "\n".join(rv)
+
+    @property
+    def is_empty(self):
+        if self._data:
+            return False
+        return all(child.is_empty for child in self.children)
+
+    @property
+    def root(self):
+        node = self
+        while node.parent is not None:
+            node = node.parent
+        return node
+
+    def has_key(self, key):
+        for node in [self, self.root]:
+            if key in node._data:
+                return True
+        return False
+
+    def get(self, key):
+        for node in [self, self.root]:
+            if key in node._data:
+                return node._data[key]
+        raise KeyError
+
+    def set(self, name, value):
+        self._data[name] = value
+
+    def remove(self):
+        if self.parent:
+            self.parent._remove_child(self)
+
+    def _remove_child(self, child):
+        self.children.remove(child)
+        child.parent = None
+
+    def iterchildren(self, name=None):
+        for item in self.children:
+            if item.name == name or name is None:
+                yield item
+
+    def _flatten(self):
+        rv = {}
+        for node in [self, self.root]:
+            for name, value in node._data.iteritems():
+                if name not in rv:
+                    rv[name] = value
+        return rv
+
+    def iteritems(self):
+        for item in self._flatten().iteritems():
+            yield item
+
+    def iterkeys(self):
+        for item in self._flatten().iterkeys():
+            yield item
+
+    def itervalues(self):
+        for item in self._flatten().itervalues():
+            yield item
+
+    def append(self, child):
+        child.parent = self
+        self.children.append(child)
+        return child
+
+
+def compile_ast(ast, expr_data, data_cls_getter=None, **kwargs):
+    return Compiler().compile(ast,
+                              expr_data,
+                              data_cls_getter=data_cls_getter,
+                              **kwargs)
+
+
+def compile(stream, expr_data, data_cls_getter=None, **kwargs):
+    return compile_ast(parse(stream),
+                       expr_data,
+                       data_cls_getter=data_cls_getter,
+                       **kwargs)

testing/web-platform/harness/wptrunner/wptmanifest/node.py

@@ -0,0 +1,161 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+class NodeVisitor(object):
+    def visit(self, node):
+        # This is ugly as hell, but we don't have multimethods and
+        # they aren't trivial to fake without access to the class
+        # object from the class body
+        func = getattr(self, "visit_%s" % (node.__class__.__name__))
+        return func(node)
+
+
+class Node(object):
+    def __init__(self, data=None):
+        self.data = data
+        self.parent = None
+        self.children = []
+
+    def append(self, other):
+        other.parent = self
+        self.children.append(other)
+
+    def remove(self):
+        self.parent.children.remove(self)
+
+    def __repr__(self):
+        return "<%s %s>" % (self.__class__.__name__, self.data)
+
+    def __str__(self):
+        rv = [repr(self)]
+        for item in self.children:
+            rv.extend("  %s" % line for line in str(item).split("\n"))
+        return "\n".join(rv)
+
+    def __eq__(self, other):
+        if not (self.__class__ == other.__class__ and
+                self.data == other.data and
+                len(self.children) == len(other.children)):
+            return False
+        for child, other_child in zip(self.children, other.children):
+            if not child == other_child:
+                return False
+        return True
+
+    def copy(self):
+        new = self.__class__(self.data)
+        for item in self.children:
+            new.append(item.copy())
+        return new
+
+
+class DataNode(Node):
+    def append(self, other):
+        # Append that retains the invariant that child data nodes
+        # come after child nodes of other types
+        other.parent = self
+        if isinstance(other, DataNode):
+            self.children.append(other)
+        else:
+            index = len(self.children)
+            while index > 0 and isinstance(self.children[index - 1], DataNode):
+                index -= 1
+            for i in xrange(index):
+                assert other.data != self.children[i].data
+            self.children.insert(index, other)
+
+
+class KeyValueNode(Node):
+    def append(self, other):
+        # Append that retains the invariant that conditional nodes
+        # come before unconditional nodes
+        other.parent = self
+        if isinstance(other, ValueNode):
+            if self.children:
+                assert not isinstance(self.children[-1], ValueNode)
+            self.children.append(other)
+        else:
+            if self.children and isinstance(self.children[-1], ValueNode):
+                self.children.insert(len(self.children) - 1, other)
+            else:
+                self.children.append(other)
+
+
+class ListNode(Node):
+    def append(self, other):
+        other.parent = self
+        self.children.append(other)
+
+
+class ValueNode(Node):
+    def append(self, other):
+        raise TypeError
+
+
+class AtomNode(ValueNode):
+    pass
+
+
+class ConditionalNode(Node):
+    pass
+
+
+class UnaryExpressionNode(Node):
+    def __init__(self, operator, operand):
+        Node.__init__(self)
+        self.append(operator)
+        self.append(operand)
+
+    def append(self, other):
+        Node.append(self, other)
+        assert len(self.children) <= 2
+
+    def copy(self):
+        new = self.__class__(self.children[0].copy(),
+                             self.children[1].copy())
+        return new
+
+
+class BinaryExpressionNode(Node):
+    def __init__(self, operator, operand_0, operand_1):
+        Node.__init__(self)
+        self.append(operator)
+        self.append(operand_0)
+        self.append(operand_1)
+
+    def append(self, other):
+        Node.append(self, other)
+        assert len(self.children) <= 3
+
+    def copy(self):
+        new = self.__class__(self.children[0].copy(),
+                             self.children[1].copy(),
+                             self.children[2].copy())
+        return new
+
+
+class UnaryOperatorNode(Node):
+    def append(self, other):
+        raise TypeError
+
+
+class BinaryOperatorNode(Node):
+    def append(self, other):
+        raise TypeError
+
+
+class IndexNode(Node):
+    pass
+
+
+class VariableNode(Node):
+    pass
+
+
+class StringNode(Node):
+    pass
+
+
+class NumberNode(ValueNode):
+    pass

testing/web-platform/harness/wptrunner/wptmanifest/parser.py

@@ -0,0 +1,744 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#default_value:foo
+#include: other.manifest
+#
+#[test_name.js]
+#  expected: ERROR
+#
+#  [subtest 1]
+#    expected:
+#      os == win: FAIL #This is a comment
+#      PASS
+#
+
+# TODO: keep comments in the tree
+
+import types
+from cStringIO import StringIO
+
+from node import *
+
+
+class ParseError(Exception):
+    def __init__(self, filename, line, detail):
+        self.line = line
+        self.filename = filename
+        self.detail = detail
+        self.message = "%s: %s line %s" % (self.detail, self.filename, self.line)
+        Exception.__init__(self, self.message)
+
+eol = object
+group_start = object
+group_end = object
+digits = "0123456789"
+open_parens = "[("
+close_parens = "])"
+parens = open_parens + close_parens
+operator_chars = "=!"
+
+unary_operators = ["not"]
+binary_operators = ["==", "!=", "and", "or"]
+
+operators = ["==", "!=", "not", "and", "or"]
+
+atoms = {"True": True,
+         "False": False,
+         "Reset": object()}
+
+def decode(byte_str):
+    return byte_str.decode("utf8")
+
+
+def precedence(operator_node):
+    return len(operators) - operators.index(operator_node.data)
+
+
+class TokenTypes(object):
+    def __init__(self):
+        for type in ["group_start", "group_end", "paren", "list_start", "list_end", "separator", "ident", "string", "number", "atom", "eof"]:
+            setattr(self, type, type)
+
+token_types = TokenTypes()
+
+
+class Tokenizer(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.indent_levels = [0]
+        self.state = self.line_start_state
+        self.next_state = self.data_line_state
+        self.line_number = 0
+
+    def tokenize(self, stream):
+        self.reset()
+        if type(stream) in types.StringTypes:
+            stream = StringIO(stream)
+        if not hasattr(stream, "name"):
+            self.filename = ""
+        else:
+            self.filename = stream.name
+
+        self.next_line_state = self.line_start_state
+        for i, line in enumerate(stream):
+            self.state = self.next_line_state
+            assert self.state is not None
+            states = []
+            self.next_line_state = None
+            self.line_number = i + 1
+            self.index = 0
+            self.line = line.rstrip()
+            while self.state != self.eol_state:
+                states.append(self.state)
+                tokens = self.state()
+                if tokens:
+                    for token in tokens:
+                        yield token
+            self.state()
+        while True:
+            yield (token_types.eof, None)
+
+    def char(self):
+        if self.index == len(self.line):
+            return eol
+        return self.line[self.index]
+
+    def consume(self):
+        if self.index < len(self.line):
+            self.index += 1
+
+    def peek(self, length):
+        return self.line[self.index:self.index + length]
+
+    def skip_whitespace(self):
+        while self.char() == " ":
+            self.consume()
+
+    def eol_state(self):
+        if self.next_line_state is None:
+            self.next_line_state = self.line_start_state
+
+    def line_start_state(self):
+        self.skip_whitespace()
+        if self.char() == eol:
+            self.state = self.eol_state
+            return
+        if self.index > self.indent_levels[-1]:
+            self.indent_levels.append(self.index)
+            yield (token_types.group_start, None)
+        else:
+            while self.index < self.indent_levels[-1]:
+                self.indent_levels.pop()
+                yield (token_types.group_end, None)
+                # This is terrible; if we were parsing an expression
+                # then the next_state will be expr_or_value but when we deindent
+                # it must always be a heading or key next so we go back to data_line_state
+                self.next_state = self.data_line_state
+            if self.index != self.indent_levels[-1]:
+                raise ParseError(self.filename, self.line_number, "Unexpected indent")
+
+        self.state = self.next_state
+
+    def data_line_state(self):
+        if self.char() == "[":
+            yield (token_types.paren, self.char())
+            self.consume()
+            self.state = self.heading_state
+        else:
+            self.state = self.key_state
+
+    def heading_state(self):
+        rv = ""
+        while True:
+            c = self.char()
+            if c == "\\":
+                rv += self.consume_escape()
+            elif c == "]":
+                break
+            elif c == eol:
+                raise ParseError(self.filename, self.line_number, "EOL in heading")
+            else:
+                rv += c
+                self.consume()
+
+        yield (token_types.string, decode(rv))
+        yield (token_types.paren, "]")
+        self.consume()
+        self.state = self.line_end_state
+        self.next_state = self.data_line_state
+
+    def key_state(self):
+        rv = ""
+        while True:
+            c = self.char()
+            if c == " ":
+                self.skip_whitespace()
+                if self.char() != ":":
+                    raise ParseError(self.filename, self.line_number, "Space in key name")
+                break
+            elif c == ":":
+                break
+            elif c == eol:
+                raise ParseError(self.filename, self.line_number, "EOL in key name (missing ':'?)")
+            elif c == "\\":
+                rv += self.consume_escape()
+            else:
+                rv += c
+                self.consume()
+        yield (token_types.string, decode(rv))
+        yield (token_types.separator, ":")
+        self.consume()
+        self.state = self.after_key_state
+
+    def after_key_state(self):
+        self.skip_whitespace()
+        c = self.char()
+        if c == "#":
+            self.next_state = self.expr_or_value_state
+            self.state = self.comment_state
+        elif c == eol:
+            self.next_state = self.expr_or_value_state
+            self.state = self.eol_state
+        elif c == "[":
+            self.state = self.list_start_state
+        else:
+            self.state = self.value_state
+
+    def list_start_state(self):
+        yield (token_types.list_start, "[")
+        self.consume()
+        self.state = self.list_value_start_state
+
+    def list_value_start_state(self):
+        self.skip_whitespace()
+        if self.char() == "]":
+            self.state = self.list_end_state
+        elif self.char() in ("'", '"'):
+            quote_char = self.char()
+            self.consume()
+            yield (token_types.string, self.consume_string(quote_char))
+            self.skip_whitespace()
+            if self.char() == "]":
+                self.state = self.list_end_state
+            elif self.char() != ",":
+                raise ParseError(self.filename, self.line_number, "Junk after quoted string")
+            self.consume()
+        elif self.char() == "#":
+            self.state = self.comment_state
+            self.next_line_state = self.list_value_start_state
+        elif self.char() == eol:
+            self.next_line_state = self.list_value_start_state
+            self.state = self.eol_state
+        elif self.char() == ",":
+            raise ParseError(self.filename, self.line_number, "List item started with separator")
+        elif self.char() == "@":
+            self.state = self.list_value_atom_state
+        else:
+            self.state = self.list_value_state
+
+    def list_value_state(self):
+        rv = ""
+        spaces = 0
+        while True:
+            c = self.char()
+            if c == "\\":
+                escape = self.consume_escape()
+                rv += escape
+            elif c == eol:
+                raise ParseError(self.filename, self.line_number, "EOL in list value")
+            elif c == "#":
+                raise ParseError(self.filename, self.line_number, "EOL in list value (comment)")
+            elif c == ",":
+                self.state = self.list_value_start_state
+                self.consume()
+                break
+            elif c == " ":
+                spaces += 1
+                self.consume()
+            elif c == "]":
+                self.state = self.list_end_state
+                self.consume()
+                break
+            else:
+                rv += " " * spaces
+                spaces = 0
+                rv += c
+                self.consume()
+
+        if rv:
+            yield (token_types.string, decode(rv))
+
+    def list_value_atom_state(self):
+        self.consume()
+        for _, value in self.list_value_state():
+            yield token_types.atom, value
+
+    def list_end_state(self):
+        self.consume()
+        yield (token_types.list_end, "]")
+        self.state = self.line_end_state
+
+    def value_state(self):
+        self.skip_whitespace()
+        if self.char() in ("'", '"'):
+            quote_char = self.char()
+            self.consume()
+            yield (token_types.string, self.consume_string(quote_char))
+            if self.char() == "#":
+                self.state = self.comment_state
+            else:
+                self.state = self.line_end_state
+        elif self.char() == "@":
+            self.consume()
+            for _, value in self.value_inner_state():
+                yield token_types.atom, value
+        else:
+            self.state = self.value_inner_state
+
+    def value_inner_state(self):
+        rv = ""
+        spaces = 0
+        while True:
+            c = self.char()
+            if c == "\\":
+                rv += self.consume_escape()
+            elif c == "#":
+                self.state = self.comment_state
+                break
+            elif c == " ":
+                # prevent whitespace before comments from being included in the value
+                spaces += 1
+                self.consume()
+            elif c == eol:
+                self.state = self.line_end_state
+                break
+            else:
+                rv += " " * spaces
+                spaces = 0
+                rv += c
+                self.consume()
+        yield (token_types.string, decode(rv))
+
+    def comment_state(self):
+        while self.char() is not eol:
+            self.consume()
+        self.state = self.eol_state
+
+    def line_end_state(self):
+        self.skip_whitespace()
+        c = self.char()
+        if c == "#":
+            self.state = self.comment_state
+        elif c == eol:
+            self.state = self.eol_state
+        else:
+            raise ParseError(self.filename, self.line_number, "Junk before EOL %s" % c)
+
+    def consume_string(self, quote_char):
+        rv = ""
+        while True:
+            c = self.char()
+            if c == "\\":
+                rv += self.consume_escape()
+            elif c == quote_char:
+                self.consume()
+                break
+            elif c == eol:
+                raise ParseError(self.filename, self.line_number, "EOL in quoted string")
+            else:
+                rv += c
+                self.consume()
+
+        return decode(rv)
+
+    def expr_or_value_state(self):
+        if self.peek(3) == "if ":
+            self.state = self.expr_state
+        else:
+            self.state = self.value_state
+
+    def expr_state(self):
+        self.skip_whitespace()
+        c = self.char()
+        if c == eol:
+            raise ParseError(self.filename, self.line_number, "EOL in expression")
+        elif c in "'\"":
+            self.consume()
+            yield (token_types.string, self.consume_string(c))
+        elif c == "#":
+            raise ParseError(self.filename, self.line_number, "Comment before end of expression")
+        elif c == ":":
+            yield (token_types.separator, c)
+            self.consume()
+            self.state = self.value_state
+        elif c in parens:
+            self.consume()
+            yield (token_types.paren, c)
+        elif c in ("!", "="):
+            self.state = self.operator_state
+        elif c in digits:
+            self.state = self.digit_state
+        else:
+            self.state = self.ident_state
+
+    def operator_state(self):
+        # Only symbolic operators
+        index_0 = self.index
+        while True:
+            c = self.char()
+            if c == eol:
+                break
+            elif c in operator_chars:
+                self.consume()
+            else:
+                self.state = self.expr_state
+                break
+        yield (token_types.ident, self.line[index_0:self.index])
+
+    def digit_state(self):
+        index_0 = self.index
+        seen_dot = False
+        while True:
+            c = self.char()
+            if c == eol:
+                break
+            elif c in digits:
+                self.consume()
+            elif c == ".":
+                if seen_dot:
+                    raise ParseError(self.filename, self.line_number, "Invalid number")
+                self.consume()
+                seen_dot = True
+            elif c in parens:
+                break
+            elif c in operator_chars:
+                break
+            elif c == " ":
+                break
+            elif c == ":":
+                break
+            else:
+                raise ParseError(self.filename, self.line_number, "Invalid character in number")
+
+        self.state = self.expr_state
+        yield (token_types.number, self.line[index_0:self.index])
+
+    def ident_state(self):
+        index_0 = self.index
+        while True:
+            c = self.char()
+            if c == eol:
+                break
+            elif c == ".":
+                break
+            elif c in parens:
+                break
+            elif c in operator_chars:
+                break
+            elif c == " ":
+                break
+            elif c == ":":
+                break
+            else:
+                self.consume()
+        self.state = self.expr_state
+        yield (token_types.ident, self.line[index_0:self.index])
+
+    def consume_escape(self):
+        assert self.char() == "\\"
+        self.consume()
+        c = self.char()
+        self.consume()
+        if c == "x":
+            return self.decode_escape(2)
+        elif c == "u":
+            return self.decode_escape(4)
+        elif c == "U":
+            return self.decode_escape(6)
+        elif c in ["a", "b", "f", "n", "r", "t", "v"]:
+            return eval("'\%s'" % c)
+        elif c is eol:
+            raise ParseError(self.filename, self.line_number, "EOL in escape")
+        else:
+            return c
+
+    def decode_escape(self, length):
+        value = 0
+        for i in xrange(length):
+            c = self.char()
+            value *= 16
+            value += self.escape_value(c)
+            self.consume()
+
+        return unichr(value).encode("utf8")
+
+    def escape_value(self, c):
+        if '0' <= c <= '9':
+            return ord(c) - ord('0')
+        elif 'a' <= c <= 'f':
+            return ord(c) - ord('a') + 10
+        elif 'A' <= c <= 'F':
+            return ord(c) - ord('A') + 10
+        else:
+            raise ParseError(self.filename, self.line_number, "Invalid character escape")
+
+
+class Parser(object):
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.token = None
+        self.unary_operators = "!"
+        self.binary_operators = frozenset(["&&", "||", "=="])
+        self.tokenizer = Tokenizer()
+        self.token_generator = None
+        self.tree = Treebuilder(DataNode(None))
+        self.expr_builder = None
+        self.expr_builders = []
+
+    def parse(self, input):
+        self.reset()
+        self.token_generator = self.tokenizer.tokenize(input)
+        self.consume()
+        self.manifest()
+        return self.tree.node
+
+    def consume(self):
+        self.token = self.token_generator.next()
+
+    def expect(self, type, value=None):
+        if self.token[0] != type:
+            raise ParseError
+        if value is not None:
+            if self.token[1] != value:
+                raise ParseError
+
+        self.consume()
+
+    def manifest(self):
+        self.data_block()
+        self.expect(token_types.eof)
+
+    def data_block(self):
+        while self.token[0] == token_types.string:
+            self.tree.append(KeyValueNode(self.token[1]))
+            self.consume()
+            self.expect(token_types.separator)
+            self.value_block()
+            self.tree.pop()
+
+        while self.token == (token_types.paren, "["):
+            self.consume()
+            if self.token[0] != token_types.string:
+                raise ParseError
+            self.tree.append(DataNode(self.token[1]))
+            self.consume()
+            self.expect(token_types.paren, "]")
+            if self.token[0] == token_types.group_start:
+                self.consume()
+                self.data_block()
+                self.eof_or_end_group()
+            self.tree.pop()
+
+    def eof_or_end_group(self):
+        if self.token[0] != token_types.eof:
+            self.expect(token_types.group_end)
+
+    def value_block(self):
+        if self.token[0] == token_types.list_start:
+            self.consume()
+            self.list_value()
+        elif self.token[0] == token_types.string:
+            self.value()
+        elif self.token[0] == token_types.group_start:
+            self.consume()
+            self.expression_values()
+            if self.token[0] == token_types.string:
+                self.value()
+            self.eof_or_end_group()
+        elif self.token[0] == token_types.atom:
+            self.atom()
+        else:
+            raise ParseError
+
+    def list_value(self):
+        self.tree.append(ListNode())
+        while self.token[0] in (token_types.atom, token_types.string):
+            if self.token[0] == token_types.atom:
+                self.atom()
+            else:
+                self.value()
+        self.expect(token_types.list_end)
+        self.tree.pop()
+
+    def expression_values(self):
+        while self.token == (token_types.ident, "if"):
+            self.consume()
+            self.tree.append(ConditionalNode())
+            self.expr_start()
+            self.expect(token_types.separator)
+            if self.token[0] == token_types.string:
+                self.value()
+            else:
+                raise ParseError
+            self.tree.pop()
+
+    def value(self):
+        self.tree.append(ValueNode(self.token[1]))
+        self.consume()
+        self.tree.pop()
+
+    def atom(self):
+        if self.token[1] not in atoms:
+            raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Unrecognised symbol @%s" % self.token[1])
+        self.tree.append(AtomNode(atoms[self.token[1]]))
+        self.consume()
+        self.tree.pop()
+
+    def expr_start(self):
+        self.expr_builder = ExpressionBuilder(self.tokenizer)
+        self.expr_builders.append(self.expr_builder)
+        self.expr()
+        expression = self.expr_builder.finish()
+        self.expr_builders.pop()
+        self.expr_builder = self.expr_builders[-1] if self.expr_builders else None
+        if self.expr_builder:
+            self.expr_builder.operands[-1].children[-1].append(expression)
+        else:
+            self.tree.append(expression)
+            self.tree.pop()
+
+    def expr(self):
+        self.expr_operand()
+        while (self.token[0] == token_types.ident and self.token[1] in binary_operators):
+            self.expr_bin_op()
+            self.expr_operand()
+
+    def expr_operand(self):
+        if self.token == (token_types.paren, "("):
+            self.consume()
+            self.expr_builder.left_paren()
+            self.expr()
+            self.expect(token_types.paren, ")")
+            self.expr_builder.right_paren()
+        elif self.token[0] == token_types.ident and self.token[1] in unary_operators:
+            self.expr_unary_op()
+            self.expr_operand()
+        elif self.token[0] in [token_types.string, token_types.ident]:
+            self.expr_value()
+        elif self.token[0] == token_types.number:
+            self.expr_number()
+        else:
+            raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Unrecognised operand")
+
+    def expr_unary_op(self):
+        if self.token[1] in unary_operators:
+            self.expr_builder.push_operator(UnaryOperatorNode(self.token[1]))
+            self.consume()
+        else:
+            raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Expected unary operator")
+
+    def expr_bin_op(self):
+        if self.token[1] in binary_operators:
+            self.expr_builder.push_operator(BinaryOperatorNode(self.token[1]))
+            self.consume()
+        else:
+            raise ParseError(self.tokenizer.filename, self.tokenizer.line_number, "Expected binary operator")
+
+    def expr_value(self):
+        node_type = {token_types.string: StringNode,
+                     token_types.ident: VariableNode}[self.token[0]]
+        self.expr_builder.push_operand(node_type(self.token[1]))
+        self.consume()
+        if self.token == (token_types.paren, "["):
+            self.consume()
+            self.expr_builder.operands[-1].append(IndexNode())
+            self.expr_start()
+            self.expect(token_types.paren, "]")
+
+    def expr_number(self):
+        self.expr_builder.push_operand(NumberNode(self.token[1]))
+        self.consume()
+
+
+class Treebuilder(object):
+    def __init__(self, root):
+        self.root = root
+        self.node = root
+
+    def append(self, node):
+        self.node.append(node)
+        self.node = node
+        return node
+
+    def pop(self):
+        node = self.node
+        self.node = self.node.parent
+        return node
+
+
+class ExpressionBuilder(object):
+    def __init__(self, tokenizer):
+        self.operands = []
+        self.operators = [None]
+        self.tokenizer = tokenizer
+
+    def finish(self):
+        while self.operators[-1] is not None:
+            self.pop_operator()
+        rv = self.pop_operand()
+        assert self.is_empty()
+        return rv
+
+    def left_paren(self):
+        self.operators.append(None)
+
+    def right_paren(self):
+        while self.operators[-1] is not None:
+            self.pop_operator()
+            if not self.operators:
+                raise ParseError(self.tokenizer.filename, self.tokenizer.line,
+                                 "Unbalanced parens")
+
+        assert self.operators.pop() is None
+
+    def push_operator(self, operator):
+        assert operator is not None
+        while self.precedence(self.operators[-1]) > self.precedence(operator):
+            self.pop_operator()
+
+        self.operators.append(operator)
+
+    def pop_operator(self):
+        operator = self.operators.pop()
+        if isinstance(operator, BinaryOperatorNode):
+            operand_1 = self.operands.pop()
+            operand_0 = self.operands.pop()
+            self.operands.append(BinaryExpressionNode(operator, operand_0, operand_1))
+        else:
+            operand_0 = self.operands.pop()
+            self.operands.append(UnaryExpressionNode(operator, operand_0))
+
+    def push_operand(self, node):
+        self.operands.append(node)
+
+    def pop_operand(self):
+        return self.operands.pop()
+
+    def is_empty(self):
+        return len(self.operands) == 0 and all(item is None for item in self.operators)
+
+    def precedence(self, operator):
+        if operator is None:
+            return 0
+        return precedence(operator)
+
+
+def parse(stream):
+    p = Parser()
+    return p.parse(stream)

testing/web-platform/harness/wptrunner/wptmanifest/serializer.py

@@ -0,0 +1,140 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from node import NodeVisitor, ValueNode, ListNode, BinaryExpressionNode
+from parser import atoms, precedence
+
+atom_names = {v:"@%s" % k for (k,v) in atoms.iteritems()}
+
+named_escapes = set(["\a", "\b", "\f", "\n", "\r", "\t", "\v"])
+
+def escape(string, extras=""):
+    rv = ""
+    for c in string:
+        if c in named_escapes:
+            rv += c.encode("unicode_escape")
+        elif c == "\\":
+            rv += "\\\\"
+        elif c < '\x20':
+            rv += "\\x%02x" % ord(c)
+        elif c in extras:
+            rv += "\\" + c
+        else:
+            rv += c
+    return rv.encode("utf8")
+
+
+class ManifestSerializer(NodeVisitor):
+    def __init__(self, skip_empty_data=False):
+        self.skip_empty_data = skip_empty_data
+
+    def serialize(self, root):
+        self.indent = 2
+        rv = "\n".join(self.visit(root))
+        if rv[-1] != "\n":
+            rv = rv + "\n"
+        return rv
+
+    def visit_DataNode(self, node):
+        rv = []
+        if not self.skip_empty_data or node.children:
+            if node.data:
+                rv.append("[%s]" % escape(node.data, extras="]"))
+                indent = self.indent * " "
+            else:
+                indent = ""
+
+            for child in node.children:
+                rv.extend("%s%s" % (indent if item else "", item) for item in self.visit(child))
+
+            if node.parent:
+                rv.append("")
+
+        return rv
+
+    def visit_KeyValueNode(self, node):
+        rv = [escape(node.data, ":") + ":"]
+        indent = " " * self.indent
+
+        if len(node.children) == 1 and isinstance(node.children[0], (ValueNode, ListNode)):
+            rv[0] += " %s" % self.visit(node.children[0])[0]
+        else:
+            for child in node.children:
+                rv.append(indent + self.visit(child)[0])
+
+        return rv
+
+    def visit_ListNode(self, node):
+        rv = ["["]
+        rv.extend(", ".join(self.visit(child)[0] for child in node.children))
+        rv.append("]")
+        return ["".join(rv)]
+
+    def visit_ValueNode(self, node):
+        if "#" in node.data or (isinstance(node.parent, ListNode) and
+                                ("," in node.data or "]" in node.data)):
+            if "\"" in node.data:
+                quote = "'"
+            else:
+                quote = "\""
+        else:
+            quote = ""
+        return [quote + escape(node.data, extras=quote) + quote]
+
+    def visit_AtomNode(self, node):
+        return [atom_names[node.data]]
+
+    def visit_ConditionalNode(self, node):
+        return ["if %s: %s" % tuple(self.visit(item)[0] for item in node.children)]
+
+    def visit_StringNode(self, node):
+        rv = ["\"%s\"" % escape(node.data, extras="\"")]
+        for child in node.children:
+            rv[0] += self.visit(child)[0]
+        return rv
+
+    def visit_NumberNode(self, node):
+        return [str(node.data)]
+
+    def visit_VariableNode(self, node):
+        rv = escape(node.data)
+        for child in node.children:
+            rv += self.visit(child)
+        return [rv]
+
+    def visit_IndexNode(self, node):
+        assert len(node.children) == 1
+        return ["[%s]" % self.visit(node.children[0])[0]]
+
+    def visit_UnaryExpressionNode(self, node):
+        children = []
+        for child in node.children:
+            child_str = self.visit(child)[0]
+            if isinstance(child, BinaryExpressionNode):
+                child_str = "(%s)" % child_str
+            children.append(child_str)
+        return [" ".join(children)]
+
+    def visit_BinaryExpressionNode(self, node):
+        assert len(node.children) == 3
+        children = []
+        for child_index in [1, 0, 2]:
+            child = node.children[child_index]
+            child_str = self.visit(child)[0]
+            if (isinstance(child, BinaryExpressionNode) and
+                precedence(node.children[0]) < precedence(child.children[0])):
+                child_str = "(%s)" % child_str
+            children.append(child_str)
+        return [" ".join(children)]
+
+    def visit_UnaryOperatorNode(self, node):
+        return [str(node.data)]
+
+    def visit_BinaryOperatorNode(self, node):
+        return [str(node.data)]
+
+
+def serialize(tree, *args, **kwargs):
+    s = ManifestSerializer(*args, **kwargs)
+    return s.serialize(tree)

testing/web-platform/harness/wptrunner/wptmanifest/tests/__init__.py

@@ -0,0 +1,3 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.

testing/web-platform/harness/wptrunner/wptmanifest/tests/test_conditional.py

@@ -0,0 +1,150 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from cStringIO import StringIO
+
+from ..backends import conditional
+from ..node import BinaryExpressionNode, BinaryOperatorNode, VariableNode, NumberNode
+
+
+class TestConditional(unittest.TestCase):
+    def parse(self, input_str):
+        return self.parser.parse(StringIO(input_str))
+
+    def compile(self, input_text):
+        return conditional.compile(input_text)
+
+    def test_get_0(self):
+        data = """
+key: value
+
+[Heading 1]
+  other_key:
+    if a == 1: value_1
+    if a == 2: value_2
+    value_3
+"""
+
+        manifest = self.compile(data)
+
+        self.assertEquals(manifest.get("key"), "value")
+        children = list(item for item in manifest.iterchildren())
+        self.assertEquals(len(children), 1)
+        section = children[0]
+        self.assertEquals(section.name, "Heading 1")
+
+        self.assertEquals(section.get("other_key", {"a": 1}), "value_1")
+        self.assertEquals(section.get("other_key", {"a": 2}), "value_2")
+        self.assertEquals(section.get("other_key", {"a": 7}), "value_3")
+        self.assertEquals(section.get("key"), "value")
+
+    def test_get_1(self):
+        data = """
+key: value
+
+[Heading 1]
+  other_key:
+    if a == "1": value_1
+    if a == 2: value_2
+    value_3
+"""
+
+        manifest = self.compile(data)
+
+        children = list(item for item in manifest.iterchildren())
+        section = children[0]
+
+        self.assertEquals(section.get("other_key", {"a": "1"}), "value_1")
+        self.assertEquals(section.get("other_key", {"a": 1}), "value_3")
+
+    def test_get_2(self):
+        data = """
+key:
+  if a[1] == "b": value_1
+  if a[1] == 2: value_2
+  value_3
+"""
+
+        manifest = self.compile(data)
+
+        self.assertEquals(manifest.get("key", {"a": "ab"}), "value_1")
+        self.assertEquals(manifest.get("key", {"a": [1, 2]}), "value_2")
+
+    def test_get_3(self):
+        data = """
+key:
+  if a[1] == "ab"[1]: value_1
+  if a[1] == 2: value_2
+  value_3
+"""
+
+        manifest = self.compile(data)
+
+        self.assertEquals(manifest.get("key", {"a": "ab"}), "value_1")
+        self.assertEquals(manifest.get("key", {"a": [1, 2]}), "value_2")
+
+    def test_set_0(self):
+        data = """
+key:
+  if a == "a": value_1
+  if a == "b": value_2
+  value_3
+"""
+        manifest = self.compile(data)
+
+        manifest.set("new_key", "value_new")
+
+        self.assertEquals(manifest.get("new_key"), "value_new")
+
+    def test_set_1(self):
+        data = """
+key:
+  if a == "a": value_1
+  if a == "b": value_2
+  value_3
+"""
+
+        manifest = self.compile(data)
+
+        manifest.set("key", "value_new")
+
+        self.assertEquals(manifest.get("key"), "value_new")
+        self.assertEquals(manifest.get("key", {"a": "a"}), "value_1")
+
+    def test_set_2(self):
+        data = """
+key:
+  if a == "a": value_1
+  if a == "b": value_2
+  value_3
+"""
+
+        manifest = self.compile(data)
+
+        expr = BinaryExpressionNode(BinaryOperatorNode("=="),
+                                    VariableNode("a"),
+                                    NumberNode("1"))
+
+        manifest.set("key", "value_new", expr)
+
+        self.assertEquals(manifest.get("key", {"a": 1}), "value_new")
+        self.assertEquals(manifest.get("key", {"a": "a"}), "value_1")
+
+    def test_api_0(self):
+        data = """
+key:
+  if a == 1.5: value_1
+  value_2
+key_1: other_value
+"""
+        manifest = self.compile(data)
+
+        self.assertFalse(manifest.is_empty)
+        self.assertEquals(manifest.root, manifest)
+        self.assertTrue(manifest.has_key("key_1"))
+        self.assertFalse(manifest.has_key("key_2"))
+
+        self.assertEquals(set(manifest.iterkeys()), set(["key", "key_1"]))

testing/web-platform/harness/wptrunner/wptmanifest/tests/test_parser.py

@@ -0,0 +1,79 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from cStringIO import StringIO
+
+from .. import parser
+
+# There aren't many tests here because it turns out to be way more convenient to
+# use test_serializer for the majority of cases
+
+
+class TestExpression(unittest.TestCase):
+    def setUp(self):
+        self.parser = parser.Parser()
+
+    def parse(self, input_str):
+        return self.parser.parse(StringIO(input_str))
+
+    def compare(self, input_text, expected):
+        actual = self.parse(input_text)
+        self.match(expected, actual)
+
+    def match(self, expected_node, actual_node):
+        self.assertEquals(expected_node[0], actual_node.__class__.__name__)
+        self.assertEquals(expected_node[1], actual_node.data)
+        self.assertEquals(len(expected_node[2]), len(actual_node.children))
+        for expected_child, actual_child in zip(expected_node[2], actual_node.children):
+            self.match(expected_child, actual_child)
+
+    def test_expr_0(self):
+        self.compare(
+            """
+key:
+  if x == 1 : value""",
+            ["DataNode", None,
+             [["KeyValueNode", "key",
+               [["ConditionalNode", None,
+                 [["BinaryExpressionNode", None,
+                   [["BinaryOperatorNode", "==", []],
+                    ["VariableNode", "x", []],
+                       ["NumberNode", "1", []]
+                    ]],
+                     ["ValueNode", "value", []],
+                  ]]]]]]
+        )
+
+    def test_expr_1(self):
+        self.compare(
+            """
+key:
+  if not x and y : value""",
+            ["DataNode", None,
+             [["KeyValueNode", "key",
+               [["ConditionalNode", None,
+                 [["BinaryExpressionNode", None,
+                   [["BinaryOperatorNode", "and", []],
+                    ["UnaryExpressionNode", None,
+                       [["UnaryOperatorNode", "not", []],
+                        ["VariableNode", "x", []]
+                        ]],
+                       ["VariableNode", "y", []]
+                    ]],
+                     ["ValueNode", "value", []],
+                  ]]]]]]
+        )
+
+    def test_atom_0(self):
+        with self.assertRaises(parser.ParseError):
+            self.parse("key: @Unknown")
+
+    def test_atom_1(self):
+        with self.assertRaises(parser.ParseError):
+            self.parse("key: @true")
+
+if __name__ == "__main__":
+    unittest.main()

testing/web-platform/harness/wptrunner/wptmanifest/tests/test_serializer.py

@@ -0,0 +1,227 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from cStringIO import StringIO
+
+from .. import parser, serializer
+
+
+class TokenizerTest(unittest.TestCase):
+    def setUp(self):
+        self.serializer = serializer.ManifestSerializer()
+        self.parser = parser.Parser()
+
+    def serialize(self, input_str):
+        return self.serializer.serialize(self.parser.parse(input_str))
+
+    def compare(self, input_str, expected=None):
+        if expected is None:
+            expected = input_str
+        expected = expected.encode("utf8")
+        actual = self.serialize(input_str)
+        self.assertEquals(actual, expected)
+
+    def test_0(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key: other_value
+""")
+
+    def test_1(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a or b: other_value
+""")
+
+    def test_2(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a or b: other_value
+    fallback_value
+""")
+
+    def test_3(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a == 1: other_value
+    fallback_value
+""")
+
+    def test_4(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a == "1": other_value
+    fallback_value
+""")
+
+    def test_5(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a == "abc"[1]: other_value
+    fallback_value
+""")
+
+    def test_6(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a == "abc"[c]: other_value
+    fallback_value
+""")
+
+    def test_7(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if (a or b) and c: other_value
+    fallback_value
+""",
+"""key: value
+[Heading 1]
+  other_key:
+    if a or b and c: other_value
+    fallback_value
+""")
+
+    def test_8(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if a or (b and c): other_value
+    fallback_value
+""")
+
+    def test_9(self):
+        self.compare("""key: value
+[Heading 1]
+  other_key:
+    if not (a and b): other_value
+    fallback_value
+""")
+
+    def test_10(self):
+        self.compare("""key: value
+[Heading 1]
+  some_key: some_value
+
+[Heading 2]
+  other_key: other_value
+""")
+
+    def test_11(self):
+        self.compare("""key:
+  if not a and b and c and d: true
+""")
+
+    def test_12(self):
+        self.compare("""[Heading 1]
+  key: [a:1, b:2]
+""")
+
+    def test_13(self):
+        self.compare("""key: [a:1, "b:#"]
+""")
+
+    def test_14(self):
+        self.compare("""key: [","]
+""")
+
+    def test_15(self):
+        self.compare("""key: ,
+""")
+
+    def test_16(self):
+        self.compare("""key: ["]", b]
+""")
+
+    def test_17(self):
+        self.compare("""key: ]
+""")
+
+    def test_18(self):
+        self.compare("""key: \]
+        """, """key: ]
+""")
+
+    def test_escape_0(self):
+        self.compare(r"""k\t\:y: \a\b\f\n\r\t\v""",
+                     r"""k\t\:y: \x07\x08\x0c\n\r\t\x0b
+""")
+
+    def test_escape_1(self):
+        self.compare(r"""k\x00: \x12A\x45""",
+                     r"""k\x00: \x12AE
+""")
+
+    def test_escape_2(self):
+        self.compare(r"""k\u0045y: \u1234A\uABc6""",
+                     u"""kEy: \u1234A\uabc6
+""")
+
+    def test_escape_3(self):
+        self.compare(r"""k\u0045y: \u1234A\uABc6""",
+                     u"""kEy: \u1234A\uabc6
+""")
+
+    def test_escape_4(self):
+        self.compare(r"""key: '\u1234A\uABc6'""",
+                     u"""key: \u1234A\uabc6
+""")
+
+    def test_escape_5(self):
+        self.compare(r"""key: [\u1234A\uABc6]""",
+                     u"""key: [\u1234A\uabc6]
+""")
+
+    def test_escape_6(self):
+        self.compare(r"""key: [\u1234A\uABc6\,]""",
+                     u"""key: ["\u1234A\uabc6,"]
+""")
+
+    def test_escape_7(self):
+        self.compare(r"""key: [\,\]\#]""",
+                     r"""key: [",]#"]
+""")
+
+    def test_escape_8(self):
+        self.compare(r"""key: \#""",
+                     r"""key: "#"
+""")
+
+    def test_escape_9(self):
+        self.compare(r"""key: \U10FFFFabc""",
+                     u"""key: \U0010FFFFabc
+""")
+
+    def test_escape_10(self):
+        self.compare(r"""key: \u10FFab""",
+                     u"""key: \u10FFab
+""")
+
+    def test_escape_11(self):
+        self.compare(r"""key: \\ab
+""")
+
+    def test_atom_1(self):
+            self.compare(r"""key: @True
+""")
+
+    def test_atom_2(self):
+            self.compare(r"""key: @False
+""")
+
+    def test_atom_3(self):
+            self.compare(r"""key: @Reset
+""")
+
+    def test_atom_4(self):
+        self.compare(r"""key: [a, @Reset, b]
+""")

testing/web-platform/harness/wptrunner/wptmanifest/tests/test_static.py

@@ -0,0 +1,105 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import unittest
+
+from cStringIO import StringIO
+
+from ..backends import static
+
+# There aren't many tests here because it turns out to be way more convenient to
+# use test_serializer for the majority of cases
+
+
+class TestStatic(unittest.TestCase):
+    def parse(self, input_str):
+        return self.parser.parse(StringIO(input_str))
+
+    def compile(self, input_text, input_data):
+        return static.compile(input_text, input_data)
+
+    def test_get_0(self):
+        data = """
+key: value
+
+[Heading 1]
+  other_key:
+    if a == 1: value_1
+    if a == 2: value_2
+    value_3
+"""
+
+        manifest = self.compile(data, {"a": 2})
+
+        self.assertEquals(manifest.get("key"), "value")
+        children = list(item for item in manifest.iterchildren())
+        self.assertEquals(len(children), 1)
+        section = children[0]
+        self.assertEquals(section.name, "Heading 1")
+
+        self.assertEquals(section.get("other_key"), "value_2")
+        self.assertEquals(section.get("key"), "value")
+
+    def test_get_1(self):
+        data = """
+key: value
+
+[Heading 1]
+  other_key:
+    if a == 1: value_1
+    if a == 2: value_2
+    value_3
+"""
+        manifest = self.compile(data, {"a": 3})
+
+        children = list(item for item in manifest.iterchildren())
+        section = children[0]
+        self.assertEquals(section.get("other_key"), "value_3")
+
+    def test_get_3(self):
+        data = """key:
+  if a == "1": value_1
+  if a[0] == "ab"[0]: value_2
+"""
+        manifest = self.compile(data, {"a": "1"})
+        self.assertEquals(manifest.get("key"), "value_1")
+
+        manifest = self.compile(data, {"a": "ac"})
+        self.assertEquals(manifest.get("key"), "value_2")
+
+    def test_get_4(self):
+        data = """key:
+  if not a: value_1
+  value_2
+"""
+        manifest = self.compile(data, {"a": True})
+        self.assertEquals(manifest.get("key"), "value_2")
+
+        manifest = self.compile(data, {"a": False})
+        self.assertEquals(manifest.get("key"), "value_1")
+
+    def test_api(self):
+        data = """key:
+  if a == 1.5: value_1
+  value_2
+key_1: other_value
+"""
+        manifest = self.compile(data, {"a": 1.5})
+
+        self.assertFalse(manifest.is_empty)
+        self.assertEquals(manifest.root, manifest)
+        self.assertTrue(manifest.has_key("key_1"))
+        self.assertFalse(manifest.has_key("key_2"))
+
+        self.assertEquals(set(manifest.iterkeys()), set(["key", "key_1"]))
+        self.assertEquals(set(manifest.itervalues()), set(["value_1", "other_value"]))
+
+    def test_is_empty_1(self):
+        data = """
+[Section]
+  [Subsection]
+"""
+        manifest = self.compile(data, {})
+
+        self.assertTrue(manifest.is_empty)

testing/web-platform/harness/wptrunner/wptmanifest/tests/test_tokenizer.py

@@ -0,0 +1,361 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import os
+import unittest
+
+sys.path.insert(0, os.path.abspath(".."))
+from cStringIO import StringIO
+
+from .. import parser
+from ..parser import token_types
+
+
+class TokenizerTest(unittest.TestCase):
+    def setUp(self):
+        self.tokenizer = parser.Tokenizer()
+
+    def tokenize(self, input_str):
+        rv = []
+        for item in self.tokenizer.tokenize(StringIO(input_str)):
+            rv.append(item)
+            if item[0] == token_types.eof:
+                break
+        return rv
+
+    def compare(self, input_text, expected):
+        expected = expected + [(token_types.eof, None)]
+        actual = self.tokenize(input_text)
+        self.assertEquals(actual, expected)
+
+    def test_heading_0(self):
+        self.compare("""[Heading text]""",
+                     [(token_types.paren, "["),
+                      (token_types.string, "Heading text"),
+                      (token_types.paren, "]")])
+
+    def test_heading_1(self):
+        self.compare("""[Heading [text\]]""",
+                     [(token_types.paren, "["),
+                      (token_types.string, "Heading [text]"),
+                      (token_types.paren, "]")])
+
+    def test_heading_2(self):
+        self.compare("""[Heading #text]""",
+                     [(token_types.paren, "["),
+                      (token_types.string, "Heading #text"),
+                      (token_types.paren, "]")])
+
+    def test_heading_3(self):
+        self.compare("""[Heading [\]text]""",
+                     [(token_types.paren, "["),
+                      (token_types.string, "Heading []text"),
+                      (token_types.paren, "]")])
+
+    def test_heading_4(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("[Heading")
+
+    def test_heading_5(self):
+        self.compare("""[Heading [\]text] #comment""",
+                     [(token_types.paren, "["),
+                      (token_types.string, "Heading []text"),
+                      (token_types.paren, "]")])
+
+    def test_heading_6(self):
+        self.compare(r"""[Heading \ttext]""",
+                     [(token_types.paren, "["),
+                      (token_types.string, "Heading \ttext"),
+                      (token_types.paren, "]")])
+
+    def test_key_0(self):
+        self.compare("""key:value""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "value")])
+
+    def test_key_1(self):
+        self.compare("""key  :  value""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "value")])
+
+    def test_key_2(self):
+        self.compare("""key  :  val ue""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "val ue")])
+
+    def test_key_3(self):
+        self.compare("""key: value#comment""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "value")])
+
+    def test_key_4(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""ke y: value""")
+
+    def test_key_5(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key""")
+
+    def test_key_6(self):
+        self.compare("""key: "value\"""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "value")])
+
+    def test_key_7(self):
+        self.compare("""key: 'value'""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "value")])
+
+    def test_key_8(self):
+        self.compare("""key: "#value\"""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "#value")])
+
+    def test_key_9(self):
+        self.compare("""key: '#value\'""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, "#value")])
+
+    def test_key_10(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key: "value""")
+
+    def test_key_11(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key: 'value""")
+
+    def test_key_12(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key: 'value""")
+
+    def test_key_13(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key: 'value' abc""")
+
+    def test_key_14(self):
+        self.compare(r"""key: \\nb""",
+                     [(token_types.string, "key"),
+                      (token_types.separator, ":"),
+                      (token_types.string, r"\nb")])
+
+    def test_list_0(self):
+        self.compare(
+"""
+key: []""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.list_start, "["),
+             (token_types.list_end, "]")])
+
+    def test_list_1(self):
+        self.compare(
+"""
+key: [a, "b"]""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.list_start, "["),
+             (token_types.string, "a"),
+             (token_types.string, "b"),
+             (token_types.list_end, "]")])
+
+    def test_list_2(self):
+        self.compare(
+"""
+key: [a,
+      b]""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.list_start, "["),
+             (token_types.string, "a"),
+             (token_types.string, "b"),
+             (token_types.list_end, "]")])
+
+    def test_list_3(self):
+        self.compare(
+"""
+key: [a, #b]
+      c]""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.list_start, "["),
+             (token_types.string, "a"),
+             (token_types.string, "c"),
+             (token_types.list_end, "]")])
+
+    def test_list_4(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key: [a #b]
+            c]""")
+
+    def test_list_5(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize("""key: [a \\
+            c]""")
+
+    def test_list_6(self):
+        self.compare(
+"""key: [a , b]""",
+        [(token_types.string, "key"),
+         (token_types.separator, ":"),
+         (token_types.list_start, "["),
+         (token_types.string, "a"),
+         (token_types.string, "b"),
+         (token_types.list_end, "]")])
+
+    def test_expr_0(self):
+        self.compare(
+"""
+key:
+  if cond == 1: value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.ident, "cond"),
+             (token_types.ident, "=="),
+             (token_types.number, "1"),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+    def test_expr_1(self):
+        self.compare(
+"""
+key:
+  if cond == 1: value1
+  value2""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.ident, "cond"),
+             (token_types.ident, "=="),
+             (token_types.number, "1"),
+             (token_types.separator, ":"),
+             (token_types.string, "value1"),
+             (token_types.string, "value2")])
+
+    def test_expr_2(self):
+        self.compare(
+"""
+key:
+  if cond=="1": value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.ident, "cond"),
+             (token_types.ident, "=="),
+             (token_types.string, "1"),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+    def test_expr_3(self):
+        self.compare(
+"""
+key:
+  if cond==1.1: value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.ident, "cond"),
+             (token_types.ident, "=="),
+             (token_types.number, "1.1"),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+    def test_expr_4(self):
+        self.compare(
+            """
+key:
+  if cond==1.1 and cond2 == "a": value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.ident, "cond"),
+             (token_types.ident, "=="),
+             (token_types.number, "1.1"),
+             (token_types.ident, "and"),
+             (token_types.ident, "cond2"),
+             (token_types.ident, "=="),
+             (token_types.string, "a"),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+    def test_expr_5(self):
+        self.compare(
+"""
+key:
+  if (cond==1.1 ): value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.paren, "("),
+             (token_types.ident, "cond"),
+             (token_types.ident, "=="),
+             (token_types.number, "1.1"),
+             (token_types.paren, ")"),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+    def test_expr_6(self):
+        self.compare(
+"""
+key:
+  if "\\ttest": value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.string, "\ttest"),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+    def test_expr_7(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize(
+"""
+key:
+  if 1A: value""")
+
+    def test_expr_8(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize(
+"""
+key:
+  if 1a: value""")
+
+    def test_expr_9(self):
+        with self.assertRaises(parser.ParseError):
+            self.tokenize(
+"""
+key:
+  if 1.1.1: value""")
+
+    def test_expr_10(self):
+        self.compare(
+"""
+key:
+  if 1.: value""",
+            [(token_types.string, "key"),
+             (token_types.separator, ":"),
+             (token_types.group_start, None),
+             (token_types.ident, "if"),
+             (token_types.number, "1."),
+             (token_types.separator, ":"),
+             (token_types.string, "value")])
+
+if __name__ == "__main__":
+    unittest.main()