import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

services/common/tests/unit/head_global.js

@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu, manager: Cm} = Components;
+
+var gSyncProfile = do_get_profile();
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+Cu.import("resource://testing-common/AppInfo.jsm", this);
+updateAppInfo({
+  name: "XPCShell",
+  ID: "xpcshell@tests.mozilla.org",
+  version: "1",
+  platformVersion: "",
+});
+
+function addResourceAlias() {
+  Cu.import("resource://gre/modules/Services.jsm");
+  const handler = Services.io.getProtocolHandler("resource")
+                  .QueryInterface(Ci.nsIResProtocolHandler);
+
+  let modules = ["common", "crypto"];
+  for (let module of modules) {
+    let uri = Services.io.newURI("resource://gre/modules/services-" + module + "/",
+                                 null, null);
+    handler.setSubstitution("services-" + module, uri);
+  }
+}
+addResourceAlias();

services/common/tests/unit/head_helpers.js

@@ -0,0 +1,172 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://testing-common/httpd.js");
+Cu.import("resource://testing-common/services/common/logging.js");
+Cu.import("resource://testing-common/MockRegistrar.jsm");
+
+var btoa = Cu.import("resource://gre/modules/Log.jsm").btoa;
+var atob = Cu.import("resource://gre/modules/Log.jsm").atob;
+
+function do_check_empty(obj) {
+  do_check_attribute_count(obj, 0);
+}
+
+function do_check_attribute_count(obj, c) {
+  do_check_eq(c, Object.keys(obj).length);
+}
+
+function do_check_throws(aFunc, aResult, aStack) {
+  if (!aStack) {
+    try {
+      // We might not have a 'Components' object.
+      aStack = Components.stack.caller;
+    } catch (e) {}
+  }
+
+  try {
+    aFunc();
+  } catch (e) {
+    do_check_eq(e.result, aResult, aStack);
+    return;
+  }
+  do_throw("Expected result " + aResult + ", none thrown.", aStack);
+}
+
+
+/**
+ * Test whether specified function throws exception with expected
+ * result.
+ *
+ * @param func
+ *        Function to be tested.
+ * @param message
+ *        Message of expected exception. <code>null</code> for no throws.
+ */
+function do_check_throws_message(aFunc, aResult) {
+  try {
+    aFunc();
+  } catch (e) {
+    do_check_eq(e.message, aResult);
+    return;
+  }
+  do_throw("Expected an error, none thrown.");
+}
+
+/**
+ * Print some debug message to the console. All arguments will be printed,
+ * separated by spaces.
+ *
+ * @param [arg0, arg1, arg2, ...]
+ *        Any number of arguments to print out
+ * @usage _("Hello World") -> prints "Hello World"
+ * @usage _(1, 2, 3) -> prints "1 2 3"
+ */
+var _ = function(some, debug, text, to) {
+  print(Array.slice(arguments).join(" "));
+};
+
+function httpd_setup (handlers, port=-1) {
+  let server = new HttpServer();
+  for (let path in handlers) {
+    server.registerPathHandler(path, handlers[path]);
+  }
+  try {
+    server.start(port);
+  } catch (ex) {
+    _("==========================================");
+    _("Got exception starting HTTP server on port " + port);
+    _("Error: " + Log.exceptionStr(ex));
+    _("Is there a process already listening on port " + port + "?");
+    _("==========================================");
+    do_throw(ex);
+  }
+
+  // Set the base URI for convenience.
+  let i = server.identity;
+  server.baseURI = i.primaryScheme + "://" + i.primaryHost + ":" + i.primaryPort;
+
+  return server;
+}
+
+function httpd_handler(statusCode, status, body) {
+  return function handler(request, response) {
+    _("Processing request");
+    // Allow test functions to inspect the request.
+    request.body = readBytesFromInputStream(request.bodyInputStream);
+    handler.request = request;
+
+    response.setStatusLine(request.httpVersion, statusCode, status);
+    if (body) {
+      response.bodyOutputStream.write(body, body.length);
+    }
+  };
+}
+
+/*
+ * Read bytes string from an nsIInputStream.  If 'count' is omitted,
+ * all available input is read.
+ */
+function readBytesFromInputStream(inputStream, count) {
+  return CommonUtils.readBytesFromInputStream(inputStream, count);
+}
+
+/*
+ * Ensure exceptions from inside callbacks leads to test failures.
+ */
+function ensureThrows(func) {
+  return function() {
+    try {
+      func.apply(this, arguments);
+    } catch (ex) {
+      do_throw(ex);
+    }
+  };
+}
+
+/**
+ * Proxy auth helpers.
+ */
+
+/**
+ * Fake a PAC to prompt a channel replacement.
+ */
+var PACSystemSettings = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISystemProxySettings]),
+
+  // Replace this URI for each test to avoid caching. We want to ensure that
+  // each test gets a completely fresh setup.
+  mainThreadOnly: true,
+  PACURI: null,
+  getProxyForURI: function getProxyForURI(aURI) {
+    throw Cr.NS_ERROR_NOT_IMPLEMENTED;
+  }
+};
+
+var fakePACCID;
+function installFakePAC() {
+  _("Installing fake PAC.");
+  fakePACCID = MockRegistrar.register("@mozilla.org/system-proxy-settings;1",
+                                      PACSystemSettings);
+}
+
+function uninstallFakePAC() {
+  _("Uninstalling fake PAC.");
+  MockRegistrar.unregister(fakePACCID);
+}
+
+// Many tests do service.startOver() and don't expect the provider type to
+// change (whereas by default, a startOver will do exactly that so FxA is
+// subsequently used). The tests that know how to deal with
+// the Firefox Accounts identity hack things to ensure that still works.
+function ensureStartOverKeepsIdentity() {
+  Cu.import("resource://gre/modules/Services.jsm");
+  Services.prefs.setBoolPref("services.sync-testing.startOverKeepIdentity", true);
+  do_register_cleanup(function() {
+    Services.prefs.clearUserPref("services.sync-testing.startOverKeepIdentity");
+  });
+}
+ensureStartOverKeepsIdentity();

services/common/tests/unit/head_http.js

@@ -0,0 +1,29 @@
+ /* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function basic_auth_header(user, password) {
+  return "Basic " + btoa(user + ":" + CommonUtils.encodeUTF8(password));
+}
+
+function basic_auth_matches(req, user, password) {
+  if (!req.hasHeader("Authorization")) {
+    return false;
+  }
+
+  let expected = basic_auth_header(user, CommonUtils.encodeUTF8(password));
+  return req.getHeader("Authorization") == expected;
+}
+
+function httpd_basic_auth_handler(body, metadata, response) {
+  if (basic_auth_matches(metadata, "guest", "guest")) {
+    response.setStatusLine(metadata.httpVersion, 200, "OK, authorized");
+    response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false);
+  } else {
+    body = "This path exists and is protected - failed";
+    response.setStatusLine(metadata.httpVersion, 401, "Unauthorized");
+    response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false);
+  }
+  response.bodyOutputStream.write(body, body.length);
+}

services/common/tests/unit/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+TEST_DIRS += [
+    'test_blocklist_signatures'
+]

services/common/tests/unit/test_async_chain.js

@@ -0,0 +1,30 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+
+function run_test() {
+  _("Chain a few async methods, making sure the 'this' object is correct.");
+
+  let methods = {
+    save: function(x, callback) {
+      this.x = x;
+      callback(x);
+    },
+    addX: function(x, callback) {
+      callback(x + this.x);
+    },
+    double: function(x, callback) {
+      callback(x * 2);
+    },
+    neg: function(x, callback) {
+      callback(-x);
+    }
+  };
+  methods.chain = Async.chain;
+
+  // ((1 + 1 + 1) * (-1) + 1) * 2 + 1 = -3
+  methods.chain(methods.save, methods.addX, methods.addX, methods.neg,
+                methods.addX, methods.double, methods.addX, methods.save)(1);
+  do_check_eq(methods.x, -3);
+}

services/common/tests/unit/test_async_querySpinningly.js

@@ -0,0 +1,103 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/utils.js");
+
+_("Make sure querySpinningly will synchronously fetch rows for a query asyncly");
+
+const SQLITE_CONSTRAINT_VIOLATION = 19;  // http://www.sqlite.org/c3ref/c_abort.html
+
+var Svc = {};
+XPCOMUtils.defineLazyServiceGetter(Svc, "Form",
+                                   "@mozilla.org/satchel/form-history;1",
+                                   "nsIFormHistory2");
+
+function querySpinningly(query, names) {
+  let q = Svc.Form.DBConnection.createStatement(query);
+  let r = Async.querySpinningly(q, names);
+  q.finalize();
+  return r;
+}
+
+function run_test() {
+  initTestLogging("Trace");
+
+  _("Make sure the call is async and allows other events to process");
+  let isAsync = false;
+  CommonUtils.nextTick(function() { isAsync = true; });
+  do_check_false(isAsync);
+
+  _("Empty out the formhistory table");
+  let r0 = querySpinningly("DELETE FROM moz_formhistory");
+  do_check_eq(r0, null);
+
+  _("Make sure there's nothing there");
+  let r1 = querySpinningly("SELECT 1 FROM moz_formhistory");
+  do_check_eq(r1, null);
+
+  _("Insert a row");
+  let r2 = querySpinningly("INSERT INTO moz_formhistory (fieldname, value) VALUES ('foo', 'bar')");
+  do_check_eq(r2, null);
+
+  _("Request a known value for the one row");
+  let r3 = querySpinningly("SELECT 42 num FROM moz_formhistory", ["num"]);
+  do_check_eq(r3.length, 1);
+  do_check_eq(r3[0].num, 42);
+
+  _("Get multiple columns");
+  let r4 = querySpinningly("SELECT fieldname, value FROM moz_formhistory", ["fieldname", "value"]);
+  do_check_eq(r4.length, 1);
+  do_check_eq(r4[0].fieldname, "foo");
+  do_check_eq(r4[0].value, "bar");
+
+  _("Get multiple columns with a different order");
+  let r5 = querySpinningly("SELECT fieldname, value FROM moz_formhistory", ["value", "fieldname"]);
+  do_check_eq(r5.length, 1);
+  do_check_eq(r5[0].fieldname, "foo");
+  do_check_eq(r5[0].value, "bar");
+
+  _("Add multiple entries (sqlite doesn't support multiple VALUES)");
+  let r6 = querySpinningly("INSERT INTO moz_formhistory (fieldname, value) SELECT 'foo', 'baz' UNION SELECT 'more', 'values'");
+  do_check_eq(r6, null);
+
+  _("Get multiple rows");
+  let r7 = querySpinningly("SELECT fieldname, value FROM moz_formhistory WHERE fieldname = 'foo'", ["fieldname", "value"]);
+  do_check_eq(r7.length, 2);
+  do_check_eq(r7[0].fieldname, "foo");
+  do_check_eq(r7[1].fieldname, "foo");
+
+  _("Make sure updates work");
+  let r8 = querySpinningly("UPDATE moz_formhistory SET value = 'updated' WHERE fieldname = 'more'");
+  do_check_eq(r8, null);
+
+  _("Get the updated");
+  let r9 = querySpinningly("SELECT value, fieldname FROM moz_formhistory WHERE fieldname = 'more'", ["fieldname", "value"]);
+  do_check_eq(r9.length, 1);
+  do_check_eq(r9[0].fieldname, "more");
+  do_check_eq(r9[0].value, "updated");
+
+  _("Grabbing fewer fields than queried is fine");
+  let r10 = querySpinningly("SELECT value, fieldname FROM moz_formhistory", ["fieldname"]);
+  do_check_eq(r10.length, 3);
+
+  _("Generate an execution error");
+  let query = "INSERT INTO moz_formhistory (fieldname, value) VALUES ('one', NULL)";
+  let stmt = Svc.Form.DBConnection.createStatement(query);
+  let r11, except; ;
+  try {
+    r11 = Async.querySpinningly(stmt);
+  } catch(e) {
+    except = e;
+  }
+  stmt.finalize()
+  do_check_true(!!except);
+  do_check_eq(except.result, SQLITE_CONSTRAINT_VIOLATION);
+
+  _("Cleaning up");
+  querySpinningly("DELETE FROM moz_formhistory");
+
+  _("Make sure the timeout got to run before this function ends");
+  do_check_true(isAsync);
+}

services/common/tests/unit/test_blocklist_certificates.js

@@ -0,0 +1,224 @@
+const { Constructor: CC } = Components;
+
+Cu.import("resource://testing-common/httpd.js");
+
+const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+let server;
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let kintoClient;
+
+function do_get_kinto_collection(collectionName) {
+  if (!kintoClient) {
+    let config = {
+      // Set the remote to be some server that will cause test failure when
+      // hit since we should never hit the server directly, only via maybeSync()
+      remote: "https://firefox.settings.services.mozilla.com/v1/",
+      // Set up the adapter and bucket as normal
+      adapter: FirefoxAdapter,
+      bucket: "blocklists"
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection(collectionName);
+}
+
+// Some simple tests to demonstrate that the logic inside maybeSync works
+// correctly and that simple kinto operations are working as expected. There
+// are more tests for core Kinto.js (and its storage adapter) in the
+// xpcshell tests under /services/common
+add_task(function* test_something(){
+  const configPath = "/v1/";
+  const recordsPath = "/v1/buckets/blocklists/collections/certificates/records";
+
+  Services.prefs.setCharPref("services.settings.server",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // register a handler
+  function handleResponse (request, response) {
+    try {
+      const sample = getSampleResponse(request, server.identity.primaryPort);
+      if (!sample) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sample.status.status,
+                             sample.status.statusText);
+      // send the headers
+      for (let headerLine of sample.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sample.responseBody);
+    } catch (e) {
+      do_print(e);
+    }
+  }
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(recordsPath, handleResponse);
+
+  // Test an empty db populates
+  let result = yield OneCRLBlocklistClient.maybeSync(2000, Date.now());
+
+  // Open the collection, verify it's been populated:
+  // Our test data has a single record; it should be in the local collection
+  let collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  let list = yield collection.list();
+  do_check_eq(list.data.length, 1);
+  yield collection.db.close();
+
+  // Test the db is updated when we call again with a later lastModified value
+  result = yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
+
+  // Open the collection, verify it's been updated:
+  // Our test data now has two records; both should be in the local collection
+  collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  list = yield collection.list();
+  do_check_eq(list.data.length, 3);
+  yield collection.db.close();
+
+  // Try to maybeSync with the current lastModified value - no connection
+  // should be attempted.
+  // Clear the kinto base pref so any connections will cause a test failure
+  Services.prefs.clearUserPref("services.settings.server");
+  yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
+
+  // Try again with a lastModified value at some point in the past
+  yield OneCRLBlocklistClient.maybeSync(3000, Date.now());
+
+  // Check the OneCRL check time pref is modified, even if the collection
+  // hasn't changed
+  Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
+  yield OneCRLBlocklistClient.maybeSync(3000, Date.now());
+  let newValue = Services.prefs.getIntPref("services.blocklist.onecrl.checked");
+  do_check_neq(newValue, 0);
+
+  // Check that a sync completes even when there's bad data in the
+  // collection. This will throw on fail, so just calling maybeSync is an
+  // acceptible test.
+  Services.prefs.setCharPref("services.settings.server",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+  yield OneCRLBlocklistClient.maybeSync(5000, Date.now());
+});
+
+function run_test() {
+  // Ensure that signature verification is disabled to prevent interference
+  // with basic certificate sync tests
+  Services.prefs.setBoolPref("services.blocklist.signing.enforced", false);
+
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(() => { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName": "MEQxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xHjAcBgNVBAMTFXRoYXd0ZSBFViBTU0wgQ0EgLSBHMw==",
+        "serialNumber":"CrTHPEE6AZSfI3jysin2bA==",
+        "id":"78cf8900-fdea-4ce5-f8fb-b78710617718",
+        "last_modified":3000
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName":"MFkxCzAJBgNVBAYTAk5MMR4wHAYDVQQKExVTdGFhdCBkZXIgTmVkZXJsYW5kZW4xKjAoBgNVBAMTIVN0YWF0IGRlciBOZWRlcmxhbmRlbiBPdmVyaGVpZCBDQQ",
+        "serialNumber":"ATFpsA==",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02c",
+        "last_modified":4000
+      },{
+        "subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
+        "pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02d",
+        "last_modified":4000
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"5000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName":"not a base64 encoded issuer",
+        "serialNumber":"not a base64 encoded serial",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02e",
+        "last_modified":5000
+      },{
+        "subject":"not a base64 encoded subject",
+        "pubKeyHash":"not a base64 encoded pubKeyHash",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02f",
+        "last_modified":5000
+      },{
+        "subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
+        "pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02g",
+        "last_modified":5000
+      }]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/test_blocklist_clients.js

@@ -0,0 +1,412 @@
+const { Constructor: CC } = Components;
+
+const KEY_PROFILEDIR = "ProfD";
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://testing-common/httpd.js");
+Cu.import("resource://gre/modules/Timer.jsm");
+const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm");
+const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
+
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const BlocklistClients = Cu.import("resource://services-common/blocklist-clients.js");
+
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+const gBlocklistClients = [
+  {client: BlocklistClients.AddonBlocklistClient, filename: BlocklistClients.FILENAME_ADDONS_JSON, testData: ["i808","i720", "i539"]},
+  {client: BlocklistClients.PluginBlocklistClient, filename: BlocklistClients.FILENAME_PLUGINS_JSON, testData: ["p1044","p32","p28"]},
+  {client: BlocklistClients.GfxBlocklistClient, filename: BlocklistClients.FILENAME_GFX_JSON, testData: ["g204","g200","g36"]},
+];
+
+
+let server;
+let kintoClient;
+
+function kintoCollection(collectionName) {
+  if (!kintoClient) {
+    const Kinto = loadKinto();
+    const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+    const config = {
+      // Set the remote to be some server that will cause test failure when
+      // hit since we should never hit the server directly, only via maybeSync()
+      remote: "https://firefox.settings.services.mozilla.com/v1/",
+      adapter: FirefoxAdapter,
+      bucket: "blocklists"
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection(collectionName);
+}
+
+function* readJSON(filepath) {
+  const binaryData = yield OS.File.read(filepath);
+  const textData = (new TextDecoder()).decode(binaryData);
+  return Promise.resolve(JSON.parse(textData));
+}
+
+function* clear_state() {
+  for (let {client} of gBlocklistClients) {
+    // Remove last server times.
+    Services.prefs.clearUserPref(client.lastCheckTimePref);
+
+    // Clear local DB.
+    const collection = kintoCollection(client.collectionName);
+    try {
+      yield collection.db.open();
+      yield collection.clear();
+    } finally {
+      yield collection.db.close();
+    }
+  }
+
+  // Remove profile data.
+  for (let {filename} of gBlocklistClients) {
+    const blocklist = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    if (blocklist.exists()) {
+      blocklist.remove(true);
+    }
+  }
+}
+
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  // Point the blocklist clients to use this local HTTP server.
+  Services.prefs.setCharPref("services.settings.server",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // Setup server fake responses.
+  function handleResponse(request, response) {
+    try {
+      const sample = getSampleResponse(request, server.identity.primaryPort);
+      if (!sample) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sample.status.status,
+                             sample.status.statusText);
+      // send the headers
+      for (let headerLine of sample.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sample.responseBody);
+      response.finish();
+    } catch (e) {
+      do_print(e);
+    }
+  }
+  const configPath = "/v1/";
+  const addonsRecordsPath  = "/v1/buckets/blocklists/collections/addons/records";
+  const gfxRecordsPath     = "/v1/buckets/blocklists/collections/gfx/records";
+  const pluginsRecordsPath = "/v1/buckets/blocklists/collections/plugins/records";
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(addonsRecordsPath, handleResponse);
+  server.registerPathHandler(gfxRecordsPath, handleResponse);
+  server.registerPathHandler(pluginsRecordsPath, handleResponse);
+
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(() => { });
+  });
+}
+
+add_task(function* test_records_obtained_from_server_are_stored_in_db(){
+  for (let {client} of gBlocklistClients) {
+    // Test an empty db populates
+    let result = yield client.maybeSync(2000, Date.now());
+
+    // Open the collection, verify it's been populated:
+    // Our test data has a single record; it should be in the local collection
+    let collection = kintoCollection(client.collectionName);
+    yield collection.db.open();
+    let list = yield collection.list();
+    equal(list.data.length, 1);
+    yield collection.db.close();
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_list_is_written_to_file_in_profile(){
+  for (let {client, filename, testData} of gBlocklistClients) {
+    const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    strictEqual(profFile.exists(), false);
+
+    let result = yield client.maybeSync(2000, Date.now());
+
+    strictEqual(profFile.exists(), true);
+    const content = yield readJSON(profFile.path);
+    equal(content.data[0].blockID, testData[testData.length - 1]);
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_current_server_time_is_saved_in_pref(){
+  for (let {client} of gBlocklistClients) {
+    const before = Services.prefs.getIntPref(client.lastCheckTimePref);
+    const serverTime = Date.now();
+    yield client.maybeSync(2000, serverTime);
+    const after = Services.prefs.getIntPref(client.lastCheckTimePref);
+    equal(after, Math.round(serverTime / 1000));
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_update_json_file_when_addons_has_changes(){
+  for (let {client, filename, testData} of gBlocklistClients) {
+    yield client.maybeSync(2000, Date.now() - 1000);
+    const before = Services.prefs.getIntPref(client.lastCheckTimePref);
+    const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
+    const serverTime = Date.now();
+
+    yield client.maybeSync(3001, serverTime);
+
+    // File was updated.
+    notEqual(fileLastModified, profFile.lastModifiedTime);
+    const content = yield readJSON(profFile.path);
+    deepEqual(content.data.map((r) => r.blockID), testData);
+    // Server time was updated.
+    const after = Services.prefs.getIntPref(client.lastCheckTimePref);
+    equal(after, Math.round(serverTime / 1000));
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_sends_reload_message_when_blocklist_has_changes(){
+  for (let {client, filename} of gBlocklistClients) {
+    let received = yield new Promise((resolve, reject) => {
+      Services.ppmm.addMessageListener("Blocklist:reload-from-disk", {
+        receiveMessage(aMsg) { resolve(aMsg) }
+      });
+
+      client.maybeSync(2000, Date.now() - 1000);
+    });
+
+    equal(received.data.filename, filename);
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_do_nothing_when_blocklist_is_up_to_date(){
+  for (let {client, filename} of gBlocklistClients) {
+    yield client.maybeSync(2000, Date.now() - 1000);
+    const before = Services.prefs.getIntPref(client.lastCheckTimePref);
+    const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
+    const serverTime = Date.now();
+
+    yield client.maybeSync(3000, serverTime);
+
+    // File was not updated.
+    equal(fileLastModified, profFile.lastModifiedTime);
+    // Server time was updated.
+    const after = Services.prefs.getIntPref(client.lastCheckTimePref);
+    equal(after, Math.round(serverTime / 1000));
+  }
+});
+add_task(clear_state);
+
+
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "prefs": [],
+        "blockID": "i539",
+        "last_modified": 3000,
+        "versionRange": [{
+          "targetApplication": [],
+          "maxVersion": "*",
+          "minVersion": "0",
+          "severity": "1"
+        }],
+        "guid": "ScorpionSaver@jetpack",
+        "id": "9d500963-d80e-3a91-6e74-66f3811b99cc"
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/plugins/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "matchFilename": "NPFFAddOn.dll",
+        "blockID": "p28",
+        "id": "7b1e0b3c-e390-a817-11b6-a6887f65f56e",
+        "last_modified": 3000,
+        "versionRange": []
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/gfx/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "driverVersionComparator": "LESS_THAN_OR_EQUAL",
+        "driverVersion": "8.17.12.5896",
+        "vendor": "0x10de",
+        "blockID": "g36",
+        "feature": "DIRECT3D_9_LAYERS",
+        "devices": ["0x0a6c"],
+        "featureStatus": "BLOCKED_DRIVER_VERSION",
+        "last_modified": 3000,
+        "os": "WINNT 6.1",
+        "id": "3f947f16-37c2-4e96-d356-78b26363729b"
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "prefs": [],
+        "blockID": "i808",
+        "last_modified": 4000,
+        "versionRange": [{
+          "targetApplication": [],
+          "maxVersion": "*",
+          "minVersion": "0",
+          "severity": "3"
+        }],
+        "guid": "{c96d1ae6-c4cf-4984-b110-f5f561b33b5a}",
+        "id": "9ccfac91-e463-c30c-f0bd-14143794a8dd"
+      }, {
+        "prefs": ["browser.startup.homepage"],
+        "blockID": "i720",
+        "last_modified": 3500,
+        "versionRange": [{
+          "targetApplication": [],
+          "maxVersion": "*",
+          "minVersion": "0",
+          "severity": "1"
+        }],
+        "guid": "FXqG@xeeR.net",
+        "id": "cf9b3129-a97e-dbd7-9525-a8575ac03c25"
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/plugins/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "infoURL": "https://get.adobe.com/flashplayer/",
+        "blockID": "p1044",
+        "matchFilename": "libflashplayer\\.so",
+        "last_modified": 4000,
+        "versionRange": [{
+          "targetApplication": [],
+          "minVersion": "11.2.202.509",
+          "maxVersion": "11.2.202.539",
+          "severity": "0",
+          "vulnerabilityStatus": "1"
+        }],
+        "os": "Linux",
+        "id": "aabad965-e556-ffe7-4191-074f5dee3df3"
+      }, {
+        "matchFilename": "npViewpoint.dll",
+        "blockID": "p32",
+        "id": "1f48af42-c508-b8ef-b8d5-609d48e4f6c9",
+        "last_modified": 3500,
+        "versionRange": [{
+          "targetApplication": [{
+            "minVersion": "3.0",
+            "guid": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
+            "maxVersion": "*"
+          }]
+        }]
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/gfx/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "vendor": "0x8086",
+        "blockID": "g204",
+        "feature": "WEBGL_MSAA",
+        "devices": [],
+        "id": "c96bca82-e6bd-044d-14c4-9c1d67e9283a",
+        "last_modified": 4000,
+        "os": "Darwin 10",
+        "featureStatus": "BLOCKED_DEVICE"
+      }, {
+        "vendor": "0x10de",
+        "blockID": "g200",
+        "feature": "WEBGL_MSAA",
+        "devices": [],
+        "id": "c3a15ba9-e0e2-421f-e399-c995e5b8d14e",
+        "last_modified": 3500,
+        "os": "Darwin 11",
+        "featureStatus": "BLOCKED_DEVICE"
+      }]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/test_blocklist_signatures.js

@@ -0,0 +1,510 @@
+"use strict";
+
+Cu.import("resource://services-common/blocklist-updater.js");
+Cu.import("resource://testing-common/httpd.js");
+
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
+const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
+
+let server;
+
+const PREF_BLOCKLIST_BUCKET            = "services.blocklist.bucket";
+const PREF_BLOCKLIST_ENFORCE_SIGNING   = "services.blocklist.signing.enforced";
+const PREF_BLOCKLIST_ONECRL_COLLECTION = "services.blocklist.onecrl.collection";
+const PREF_SETTINGS_SERVER             = "services.settings.server";
+const PREF_SIGNATURE_ROOT              = "security.content.signature.root_hash";
+
+
+const CERT_DIR = "test_blocklist_signatures/";
+const CHAIN_FILES =
+    ["collection_signing_ee.pem",
+     "collection_signing_int.pem",
+     "collection_signing_root.pem"];
+
+function getFileData(file) {
+  const stream = Cc["@mozilla.org/network/file-input-stream;1"]
+                   .createInstance(Ci.nsIFileInputStream);
+  stream.init(file, -1, 0, 0);
+  const data = NetUtil.readInputStreamToString(stream, stream.available());
+  stream.close();
+  return data;
+}
+
+function setRoot() {
+  const filename = CERT_DIR + CHAIN_FILES[0];
+
+  const certFile = do_get_file(filename, false);
+  const b64cert = getFileData(certFile)
+                    .replace(/-----BEGIN CERTIFICATE-----/, "")
+                    .replace(/-----END CERTIFICATE-----/, "")
+                    .replace(/[\r\n]/g, "");
+  const certdb = Cc["@mozilla.org/security/x509certdb;1"]
+                   .getService(Ci.nsIX509CertDB);
+  const cert = certdb.constructX509FromBase64(b64cert);
+  Services.prefs.setCharPref(PREF_SIGNATURE_ROOT, cert.sha256Fingerprint);
+}
+
+function getCertChain() {
+  const chain = [];
+  for (let file of CHAIN_FILES) {
+    chain.push(getFileData(do_get_file(CERT_DIR + file)));
+  }
+  return chain.join("\n");
+}
+
+function* checkRecordCount(count) {
+  // open the collection manually
+  const base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+  const bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
+  const collectionName =
+      Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION);
+
+  const Kinto = loadKinto();
+
+  const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+
+  const config = {
+    remote: base,
+    bucket: bucket,
+    adapter: FirefoxAdapter,
+  };
+
+  const db = new Kinto(config);
+  const collection = db.collection(collectionName);
+
+  yield collection.db.open();
+
+  // Check we have the expected number of records
+  let records = yield collection.list();
+  do_check_eq(count, records.data.length);
+
+  // Close the collection so the test can exit cleanly
+  yield collection.db.close();
+}
+
+// Check to ensure maybeSync is called with correct values when a changes
+// document contains information on when a collection was last modified
+add_task(function* test_check_signatures(){
+  const port = server.identity.primaryPort;
+
+  // a response to give the client when the cert chain is expected
+  function makeMetaResponseBody(lastModified, signature) {
+    return {
+      data: {
+        id: "certificates",
+        last_modified: lastModified,
+        signature: {
+          x5u: `http://localhost:${port}/test_blocklist_signatures/test_cert_chain.pem`,
+          public_key: "fake",
+          "content-signature": `x5u=http://localhost:${port}/test_blocklist_signatures/test_cert_chain.pem;p384ecdsa=${signature}`,
+          signature_encoding: "rs_base64url",
+          signature: signature,
+          hash_algorithm: "sha384",
+          ref: "1yryrnmzou5rf31ou80znpnq8n"
+        }
+      }
+    };
+  }
+
+  function makeMetaResponse(eTag, body, comment) {
+    return {
+      comment: comment,
+      sampleHeaders: [
+        "Content-Type: application/json; charset=UTF-8",
+        `ETag: \"${eTag}\"`
+      ],
+      status: {status: 200, statusText: "OK"},
+      responseBody: JSON.stringify(body)
+    };
+  }
+
+  function registerHandlers(responses){
+    function handleResponse (serverTimeMillis, request, response) {
+      const key = `${request.method}:${request.path}?${request.queryString}`;
+      const available = responses[key];
+      const sampled = available.length > 1 ? available.shift() : available[0];
+
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                            sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+
+      // set the server date
+      response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
+
+      response.write(sampled.responseBody);
+    }
+
+    for (let key of Object.keys(responses)) {
+      const keyParts = key.split(":");
+      const method = keyParts[0];
+      const valueParts = keyParts[1].split("?");
+      const path = valueParts[0];
+
+      server.registerPathHandler(path, handleResponse.bind(null, 2000));
+    }
+  }
+
+  // First, perform a signature verification with known data and signature
+  // to ensure things are working correctly
+  let verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
+                   .createInstance(Ci.nsIContentSignatureVerifier);
+
+  const emptyData = '[]';
+  const emptySignature = "p384ecdsa=zbugm2FDitsHwk5-IWsas1PpWwY29f0Fg5ZHeqD8fzep7AVl2vfcaHA7LdmCZ28qZLOioGKvco3qT117Q4-HlqFTJM7COHzxGyU2MMJ0ZTnhJrPOC1fP3cVQjU1PTWi9";
+  const name = "onecrl.content-signature.mozilla.org";
+  ok(verifier.verifyContentSignature(emptyData, emptySignature,
+                                     getCertChain(), name));
+
+  verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
+               .createInstance(Ci.nsIContentSignatureVerifier);
+
+  const collectionData = '[{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:43:37Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"97fbf7c4-3ef2-f54f-0029-1ba6540c63ea","issuerName":"MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==","last_modified":2000,"serialNumber":"BAAAAAABA/A35EU="},{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:48:11Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc","issuerName":"MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB","last_modified":3000,"serialNumber":"BAAAAAABI54PryQ="}]';
+  const collectionSignature = "p384ecdsa=f4pA2tYM5jQgWY6YUmhUwQiBLj6QO5sHLD_5MqLePz95qv-7cNCuQoZnPQwxoptDtW8hcWH3kLb0quR7SB-r82gkpR9POVofsnWJRA-ETb0BcIz6VvI3pDT49ZLlNg3p";
+
+  ok(verifier.verifyContentSignature(collectionData, collectionSignature, getCertChain(), name));
+
+  // set up prefs so the kinto updater talks to the test server
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // Set up some data we need for our test
+  let startTime = Date.now();
+
+  // These are records we'll use in the test collections
+  const RECORD1 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:43:37Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: "."
+    },
+    enabled: true,
+    id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
+    issuerName: "MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==",
+    last_modified: 2000,
+    serialNumber: "BAAAAAABA/A35EU="
+  };
+
+  const RECORD2 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:48:11Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: "."
+    },
+    enabled: true,
+    id: "e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc",
+    issuerName: "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
+    last_modified: 3000,
+    serialNumber: "BAAAAAABI54PryQ="
+  };
+
+  const RECORD3 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:48:11Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: "."
+    },
+    enabled: true,
+    id: "c7c49b69-a4ab-418e-92a9-e1961459aa7f",
+    issuerName: "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
+    last_modified: 4000,
+    serialNumber: "BAAAAAABI54PryQ="
+  };
+
+  const RECORD1_DELETION = {
+    deleted: true,
+    enabled: true,
+    id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
+    last_modified: 3500,
+  };
+
+  // Check that a signature on an empty collection is OK
+  // We need to set up paths on the HTTP server to return specific data from
+  // specific paths for each test. Here we prepare data for each response.
+
+  // A cert chain response (this the cert chain that contains the signing
+  // cert, the root and any intermediates in between). This is used in each
+  // sync.
+  const RESPONSE_CERT_CHAIN = {
+    comment: "RESPONSE_CERT_CHAIN",
+    sampleHeaders: [
+      "Content-Type: text/plain; charset=UTF-8"
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: getCertChain()
+  };
+
+  // A server settings response. This is used in each sync.
+  const RESPONSE_SERVER_SETTINGS = {
+    comment: "RESPONSE_SERVER_SETTINGS",
+    sampleHeaders: [
+      "Access-Control-Allow-Origin: *",
+      "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+      "Content-Type: application/json; charset=UTF-8",
+      "Server: waitress"
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+  };
+
+  // This is the initial, empty state of the collection. This is only used
+  // for the first sync.
+  const RESPONSE_EMPTY_INITIAL = {
+    comment: "RESPONSE_EMPTY_INITIAL",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"1000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": []})
+  };
+
+  const RESPONSE_BODY_META_EMPTY_SIG = makeMetaResponseBody(1000,
+    "vxuAg5rDCB-1pul4a91vqSBQRXJG_j7WOYUTswxRSMltdYmbhLRH8R8brQ9YKuNDF56F-w6pn4HWxb076qgKPwgcEBtUeZAO_RtaHXRkRUUgVzAr86yQL4-aJTbv3D6u");
+
+  // The collection metadata containing the signature for the empty
+  // collection.
+  const RESPONSE_META_EMPTY_SIG =
+    makeMetaResponse(1000, RESPONSE_BODY_META_EMPTY_SIG,
+                     "RESPONSE_META_EMPTY_SIG");
+
+  // Here, we map request method and path to the available responses
+  const emptyCollectionResponses = {
+    "GET:/test_blocklist_signatures/test_cert_chain.pem?":[RESPONSE_CERT_CHAIN],
+    "GET:/v1/?": [RESPONSE_SERVER_SETTINGS],
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified":
+      [RESPONSE_EMPTY_INITIAL],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_EMPTY_SIG]
+  };
+
+  // .. and use this map to register handlers for each path
+  registerHandlers(emptyCollectionResponses);
+
+  // With all of this set up, we attempt a sync. This will resolve if all is
+  // well and throw if something goes wrong.
+  yield OneCRLBlocklistClient.maybeSync(1000, startTime);
+
+  // Check that some additions (2 records) to the collection have a valid
+  // signature.
+
+  // This response adds two entries (RECORD1 and RECORD2) to the collection
+  const RESPONSE_TWO_ADDED = {
+    comment: "RESPONSE_TWO_ADDED",
+    sampleHeaders: [
+        "Content-Type: application/json; charset=UTF-8",
+        "ETag: \"3000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD2, RECORD1]})
+  };
+
+  const RESPONSE_BODY_META_TWO_ITEMS_SIG = makeMetaResponseBody(3000,
+    "dwhJeypadNIyzGj3QdI0KMRTPnHhFPF_j73mNrsPAHKMW46S2Ftf4BzsPMvPMB8h0TjDus13wo_R4l432DHe7tYyMIWXY0PBeMcoe5BREhFIxMxTsh9eGVXBD1e3UwRy");
+
+  // A signature response for the collection containg RECORD1 and RECORD2
+  const RESPONSE_META_TWO_ITEMS_SIG =
+    makeMetaResponse(3000, RESPONSE_BODY_META_TWO_ITEMS_SIG,
+                     "RESPONSE_META_TWO_ITEMS_SIG");
+
+  const twoItemsResponses = {
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=1000":
+      [RESPONSE_TWO_ADDED],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_TWO_ITEMS_SIG]
+  };
+  registerHandlers(twoItemsResponses);
+  yield OneCRLBlocklistClient.maybeSync(3000, startTime);
+
+  // Check the collection with one addition and one removal has a valid
+  // signature
+
+  // Remove RECORD1, add RECORD3
+  const RESPONSE_ONE_ADDED_ONE_REMOVED = {
+    comment: "RESPONSE_ONE_ADDED_ONE_REMOVED ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD3, RECORD1_DELETION]})
+  };
+
+  const RESPONSE_BODY_META_THREE_ITEMS_SIG = makeMetaResponseBody(4000,
+    "MIEmNghKnkz12UodAAIc3q_Y4a3IJJ7GhHF4JYNYmm8avAGyPM9fYU7NzVo94pzjotG7vmtiYuHyIX2rTHTbT587w0LdRWxipgFd_PC1mHiwUyjFYNqBBG-kifYk7kEw");
+
+  // signature response for the collection containing RECORD2 and RECORD3
+  const RESPONSE_META_THREE_ITEMS_SIG =
+    makeMetaResponse(4000, RESPONSE_BODY_META_THREE_ITEMS_SIG,
+                     "RESPONSE_META_THREE_ITEMS_SIG");
+
+  const oneAddedOneRemovedResponses = {
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=3000":
+      [RESPONSE_ONE_ADDED_ONE_REMOVED],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_THREE_ITEMS_SIG]
+  };
+  registerHandlers(oneAddedOneRemovedResponses);
+  yield OneCRLBlocklistClient.maybeSync(4000, startTime);
+
+  // Check the signature is still valid with no operation (no changes)
+
+  // Leave the collection unchanged
+  const RESPONSE_EMPTY_NO_UPDATE = {
+    comment: "RESPONSE_EMPTY_NO_UPDATE ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": []})
+  };
+
+  const noOpResponses = {
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_THREE_ITEMS_SIG]
+  };
+  registerHandlers(noOpResponses);
+  yield OneCRLBlocklistClient.maybeSync(4100, startTime);
+
+  // Check the collection is reset when the signature is invalid
+
+  // Prepare a (deliberately) bad signature to check the collection state is
+  // reset if something is inconsistent
+  const RESPONSE_COMPLETE_INITIAL = {
+    comment: "RESPONSE_COMPLETE_INITIAL ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD2, RECORD3]})
+  };
+
+  const RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID = {
+    comment: "RESPONSE_COMPLETE_INITIAL ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD3, RECORD2]})
+  };
+
+  const RESPONSE_BODY_META_BAD_SIG = makeMetaResponseBody(4000,
+      "aW52YWxpZCBzaWduYXR1cmUK");
+
+  const RESPONSE_META_BAD_SIG =
+      makeMetaResponse(4000, RESPONSE_BODY_META_BAD_SIG, "RESPONSE_META_BAD_SIG");
+
+  const badSigGoodSigResponses = {
+    // In this test, we deliberately serve a bad signature initially. The
+    // subsequent signature returned is a valid one for the three item
+    // collection.
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_BAD_SIG, RESPONSE_META_THREE_ITEMS_SIG],
+    // The first collection state is the three item collection (since
+    // there's a sync with no updates) - but, since the signature is wrong,
+    // another request will be made...
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    // The next request is for the full collection. This will be checked
+    // against the valid signature - so the sync should succeed.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified":
+      [RESPONSE_COMPLETE_INITIAL],
+    // The next request is for the full collection sorted by id. This will be
+    // checked against the valid signature - so the sync should succeed.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
+      [RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID]
+  };
+
+  registerHandlers(badSigGoodSigResponses);
+  yield OneCRLBlocklistClient.maybeSync(5000, startTime);
+
+  const badSigGoodOldResponses = {
+    // In this test, we deliberately serve a bad signature initially. The
+    // subsequent sitnature returned is a valid one for the three item
+    // collection.
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_BAD_SIG, RESPONSE_META_EMPTY_SIG],
+    // The first collection state is the current state (since there's no update
+    // - but, since the signature is wrong, another request will be made)
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    // The next request is for the full collection sorted by id. This will be
+    // checked against the valid signature and last_modified times will be
+    // compared. Sync should fail, even though the signature is good,
+    // because the local collection is newer.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
+      [RESPONSE_EMPTY_INITIAL],
+  };
+
+  // ensure our collection hasn't been replaced with an older, empty one
+  yield checkRecordCount(2);
+
+  registerHandlers(badSigGoodOldResponses);
+  yield OneCRLBlocklistClient.maybeSync(5000, startTime);
+
+  const allBadSigResponses = {
+    // In this test, we deliberately serve only a bad signature.
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_BAD_SIG],
+    // The first collection state is the three item collection (since
+    // there's a sync with no updates) - but, since the signature is wrong,
+    // another request will be made...
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    // The next request is for the full collection sorted by id. This will be
+    // checked against the valid signature - so the sync should succeed.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
+      [RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID]
+  };
+
+  registerHandlers(allBadSigResponses);
+  try {
+    yield OneCRLBlocklistClient.maybeSync(6000, startTime);
+    do_throw("Sync should fail (the signature is intentionally bad)");
+  } catch (e) {
+    yield checkRecordCount(2);
+  }
+});
+
+function run_test() {
+  // ensure signatures are enforced
+  Services.prefs.setBoolPref(PREF_BLOCKLIST_ENFORCE_SIGNING, true);
+
+  // get a signature verifier to ensure nsNSSComponent is initialized
+  Cc["@mozilla.org/security/contentsignatureverifier;1"]
+    .createInstance(Ci.nsIContentSignatureVerifier);
+
+  // set the content signing root to our test root
+  setRoot();
+
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+

services/common/tests/unit/test_blocklist_signatures/collection_signing_ee.pem.certspec

@@ -0,0 +1,5 @@
+issuer:collection-signer-int-CA
+subject:collection-signer-ee-int-CA
+subjectKey:secp384r1
+extension:extKeyUsage:codeSigning
+extension:subjectAlternativeName:onecrl.content-signature.mozilla.org

services/common/tests/unit/test_blocklist_signatures/collection_signing_int.pem.certspec

@@ -0,0 +1,4 @@
+issuer:collection-signer-ca
+subject:collection-signer-int-CA
+extension:basicConstraints:cA,
+extension:extKeyUsage:codeSigning

services/common/tests/unit/test_blocklist_signatures/collection_signing_root.pem.certspec

@@ -0,0 +1,4 @@
+issuer:collection-signer-ca
+subject:collection-signer-ca
+extension:basicConstraints:cA,
+extension:extKeyUsage:codeSigning

services/common/tests/unit/test_blocklist_signatures/moz.build

@@ -0,0 +1,14 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+test_certificates = (
+    'collection_signing_root.pem',
+    'collection_signing_int.pem',
+    'collection_signing_ee.pem',
+)
+
+for test_certificate in test_certificates:
+    GeneratedTestCertificate(test_certificate)

services/common/tests/unit/test_blocklist_updater.js

@@ -0,0 +1,173 @@
+Cu.import("resource://testing-common/httpd.js");
+
+var server;
+
+const PREF_SETTINGS_SERVER = "services.settings.server";
+const PREF_LAST_UPDATE = "services.blocklist.last_update_seconds";
+const PREF_LAST_ETAG = "services.blocklist.last_etag";
+const PREF_CLOCK_SKEW_SECONDS = "services.blocklist.clock_skew_seconds";
+
+// Check to ensure maybeSync is called with correct values when a changes
+// document contains information on when a collection was last modified
+add_task(function* test_check_maybeSync(){
+  const changesPath = "/v1/buckets/monitor/collections/changes/records";
+
+  // register a handler
+  function handleResponse (serverTimeMillis, request, response) {
+    try {
+      const sampled = getSampleResponse(request, server.identity.primaryPort);
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                             sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+
+      // set the server date
+      response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
+
+      response.write(sampled.responseBody);
+    } catch (e) {
+      dump(`${e}\n`);
+    }
+  }
+
+  server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
+
+  // set up prefs so the kinto updater talks to the test server
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // set some initial values so we can check these are updated appropriately
+  Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
+  Services.prefs.setIntPref(PREF_CLOCK_SKEW_SECONDS, 0);
+  Services.prefs.clearUserPref(PREF_LAST_ETAG);
+
+
+  let startTime = Date.now();
+
+  let updater = Cu.import("resource://services-common/blocklist-updater.js");
+
+  let syncPromise = new Promise(function(resolve, reject) {
+    // add a test kinto client that will respond to lastModified information
+    // for a collection called 'test-collection'
+    updater.addTestBlocklistClient("test-collection", {
+      maybeSync(lastModified, serverTime) {
+        do_check_eq(lastModified, 1000);
+        do_check_eq(serverTime, 2000);
+        resolve();
+      }
+    });
+    updater.checkVersions();
+  });
+
+  // ensure we get the maybeSync call
+  yield syncPromise;
+
+  // check the last_update is updated
+  do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
+
+  // How does the clock difference look?
+  let endTime = Date.now();
+  let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to 2 (seconds past epoch)
+  do_check_true(clockDifference <= endTime / 1000
+              && clockDifference >= Math.floor(startTime / 1000) - 2);
+  // Last timestamp was saved. An ETag header value is a quoted string.
+  let lastEtag = Services.prefs.getCharPref(PREF_LAST_ETAG);
+  do_check_eq(lastEtag, "\"1100\"");
+
+  // Simulate a poll with up-to-date collection.
+  Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
+  // If server has no change, a 304 is received, maybeSync() is not called.
+  updater.addTestBlocklistClient("test-collection", {
+    maybeSync: () => {throw new Error("Should not be called");}
+  });
+  yield updater.checkVersions();
+  // Last update is overwritten
+  do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
+
+
+  // Simulate a server error.
+  function simulateErrorResponse (request, response) {
+    response.setHeader("Date", (new Date(3000)).toUTCString());
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.write(JSON.stringify({
+      code: 503,
+      errno: 999,
+      error: "Service Unavailable",
+    }));
+    response.setStatusLine(null, 503, "Service Unavailable");
+  }
+  server.registerPathHandler(changesPath, simulateErrorResponse);
+  // checkVersions() fails with adequate error.
+  let error;
+  try {
+    yield updater.checkVersions();
+  } catch (e) {
+    error = e;
+  }
+  do_check_eq(error.message, "Polling for changes failed.");
+  // When an error occurs, last update was not overwritten (see Date header above).
+  do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
+
+  // check negative clock skew times
+
+  // set to a time in the future
+  server.registerPathHandler(changesPath, handleResponse.bind(null, Date.now() + 10000));
+
+  yield updater.checkVersions();
+
+  clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to Date.now() + 10000 ms past epoch
+  do_check_true(clockDifference <= 0 && clockDifference >= -10);
+});
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "GET:/v1/buckets/monitor/collections/changes/records?": {
+      "sampleHeaders": [
+        "Content-Type: application/json; charset=UTF-8",
+        "ETag: \"1100\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data": [{
+        "host": "localhost",
+        "last_modified": 1100,
+        "bucket": "blocklists:aurora",
+        "id": "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
+        "collection": "test-collection"
+      }, {
+        "host": "localhost",
+        "last_modified": 1000,
+        "bucket": "blocklists",
+        "id": "254cbb9e-6888-4d9f-8e60-58b74faa8778",
+        "collection": "test-collection"
+      }]})
+    }
+  };
+
+  if (req.hasHeader("if-none-match") && req.getHeader("if-none-match", "") == "\"1100\"")
+    return {sampleHeaders: [], status: {status: 304, statusText: "Not Modified"}, responseBody: ""};
+
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+}

services/common/tests/unit/test_hawkclient.js

@@ -0,0 +1,520 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://services-common/hawkclient.js");
+
+const SECOND_MS = 1000;
+const MINUTE_MS = SECOND_MS * 60;
+const HOUR_MS = MINUTE_MS * 60;
+
+const TEST_CREDS = {
+  id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+  key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+  algorithm: "sha256"
+};
+
+initTestLogging("Trace");
+
+add_task(function test_now() {
+  let client = new HawkClient("https://example.com");
+
+  do_check_true(client.now() - Date.now() < SECOND_MS);
+});
+
+add_task(function test_updateClockOffset() {
+  let client = new HawkClient("https://example.com");
+
+  let now = new Date();
+  let serverDate = now.toUTCString();
+
+  // Client's clock is off
+  client.now = () => { return now.valueOf() + HOUR_MS; }
+
+  client._updateClockOffset(serverDate);
+
+  // Check that they're close; there will likely be a one-second rounding
+  // error, so checking strict equality will likely fail.
+  //
+  // localtimeOffsetMsec is how many milliseconds to add to the local clock so
+  // that it agrees with the server.  We are one hour ahead of the server, so
+  // our offset should be -1 hour.
+  do_check_true(Math.abs(client.localtimeOffsetMsec + HOUR_MS) <= SECOND_MS);
+});
+
+add_task(function* test_authenticated_get_request() {
+  let message = "{\"msg\": \"Great Success!\"}";
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      do_check_true(request.hasHeader("Authorization"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", method, TEST_CREDS);
+  let result = JSON.parse(response.body);
+
+  do_check_eq("Great Success!", result.msg);
+
+  yield deferredStop(server);
+});
+
+function* check_authenticated_request(method) {
+  let server = httpd_setup({"/foo": (request, response) => {
+      do_check_true(request.hasHeader("Authorization"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+      response.bodyOutputStream.writeFrom(request.bodyInputStream, request.bodyInputStream.available());
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", method, TEST_CREDS, {foo: "bar"});
+  let result = JSON.parse(response.body);
+
+  do_check_eq("bar", result.foo);
+
+  yield deferredStop(server);
+}
+
+add_task(function test_authenticated_post_request() {
+  check_authenticated_request("POST");
+});
+
+add_task(function test_authenticated_put_request() {
+  check_authenticated_request("PUT");
+});
+
+add_task(function test_authenticated_patch_request() {
+  check_authenticated_request("PATCH");
+});
+
+add_task(function* test_extra_headers() {
+  let server = httpd_setup({"/foo": (request, response) => {
+      do_check_true(request.hasHeader("Authorization"));
+      do_check_true(request.hasHeader("myHeader"));
+      do_check_eq(request.getHeader("myHeader"), "fake");
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+      response.bodyOutputStream.writeFrom(request.bodyInputStream, request.bodyInputStream.available());
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", "POST", TEST_CREDS, {foo: "bar"},
+                                      {"myHeader": "fake"});
+  let result = JSON.parse(response.body);
+
+  do_check_eq("bar", result.foo);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_credentials_optional() {
+  let method = "GET";
+  let server = httpd_setup({
+    "/foo": (request, response) => {
+      do_check_false(request.hasHeader("Authorization"));
+
+      let message = JSON.stringify({msg: "you're in the friend zone"});
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  let result = yield client.request("/foo", method); // credentials undefined
+  do_check_eq(JSON.parse(result.body).msg, "you're in the friend zone");
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_server_error() {
+  let message = "Ohai!";
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 418, "I am a Teapot");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  try {
+    yield client.request("/foo", method, TEST_CREDS);
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_eq(418, err.code);
+    do_check_eq("I am a Teapot", err.message);
+  }
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_server_error_json() {
+  let message = JSON.stringify({error: "Cannot get ye flask."});
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 400, "What wouldst thou deau?");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  try {
+    yield client.request("/foo", method, TEST_CREDS);
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_eq("Cannot get ye flask.", err.error);
+  }
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_offset_after_request() {
+  let message = "Ohai!";
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  let now = Date.now();
+  client.now = () => { return now + HOUR_MS; };
+
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  let response = yield client.request("/foo", method, TEST_CREDS);
+  // Should be about an hour off
+  do_check_true(Math.abs(client.localtimeOffsetMsec + HOUR_MS) < SECOND_MS);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_offset_in_hawk_header() {
+  let message = "Ohai!";
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/first": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    },
+
+    "/second": function(request, response) {
+      // We see a better date now in the ts component of the header
+      let delta = getTimestampDelta(request.getHeader("Authorization"));
+      let message = "Delta: " + delta;
+
+      // We're now within HAWK's one-minute window.
+      // I hope this isn't a recipe for intermittent oranges ...
+      if (delta < MINUTE_MS) {
+        response.setStatusLine(request.httpVersion, 200, "OK");
+      } else {
+        response.setStatusLine(request.httpVersion, 400, "Delta: " + delta);
+      }
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() + 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+  yield client.request("/first", method, TEST_CREDS);
+
+  // After the first server response, our offset is updated to -12 hours.
+  // We should be safely in the window, now.
+  do_check_true(Math.abs(client.localtimeOffsetMsec + 12 * HOUR_MS) < MINUTE_MS);
+  yield client.request("/second", method, TEST_CREDS);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_2xx_success() {
+  // Just to ensure that we're not biased toward 200 OK for success
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 202, "Accepted");
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", method, credentials);
+
+  // Shouldn't be any content in a 202
+  do_check_eq(response.body, "");
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_retry_request_on_fail() {
+  let attempts = 0;
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/maybe": function(request, response) {
+      // This path should be hit exactly twice; once with a bad timestamp, and
+      // again when the client retries the request with a corrected timestamp.
+      attempts += 1;
+      do_check_true(attempts <= 2);
+
+      let delta = getTimestampDelta(request.getHeader("Authorization"));
+
+      // First time through, we should have a bad timestamp
+      if (attempts === 1) {
+        do_check_true(delta > MINUTE_MS);
+        let message = "never!!!";
+        response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+        response.bodyOutputStream.write(message, message.length);
+        return;
+      }
+
+      // Second time through, timestamp should be corrected by client
+      do_check_true(delta < MINUTE_MS);
+      let message = "i love you!!!";
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+      return;
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() + 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  // Request will have bad timestamp; client will retry once
+  let response = yield client.request("/maybe", method, credentials);
+  do_check_eq(response.body, "i love you!!!");
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_multiple_401_retry_once() {
+  // Like test_retry_request_on_fail, but always return a 401
+  // and ensure that the client only retries once.
+  let attempts = 0;
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/maybe": function(request, response) {
+      // This path should be hit exactly twice; once with a bad timestamp, and
+      // again when the client retries the request with a corrected timestamp.
+      attempts += 1;
+
+      do_check_true(attempts <= 2);
+
+      let message = "never!!!";
+      response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() - 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  // Request will have bad timestamp; client will retry once
+  try {
+    yield client.request("/maybe", method, credentials);
+    do_throw("Expected an error");
+  } catch (err) {
+    do_check_eq(err.code, 401);
+  }
+  do_check_eq(attempts, 2);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_500_no_retry() {
+  // If we get a 500 error, the client should not retry (as it would with a
+  // 401)
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/no-shutup": function() {
+      let message = "Cannot get ye flask.";
+      response.setStatusLine(request.httpVersion, 500, "Internal server error");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  // Throw off the clock so the HawkClient would want to retry the request if
+  // it could
+  client.now = () => {
+    return Date.now() - 12 * HOUR_MS;
+  };
+
+  // Request will 500; no retries
+  try {
+    yield client.request("/no-shutup", method, credentials);
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_eq(err.code, 500);
+  }
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_401_then_500() {
+  // Like test_multiple_401_retry_once, but return a 500 to the
+  // second request, ensuring that the promise is properly rejected
+  // in client.request.
+  let attempts = 0;
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/maybe": function(request, response) {
+      // This path should be hit exactly twice; once with a bad timestamp, and
+      // again when the client retries the request with a corrected timestamp.
+      attempts += 1;
+      do_check_true(attempts <= 2);
+
+      let delta = getTimestampDelta(request.getHeader("Authorization"));
+
+      // First time through, we should have a bad timestamp
+      // Client will retry
+      if (attempts === 1) {
+        do_check_true(delta > MINUTE_MS);
+        let message = "never!!!";
+        response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+        response.bodyOutputStream.write(message, message.length);
+        return;
+      }
+
+      // Second time through, timestamp should be corrected by client
+      // And fail on the client
+      do_check_true(delta < MINUTE_MS);
+      let message = "Cannot get ye flask.";
+      response.setStatusLine(request.httpVersion, 500, "Internal server error");
+      response.bodyOutputStream.write(message, message.length);
+      return;
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() - 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  // Request will have bad timestamp; client will retry once
+  try {
+    yield client.request("/maybe", method, credentials);
+  } catch(err) {
+    do_check_eq(err.code, 500);
+  }
+  do_check_eq(attempts, 2);
+
+  yield deferredStop(server);
+});
+
+add_task(function* throw_if_not_json_body() {
+  let client = new HawkClient("https://example.com");
+  try {
+    yield client.request("/bogus", "GET", {}, "I am not json");
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_true(!!err.message);
+  }
+});
+
+// End of tests.
+// Utility functions follow
+
+function getTimestampDelta(authHeader, now=Date.now()) {
+  let tsMS = new Date(
+      parseInt(/ts="(\d+)"/.exec(authHeader)[1], 10) * SECOND_MS);
+  return Math.abs(tsMS - now);
+}
+
+function deferredStop(server) {
+  let deferred = Promise.defer();
+  server.stop(deferred.resolve);
+  return deferred.promise;
+}
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+

services/common/tests/unit/test_hawkrequest.js

@@ -0,0 +1,235 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-common/hawkrequest.js");
+
+// https://github.com/mozilla/fxa-auth-server/wiki/onepw-protocol#wiki-use-session-certificatesign-etc
+var SESSION_KEYS = {
+  sessionToken: h("a0a1a2a3a4a5a6a7 a8a9aaabacadaeaf"+
+                  "b0b1b2b3b4b5b6b7 b8b9babbbcbdbebf"),
+
+  tokenID:      h("c0a29dcf46174973 da1378696e4c82ae"+
+                  "10f723cf4f4d9f75 e39f4ae3851595ab"),
+
+  reqHMACkey:   h("9d8f22998ee7f579 8b887042466b72d5"+
+                  "3e56ab0c094388bf 65831f702d2febc0"),
+};
+
+function do_register_cleanup() {
+  Services.prefs.resetUserPrefs();
+
+  // remove the pref change listener
+  let hawk = new HAWKAuthenticatedRESTRequest("https://example.com");
+  hawk._intl.uninit();
+}
+
+function run_test() {
+  Log.repository.getLogger("Services.Common.RESTRequest").level =
+    Log.Level.Trace;
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+
+add_test(function test_intl_accept_language() {
+  let testCount = 0;
+  let languages = [
+    "zu-NP;vo",     // Nepalese dialect of Zulu, defaulting to Volapük
+    "fa-CG;ik",     // Congolese dialect of Farsei, defaulting to Inupiaq
+  ];
+
+  function setLanguagePref(lang) {
+    let acceptLanguage = Cc["@mozilla.org/supports-string;1"]
+                           .createInstance(Ci.nsISupportsString);
+    acceptLanguage.data = lang;
+    Services.prefs.setComplexValue(
+      "intl.accept_languages", Ci.nsISupportsString, acceptLanguage);
+  }
+
+  let hawk = new HAWKAuthenticatedRESTRequest("https://example.com");
+
+  Services.prefs.addObserver("intl.accept_languages", checkLanguagePref, false);
+  setLanguagePref(languages[testCount]);
+
+  function checkLanguagePref() {
+    var _done = false;
+    CommonUtils.nextTick(function() {
+      // Ensure we're only called for the number of entries in languages[].
+      do_check_true(testCount < languages.length);
+
+      do_check_eq(hawk._intl.accept_languages, languages[testCount]);
+
+      testCount++;
+      if (testCount < languages.length) {
+        // Set next language in prefs; Pref service will call checkNextLanguage.
+        setLanguagePref(languages[testCount]);
+        return;
+      }
+
+      // We've checked all the entries in languages[]. Cleanup and move on.
+      do_print("Checked " + testCount + " languages. Removing checkLanguagePref as pref observer.");
+      Services.prefs.removeObserver("intl.accept_languages", checkLanguagePref);
+      run_next_test();
+      return;
+    });
+  }
+});
+
+add_test(function test_hawk_authenticated_request() {
+  let onProgressCalled = false;
+  let postData = {your: "data"};
+
+  // An arbitrary date - Feb 2, 1971.  It ends in a bunch of zeroes to make our
+  // computation with the hawk timestamp easier, since hawk throws away the
+  // millisecond values.
+  let then = 34329600000;
+
+  let clockSkew = 120000;
+  let timeOffset = -1 * clockSkew;
+  let localTime = then + clockSkew;
+
+  // Set the accept-languages pref to the Nepalese dialect of Zulu.
+  let acceptLanguage = Cc['@mozilla.org/supports-string;1'].createInstance(Ci.nsISupportsString);
+  acceptLanguage.data = 'zu-NP'; // omit trailing ';', which our HTTP libs snip
+  Services.prefs.setComplexValue('intl.accept_languages', Ci.nsISupportsString, acceptLanguage);
+
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+
+  let server = httpd_setup({
+    "/elysium": function(request, response) {
+      do_check_true(request.hasHeader("Authorization"));
+
+      // check that the header timestamp is our arbitrary system date, not
+      // today's date.  Note that hawk header timestamps are in seconds, not
+      // milliseconds.
+      let authorization = request.getHeader("Authorization");
+      let tsMS = parseInt(/ts="(\d+)"/.exec(authorization)[1], 10) * 1000;
+      do_check_eq(tsMS, then);
+
+      // This testing can be a little wonky. In an environment where
+      //   pref("intl.accept_languages") === 'en-US, en'
+      // the header is sent as:
+      //   'en-US,en;q=0.5'
+      // hence our fake value for acceptLanguage.
+      let lang = request.getHeader("Accept-Language");
+      do_check_eq(lang, acceptLanguage);
+
+      let message = "yay";
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  function onProgress() {
+    onProgressCalled = true;
+  }
+
+  function onComplete(error) {
+    do_check_eq(200, this.response.status);
+    do_check_eq(this.response.body, "yay");
+    do_check_true(onProgressCalled);
+
+    Services.prefs.resetUserPrefs();
+    let pref = Services.prefs.getComplexValue(
+      "intl.accept_languages", Ci.nsIPrefLocalizedString);
+    do_check_neq(acceptLanguage.data, pref.data);
+
+    server.stop(run_next_test);
+  }
+
+  let url = server.baseURI + "/elysium";
+  let extra = {
+    now: localTime,
+    localtimeOffsetMsec: timeOffset
+  };
+
+  let request = new HAWKAuthenticatedRESTRequest(url, credentials, extra);
+
+  // Allow hawk._intl to respond to the language pref change
+  CommonUtils.nextTick(function() {
+    request.post(postData, onComplete, onProgress);
+  });
+});
+
+add_test(function test_hawk_language_pref_changed() {
+  let languages = [
+    "zu-NP",        // Nepalese dialect of Zulu
+    "fa-CG",        // Congolese dialect of Farsi
+  ];
+
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256",
+  };
+
+  function setLanguage(lang) {
+    let acceptLanguage = Cc["@mozilla.org/supports-string;1"].createInstance(Ci.nsISupportsString);
+    acceptLanguage.data = lang;
+    Services.prefs.setComplexValue("intl.accept_languages", Ci.nsISupportsString, acceptLanguage);
+  }
+
+  let server = httpd_setup({
+    "/foo": function(request, response) {
+      do_check_eq(languages[1], request.getHeader("Accept-Language"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+    },
+  });
+
+  let url = server.baseURI + "/foo";
+  let postData = {};
+  let request;
+
+  setLanguage(languages[0]);
+
+  // A new request should create the stateful object for tracking the current
+  // language.
+  request = new HAWKAuthenticatedRESTRequest(url, credentials);
+  CommonUtils.nextTick(testFirstLanguage);
+
+  function testFirstLanguage() {
+    do_check_eq(languages[0], request._intl.accept_languages);
+
+    // Change the language pref ...
+    setLanguage(languages[1]); 
+    CommonUtils.nextTick(testRequest);
+  }
+
+  function testRequest() {
+    // Change of language pref should be picked up, which we can see on the
+    // server by inspecting the request headers.
+    request = new HAWKAuthenticatedRESTRequest(url, credentials);
+    request.post({}, function(error) {
+      do_check_null(error);
+      do_check_eq(200, this.response.status);
+
+      Services.prefs.resetUserPrefs();
+
+      server.stop(run_next_test);
+    });
+  }
+});
+
+add_task(function test_deriveHawkCredentials() {
+  let credentials = deriveHawkCredentials(
+    SESSION_KEYS.sessionToken, "sessionToken");
+
+  do_check_eq(credentials.algorithm, "sha256");
+  do_check_eq(credentials.id, SESSION_KEYS.tokenID);
+  do_check_eq(CommonUtils.bytesAsHex(credentials.key), SESSION_KEYS.reqHMACkey);
+});
+
+// turn formatted test vectors into normal hex strings
+function h(hexStr) {
+  return hexStr.replace(/\s+/g, "");
+}

services/common/tests/unit/test_kinto.js

@@ -0,0 +1,412 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/kinto-offline-client.js");
+Cu.import("resource://testing-common/httpd.js");
+
+const BinaryInputStream = Components.Constructor("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+var server;
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let kintoClient;
+
+function do_get_kinto_collection() {
+  if (!kintoClient) {
+    let config = {
+      remote:`http://localhost:${server.identity.primaryPort}/v1/`,
+      headers: {Authorization: "Basic " + btoa("user:pass")},
+      adapter: FirefoxAdapter
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection("test_collection");
+}
+
+function* clear_collection() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    yield collection.clear();
+  } finally {
+    yield collection.db.close();
+  }
+}
+
+// test some operations on a local collection
+add_task(function* test_kinto_add_get() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+
+    let newRecord = { foo: "bar" };
+    // check a record is created
+    let createResult = yield collection.create(newRecord);
+    do_check_eq(createResult.data.foo, newRecord.foo);
+    // check getting the record gets the same info
+    let getResult = yield collection.get(createResult.data.id);
+    deepEqual(createResult.data, getResult.data);
+    // check what happens if we create the same item again (it should throw
+    // since you can't create with id)
+    try {
+      yield collection.create(createResult.data);
+      do_throw("Creation of a record with an id should fail");
+    } catch (err) { }
+    // try a few creates without waiting for the first few to resolve
+    let promises = [];
+    promises.push(collection.create(newRecord));
+    promises.push(collection.create(newRecord));
+    promises.push(collection.create(newRecord));
+    yield collection.create(newRecord);
+    yield Promise.all(promises);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+// test some operations on multiple connections
+add_task(function* test_kinto_add_get() {
+  const collection1 = do_get_kinto_collection();
+  const collection2 = kintoClient.collection("test_collection_2");
+
+  try {
+    yield collection1.db.open();
+    yield collection2.db.open();
+
+    let newRecord = { foo: "bar" };
+
+    // perform several write operations alternately without waiting for promises
+    // to resolve
+    let promises = [];
+    for (let i = 0; i < 10; i++) {
+      promises.push(collection1.create(newRecord));
+      promises.push(collection2.create(newRecord));
+    }
+
+    // ensure subsequent operations still work
+    yield Promise.all([collection1.create(newRecord),
+                       collection2.create(newRecord)]);
+    yield Promise.all(promises);
+  } finally {
+    yield collection1.db.close();
+    yield collection2.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_kinto_update() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const newRecord = { foo: "bar" };
+    // check a record is created
+    let createResult = yield collection.create(newRecord);
+    do_check_eq(createResult.data.foo, newRecord.foo);
+    do_check_eq(createResult.data._status, "created");
+    // check we can update this OK
+    let copiedRecord = Object.assign(createResult.data, {});
+    deepEqual(createResult.data, copiedRecord);
+    copiedRecord.foo = "wibble";
+    let updateResult = yield collection.update(copiedRecord);
+    // check the field was updated
+    do_check_eq(updateResult.data.foo, copiedRecord.foo);
+    // check the status is still "created", since we haven't synced
+    // the record
+    do_check_eq(updateResult.data._status, "created");
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_kinto_clear() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+
+    // create an expected number of records
+    const expected = 10;
+    const newRecord = { foo: "bar" };
+    for (let i = 0; i < expected; i++) {
+      yield collection.create(newRecord);
+    }
+    // check the collection contains the correct number
+    let list = yield collection.list();
+    do_check_eq(list.data.length, expected);
+    // clear the collection and check again - should be 0
+    yield collection.clear();
+    list = yield collection.list();
+    do_check_eq(list.data.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_kinto_delete(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const newRecord = { foo: "bar" };
+    // check a record is created
+    let createResult = yield collection.create(newRecord);
+    do_check_eq(createResult.data.foo, newRecord.foo);
+    // check getting the record gets the same info
+    let getResult = yield collection.get(createResult.data.id);
+    deepEqual(createResult.data, getResult.data);
+    // delete that record
+    let deleteResult = yield collection.delete(createResult.data.id);
+    // check the ID is set on the result
+    do_check_eq(getResult.data.id, deleteResult.data.id);
+    // and check that get no longer returns the record
+    try {
+      getResult = yield collection.get(createResult.data.id);
+      do_throw("there should not be a result");
+    } catch (e) { }
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(function* test_kinto_list(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const expected = 10;
+    const created = [];
+    for (let i = 0; i < expected; i++) {
+      let newRecord = { foo: "test " + i };
+      let createResult = yield collection.create(newRecord);
+      created.push(createResult.data);
+    }
+    // check the collection contains the correct number
+    let list = yield collection.list();
+    do_check_eq(list.data.length, expected);
+
+    // check that all created records exist in the retrieved list
+    for (let createdRecord of created) {
+      let found = false;
+      for (let retrievedRecord of list.data) {
+        if (createdRecord.id == retrievedRecord.id) {
+          deepEqual(createdRecord, retrievedRecord);
+          found = true;
+        }
+      }
+      do_check_true(found);
+    }
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_ignores_already_imported_records(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const record = {id: "41b71c13-17e9-4ee3-9268-6a41abf9730f", title: "foo", last_modified: 1457896541};
+    yield collection.loadDump([record]);
+    let impactedRecords = yield collection.loadDump([record]);
+    do_check_eq(impactedRecords.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_should_overwrite_old_records(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const record = {id: "41b71c13-17e9-4ee3-9268-6a41abf9730f", title: "foo", last_modified: 1457896541};
+    yield collection.loadDump([record]);
+    const updated = Object.assign({}, record, {last_modified: 1457896543});
+    let impactedRecords = yield collection.loadDump([updated]);
+    do_check_eq(impactedRecords.length, 1);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_should_not_overwrite_unsynced_records(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const recordId = "41b71c13-17e9-4ee3-9268-6a41abf9730f";
+    yield collection.create({id: recordId, title: "foo"}, {useRecordId: true});
+    const record = {id: recordId, title: "bar", last_modified: 1457896541};
+    let impactedRecords = yield collection.loadDump([record]);
+    do_check_eq(impactedRecords.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_should_not_overwrite_records_without_last_modified(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const recordId = "41b71c13-17e9-4ee3-9268-6a41abf9730f";
+    yield collection.create({id: recordId, title: "foo"}, {synced: true});
+    const record = {id: recordId, title: "bar", last_modified: 1457896541};
+    let impactedRecords = yield collection.loadDump([record]);
+    do_check_eq(impactedRecords.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+// Now do some sanity checks against a server - we're not looking to test
+// core kinto.js functionality here (there is excellent test coverage in
+// kinto.js), more making sure things are basically working as expected.
+add_task(function* test_kinto_sync(){
+  const configPath = "/v1/";
+  const recordsPath = "/v1/buckets/default/collections/test_collection/records";
+  // register a handler
+  function handleResponse (request, response) {
+    try {
+      const sampled = getSampleResponse(request, server.identity.primaryPort);
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                             sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sampled.responseBody);
+    } catch (e) {
+      dump(`${e}\n`);
+    }
+  }
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(recordsPath, handleResponse);
+
+  // create an empty collection, sync to populate
+  const collection = do_get_kinto_collection();
+  try {
+    let result;
+
+    yield collection.db.open();
+    result = yield collection.sync();
+    do_check_true(result.ok);
+
+    // our test data has a single record; it should be in the local collection
+    let list = yield collection.list();
+    do_check_eq(list.data.length, 1);
+
+    // now sync again; we should now have 2 records
+    result = yield collection.sync();
+    do_check_true(result.ok);
+    list = yield collection.list();
+    do_check_eq(list.data.length, 2);
+
+    // sync again; the second records should have been modified
+    const before = list.data[0].title;
+    result = yield collection.sync();
+    do_check_true(result.ok);
+    list = yield collection.list();
+    const after = list.data[0].title;
+    do_check_neq(before, after);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1445606341071\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{"last_modified":1445606341071, "done":false, "id":"68db8313-686e-4fff-835e-07d78ad6f2af", "title":"New test"}]})
+    },
+    "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified&_since=1445606341071": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1445607941223\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{"last_modified":1445607941223, "done":false, "id":"901967b0-f729-4b30-8d8d-499cba7f4b1d", "title":"Another new test"}]})
+    },
+    "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified&_since=1445607941223": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1445607541265\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{"last_modified":1445607541265, "done":false, "id":"901967b0-f729-4b30-8d8d-499cba7f4b1d", "title":"Modified title"}]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/test_load_modules.js

@@ -0,0 +1,69 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Components.utils.import("resource://gre/modules/AppConstants.jsm");
+
+const MODULE_BASE = "resource://services-common/";
+const shared_modules = [
+  "async.js",
+  "logmanager.js",
+  "rest.js",
+  "stringbundle.js",
+  "utils.js",
+];
+
+const non_android_modules = [
+  "tokenserverclient.js",
+];
+
+const TEST_BASE = "resource://testing-common/services/common/";
+const shared_test_modules = [
+  "logging.js",
+];
+
+const non_android_test_modules = [
+  "storageserver.js",
+];
+
+function expectImportsToSucceed(mm, base=MODULE_BASE) {
+  for (let m of mm) {
+    let resource = base + m;
+    let succeeded = false;
+    try {
+      Components.utils.import(resource, {});
+      succeeded = true;
+    } catch (e) {}
+
+    if (!succeeded) {
+      throw "Importing " + resource + " should have succeeded!";
+    }
+  }
+}
+
+function expectImportsToFail(mm, base=MODULE_BASE) {
+  for (let m of mm) {
+    let resource = base + m;
+    let succeeded = false;
+    try {
+      Components.utils.import(resource, {});
+      succeeded = true;
+    } catch (e) {}
+
+    if (succeeded) {
+      throw "Importing " + resource + " should have failed!";
+    }
+  }
+}
+
+function run_test() {
+  expectImportsToSucceed(shared_modules);
+  expectImportsToSucceed(shared_test_modules, TEST_BASE);
+
+  if (AppConstants.platform != "android") {
+    expectImportsToSucceed(non_android_modules);
+    expectImportsToSucceed(non_android_test_modules, TEST_BASE);
+  } else {
+    expectImportsToFail(non_android_modules);
+    expectImportsToFail(non_android_test_modules, TEST_BASE);
+  }
+}

services/common/tests/unit/test_logmanager.js

@@ -0,0 +1,229 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// NOTE: The sync test_errorhandler_* tests have quite good coverage for
+// other aspects of this.
+
+Cu.import("resource://services-common/logmanager.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+
+function run_test() {
+  run_next_test();
+}
+
+// Returns an array of [consoleAppender, dumpAppender, [fileAppenders]] for
+// the specified log.  Note that fileAppenders will usually have length=1
+function getAppenders(log) {
+  let capps = log.appenders.filter(app => app instanceof Log.ConsoleAppender);
+  equal(capps.length, 1, "should only have one console appender");
+  let dapps = log.appenders.filter(app => app instanceof Log.DumpAppender);
+  equal(dapps.length, 1, "should only have one dump appender");
+  let fapps = log.appenders.filter(app => app instanceof Log.StorageStreamAppender);
+  return [capps[0], dapps[0], fapps];
+}
+
+// Test that the correct thing happens when no prefs exist for the log manager.
+add_task(function* test_noPrefs() {
+  // tell the log manager to init with a pref branch that doesn't exist.
+  let lm = new LogManager("no-such-branch.", ["TestLog"], "test");
+
+  let log = Log.repository.getLogger("TestLog");
+  let [capp, dapp, fapps] = getAppenders(log);
+  // The console appender gets "Fatal" while the "dump" appender gets "Error" levels
+  equal(capp.level, Log.Level.Fatal);
+  equal(dapp.level, Log.Level.Error);
+  // and the file (stream) appender gets Debug by default
+  equal(fapps.length, 1, "only 1 file appender");
+  equal(fapps[0].level, Log.Level.Debug);
+  lm.finalize();
+});
+
+// Test that changes to the prefs used by the log manager are updated dynamically.
+add_task(function* test_PrefChanges() {
+  Services.prefs.setCharPref("log-manager.test.log.appender.console", "Trace");
+  Services.prefs.setCharPref("log-manager.test.log.appender.dump", "Trace");
+  Services.prefs.setCharPref("log-manager.test.log.appender.file.level", "Trace");
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  let [capp, dapp, [fapp]] = getAppenders(log);
+  equal(capp.level, Log.Level.Trace);
+  equal(dapp.level, Log.Level.Trace);
+  equal(fapp.level, Log.Level.Trace);
+  // adjust the prefs and they should magically be reflected in the appenders.
+  Services.prefs.setCharPref("log-manager.test.log.appender.console", "Debug");
+  Services.prefs.setCharPref("log-manager.test.log.appender.dump", "Debug");
+  Services.prefs.setCharPref("log-manager.test.log.appender.file.level", "Debug");
+  equal(capp.level, Log.Level.Debug);
+  equal(dapp.level, Log.Level.Debug);
+  equal(fapp.level, Log.Level.Debug);
+  // and invalid values should cause them to fallback to their defaults.
+  Services.prefs.setCharPref("log-manager.test.log.appender.console", "xxx");
+  Services.prefs.setCharPref("log-manager.test.log.appender.dump", "xxx");
+  Services.prefs.setCharPref("log-manager.test.log.appender.file.level", "xxx");
+  equal(capp.level, Log.Level.Fatal);
+  equal(dapp.level, Log.Level.Error);
+  equal(fapp.level, Log.Level.Debug);
+  lm.finalize();
+});
+
+// Test that the same log used by multiple log managers does the right thing.
+add_task(function* test_SharedLogs() {
+  // create the prefs for the first instance.
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.console", "Trace");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.dump", "Trace");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.file.level", "Trace");
+  let lm1 = new LogManager("log-manager-1.test.", ["TestLog3"], "test");
+
+  // and the second.
+  Services.prefs.setCharPref("log-manager-2.test.log.appender.console", "Debug");
+  Services.prefs.setCharPref("log-manager-2.test.log.appender.dump", "Debug");
+  Services.prefs.setCharPref("log-manager-2.test.log.appender.file.level", "Debug");
+  let lm2 = new LogManager("log-manager-2.test.", ["TestLog3"], "test");
+
+  let log = Log.repository.getLogger("TestLog3");
+  let [capp, dapp, fapps] = getAppenders(log);
+
+  // console and dump appenders should be "trace" as it is more verbose than
+  // "debug"
+  equal(capp.level, Log.Level.Trace);
+  equal(dapp.level, Log.Level.Trace);
+
+  // Set the prefs on the -1 branch to "Error" - it should then end up with
+  // "Debug" from the -2 branch.
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.console", "Error");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.dump", "Error");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.file.level", "Error");
+
+  equal(capp.level, Log.Level.Debug);
+  equal(dapp.level, Log.Level.Debug);
+
+  lm1.finalize();
+  lm2.finalize();
+});
+
+// A little helper to test what log files exist.  We expect exactly zero (if
+// prefix is null) or exactly one with the specified prefix.
+function checkLogFile(prefix) {
+  let logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
+  let entries = logsdir.directoryEntries;
+  if (!prefix) {
+    // expecting no files.
+    ok(!entries.hasMoreElements());
+  } else {
+    // expecting 1 file.
+    ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+    equal(logfile.leafName.slice(-4), ".txt");
+    ok(logfile.leafName.startsWith(prefix + "-test-"), logfile.leafName);
+    // and remove it ready for the next check.
+    logfile.remove(false);
+  }
+}
+
+// Test that we correctly write error logs by default
+add_task(function* test_logFileErrorDefault() {
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  log.error("an error message");
+  yield lm.resetFileLog(lm.REASON_ERROR);
+  // One error log file exists.
+  checkLogFile("error");
+
+  lm.finalize();
+});
+
+// Test that we correctly write success logs.
+add_task(function* test_logFileSuccess() {
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnError", false);
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", false);
+
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  log.info("an info message");
+  yield lm.resetFileLog();
+  // Zero log files exist.
+  checkLogFile(null);
+
+  // Reset logOnSuccess and do it again - log should appear.
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", true);
+  log.info("an info message");
+  yield lm.resetFileLog();
+
+  checkLogFile("success");
+
+  // Now test with no "reason" specified and no "error" record.
+  log.info("an info message");
+  yield lm.resetFileLog();
+  // should get a "success" entry.
+  checkLogFile("success");
+
+  // With no "reason" and an error record - should get no success log.
+  log.error("an error message");
+  yield lm.resetFileLog();
+  // should get no entry
+  checkLogFile(null);
+
+  // And finally now with no error, to ensure that the fact we had an error
+  // previously doesn't persist after the .resetFileLog call.
+  log.info("an info message");
+  yield lm.resetFileLog();
+  checkLogFile("success");
+
+  lm.finalize();
+});
+
+// Test that we correctly write error logs.
+add_task(function* test_logFileError() {
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnError", false);
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", false);
+
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  log.info("an info message");
+  let reason = yield lm.resetFileLog();
+  Assert.equal(reason, null, "null returned when no file created.");
+  // Zero log files exist.
+  checkLogFile(null);
+
+  // Reset logOnSuccess - success logs should appear if no error records.
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", true);
+  log.info("an info message");
+  reason = yield lm.resetFileLog();
+  Assert.equal(reason, lm.SUCCESS_LOG_WRITTEN);
+  checkLogFile("success");
+
+  // Set logOnError and unset logOnSuccess - error logs should appear.
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", false);
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnError", true);
+  log.error("an error message");
+  reason = yield lm.resetFileLog();
+  Assert.equal(reason, lm.ERROR_LOG_WRITTEN);
+  checkLogFile("error");
+
+  // Now test with no "error" record.
+  log.info("an info message");
+  reason = yield lm.resetFileLog();
+  // should get no file
+  Assert.equal(reason, null);
+  checkLogFile(null);
+
+  // With an error record we should get an error log.
+  log.error("an error message");
+  reason = yield lm.resetFileLog();
+  // should get en error log
+  Assert.equal(reason, lm.ERROR_LOG_WRITTEN);
+  checkLogFile("error");
+
+  // And finally now with success, to ensure that the fact we had an error
+  // previously doesn't persist after the .resetFileLog call.
+  log.info("an info message");
+  yield lm.resetFileLog();
+  checkLogFile(null);
+
+  lm.finalize();
+});

services/common/tests/unit/test_observers.js

@@ -0,0 +1,84 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Components.utils.import("resource://services-common/observers.js");
+
+var gSubject = {};
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_function_observer() {
+  let foo = false;
+
+  let onFoo = function(subject, data) {
+    foo = !foo;
+    do_check_eq(subject, gSubject);
+    do_check_eq(data, "some data");
+  };
+
+  Observers.add("foo", onFoo);
+  Observers.notify("foo", gSubject, "some data");
+
+  // The observer was notified after being added.
+  do_check_true(foo);
+
+  Observers.remove("foo", onFoo);
+  Observers.notify("foo");
+
+  // The observer was not notified after being removed.
+  do_check_true(foo);
+
+  run_next_test();
+});
+
+add_test(function test_method_observer() {
+  let obj = {
+    foo: false,
+    onFoo: function(subject, data) {
+      this.foo = !this.foo;
+      do_check_eq(subject, gSubject);
+      do_check_eq(data, "some data");
+    }
+  };
+
+  // The observer is notified after being added.
+  Observers.add("foo", obj.onFoo, obj);
+  Observers.notify("foo", gSubject, "some data");
+  do_check_true(obj.foo);
+
+  // The observer is not notified after being removed.
+  Observers.remove("foo", obj.onFoo, obj);
+  Observers.notify("foo");
+  do_check_true(obj.foo);
+
+  run_next_test();
+});
+
+add_test(function test_object_observer() {
+  let obj = {
+    foo: false,
+    observe: function(subject, topic, data) {
+      this.foo = !this.foo;
+
+      do_check_eq(subject, gSubject);
+      do_check_eq(topic, "foo");
+      do_check_eq(data, "some data");
+    }
+  };
+
+  Observers.add("foo", obj);
+  Observers.notify("foo", gSubject, "some data");
+
+  // The observer is notified after being added.
+  do_check_true(obj.foo);
+
+  Observers.remove("foo", obj);
+  Observers.notify("foo");
+
+  // The observer is not notified after being removed.
+  do_check_true(obj.foo);
+
+  run_next_test();
+});

services/common/tests/unit/test_restrequest.js

@@ -0,0 +1,873 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/NetUtil.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  Log.repository.getLogger("Services.Common.RESTRequest").level =
+    Log.Level.Trace;
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+/**
+ * Initializing a RESTRequest with an invalid URI throws
+ * NS_ERROR_MALFORMED_URI.
+ */
+add_test(function test_invalid_uri() {
+  do_check_throws(function() {
+    new RESTRequest("an invalid URI");
+  }, Cr.NS_ERROR_MALFORMED_URI);
+  run_next_test();
+});
+
+/**
+ * Verify initial values for attributes.
+ */
+add_test(function test_attributes() {
+  let uri = "http://foo.com/bar/baz";
+  let request = new RESTRequest(uri);
+
+  do_check_true(request.uri instanceof Ci.nsIURI);
+  do_check_eq(request.uri.spec, uri);
+  do_check_eq(request.response, null);
+  do_check_eq(request.status, request.NOT_SENT);
+  let expectedLoadFlags = Ci.nsIRequest.LOAD_BYPASS_CACHE |
+                          Ci.nsIRequest.INHIBIT_CACHING |
+                          Ci.nsIRequest.LOAD_ANONYMOUS;
+  do_check_eq(request.loadFlags, expectedLoadFlags);
+
+  run_next_test();
+});
+
+/**
+ * Verify that a proxy auth redirect doesn't break us. This has to be the first
+ * request made in the file!
+ */
+add_test(function test_proxy_auth_redirect() {
+  let pacFetched = false;
+  function pacHandler(metadata, response) {
+    pacFetched = true;
+    let body = 'function FindProxyForURL(url, host) { return "DIRECT"; }';
+    response.setStatusLine(metadata.httpVersion, 200, "OK");
+    response.setHeader("Content-Type", "application/x-ns-proxy-autoconfig", false);
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let fetched = false;
+  function original(metadata, response) {
+    fetched = true;
+    let body = "TADA!";
+    response.setStatusLine(metadata.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let server = httpd_setup({
+    "/original": original,
+    "/pac3":     pacHandler
+  });
+  PACSystemSettings.PACURI = server.baseURI + "/pac3";
+  installFakePAC();
+
+  let res = new RESTRequest(server.baseURI + "/original");
+  res.get(function (error) {
+    do_check_true(pacFetched);
+    do_check_true(fetched);
+    do_check_true(!error);
+    do_check_true(this.response.success);
+    do_check_eq("TADA!", this.response.body);
+    uninstallFakePAC();
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Ensure that failures that cause asyncOpen to throw
+ * result in callbacks being invoked.
+ * Bug 826086.
+ */
+add_test(function test_forbidden_port() {
+  let request = new RESTRequest("http://localhost:6000/");
+  request.get(function(error) {
+    if (!error) {
+      do_throw("Should have got an error.");
+    }
+    do_check_eq(error.result, Components.results.NS_ERROR_PORT_ACCESS_NOT_ALLOWED);
+    run_next_test();
+  });
+});
+
+/**
+ * Demonstrate API short-hand: create a request and dispatch it immediately.
+ */
+add_test(function test_simple_get() {
+  let handler = httpd_handler(200, "OK", "Huzzah!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource").get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Huzzah!");
+
+    server.stop(run_next_test);
+  });
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, "GET");
+});
+
+/**
+ * Test HTTP GET with all bells and whistles.
+ */
+add_test(function test_get() {
+  let handler = httpd_handler(200, "OK", "Huzzah!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  do_check_eq(request.status, request.NOT_SENT);
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("This function should have been overwritten!");
+  };
+
+  let onProgress_called = false;
+  function onProgress() {
+    onProgress_called = true;
+    do_check_eq(this.status, request.IN_PROGRESS);
+    do_check_true(this.response.body.length > 0);
+
+    do_check_true(!!(this.channel.loadFlags & Ci.nsIRequest.LOAD_BYPASS_CACHE));
+    do_check_true(!!(this.channel.loadFlags & Ci.nsIRequest.INHIBIT_CACHING));
+  };
+
+  function onComplete(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Huzzah!");
+    do_check_eq(handler.request.method, "GET");
+
+    do_check_true(onProgress_called);
+    CommonUtils.nextTick(function () {
+      do_check_eq(request.onComplete, null);
+      do_check_eq(request.onProgress, null);
+      server.stop(run_next_test);
+    });
+  };
+
+  do_check_eq(request.get(onComplete, onProgress), request);
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, "GET");
+  do_check_throws(function () {
+    request.get();
+  });
+});
+
+/**
+ * Test HTTP GET with UTF-8 content, and custom Content-Type.
+ */
+add_test(function test_get_utf8() {
+  let response = "Hello World or Καλημέρα κόσμε or こんにちは 世界";
+
+  let contentType = "text/plain";
+  let charset = true;
+  let charsetSuffix = "; charset=UTF-8";
+
+  let server = httpd_setup({"/resource": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", contentType + (charset ? charsetSuffix : ""));
+
+    let converter = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                    .createInstance(Ci.nsIConverterOutputStream);
+    converter.init(res.bodyOutputStream, "UTF-8", 0, 0x0000);
+    converter.writeString(response);
+    converter.close();
+  }});
+
+  // Check if charset in Content-Type is propertly interpreted.
+  let request1 = new RESTRequest(server.baseURI + "/resource");
+  request1.get(function(error) {
+    do_check_null(error);
+
+    do_check_eq(request1.response.status, 200);
+    do_check_eq(request1.response.body, response);
+    do_check_eq(request1.response.headers["content-type"],
+                contentType + charsetSuffix);
+
+    // Check that we default to UTF-8 if Content-Type doesn't have a charset.
+    charset = false;
+    let request2 = new RESTRequest(server.baseURI + "/resource");
+    request2.get(function(error) {
+      do_check_null(error);
+
+      do_check_eq(request2.response.status, 200);
+      do_check_eq(request2.response.body, response);
+      do_check_eq(request2.response.headers["content-type"], contentType);
+      do_check_eq(request2.response.charset, "utf-8");
+
+      server.stop(run_next_test);
+    });
+  });
+});
+
+/**
+ * Test HTTP POST data is encoded as UTF-8 by default.
+ */
+add_test(function test_post_utf8() {
+  // We setup a handler that responds with exactly what it received.
+  // Given we've already tested above that responses are correctly utf-8
+  // decoded we can surmise that the correct response coming back means the
+  // input must also have been encoded.
+  let server = httpd_setup({"/echo": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", req.getHeader("content-type"));
+    // Get the body as bytes and write them back without touching them
+    let sis = Cc["@mozilla.org/scriptableinputstream;1"]
+              .createInstance(Ci.nsIScriptableInputStream);
+    sis.init(req.bodyInputStream);
+    let body = sis.read(sis.available());
+    sis.close()
+    res.write(body);
+  }});
+
+  let data = {copyright: "\xa9"}; // \xa9 is the copyright symbol
+  let request1 = new RESTRequest(server.baseURI + "/echo");
+  request1.post(data, function(error) {
+    do_check_null(error);
+
+    do_check_eq(request1.response.status, 200);
+    deepEqual(JSON.parse(request1.response.body), data);
+    do_check_eq(request1.response.headers["content-type"],
+                "application/json; charset=utf-8")
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test more variations of charset handling.
+ */
+add_test(function test_charsets() {
+  let response = "Hello World, I can't speak Russian";
+
+  let contentType = "text/plain";
+  let charset = true;
+  let charsetSuffix = "; charset=us-ascii";
+
+  let server = httpd_setup({"/resource": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", contentType + (charset ? charsetSuffix : ""));
+
+    let converter = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                    .createInstance(Ci.nsIConverterOutputStream);
+    converter.init(res.bodyOutputStream, "us-ascii", 0, 0x0000);
+    converter.writeString(response);
+    converter.close();
+  }});
+
+  // Check that provided charset overrides hint.
+  let request1 = new RESTRequest(server.baseURI + "/resource");
+  request1.charset = "not-a-charset";
+  request1.get(function(error) {
+    do_check_null(error);
+
+    do_check_eq(request1.response.status, 200);
+    do_check_eq(request1.response.body, response);
+    do_check_eq(request1.response.headers["content-type"],
+                contentType + charsetSuffix);
+    do_check_eq(request1.response.charset, "us-ascii");
+
+    // Check that hint is used if Content-Type doesn't have a charset.
+    charset = false;
+    let request2 = new RESTRequest(server.baseURI + "/resource");
+    request2.charset = "us-ascii";
+    request2.get(function(error) {
+      do_check_null(error);
+
+      do_check_eq(request2.response.status, 200);
+      do_check_eq(request2.response.body, response);
+      do_check_eq(request2.response.headers["content-type"], contentType);
+      do_check_eq(request2.response.charset, "us-ascii");
+
+      server.stop(run_next_test);
+    });
+  });
+});
+
+/**
+ * Used for testing PATCH/PUT/POST methods.
+ */
+function check_posting_data(method) {
+  let funcName = method.toLowerCase();
+  let handler = httpd_handler(200, "OK", "Got it!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  do_check_eq(request.status, request.NOT_SENT);
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("This function should have been overwritten!");
+  };
+
+  let onProgress_called = false;
+  function onProgress() {
+    onProgress_called = true;
+    do_check_eq(this.status, request.IN_PROGRESS);
+    do_check_true(this.response.body.length > 0);
+  };
+
+  function onComplete(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Got it!");
+
+    do_check_eq(handler.request.method, method);
+    do_check_eq(handler.request.body, "Hullo?");
+    do_check_eq(handler.request.getHeader("Content-Type"), "text/plain");
+
+    do_check_true(onProgress_called);
+    CommonUtils.nextTick(function () {
+      do_check_eq(request.onComplete, null);
+      do_check_eq(request.onProgress, null);
+      server.stop(run_next_test);
+    });
+  };
+
+  do_check_eq(request[funcName]("Hullo?", onComplete, onProgress), request);
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, method);
+  do_check_throws(function () {
+    request[funcName]("Hai!");
+  });
+}
+
+/**
+ * Test HTTP PATCH with a simple string argument and default Content-Type.
+ */
+add_test(function test_patch() {
+  check_posting_data("PATCH");
+});
+
+/**
+ * Test HTTP PUT with a simple string argument and default Content-Type.
+ */
+add_test(function test_put() {
+  check_posting_data("PUT");
+});
+
+/**
+ * Test HTTP POST with a simple string argument and default Content-Type.
+ */
+add_test(function test_post() {
+  check_posting_data("POST");
+});
+
+/**
+ * Test HTTP DELETE.
+ */
+add_test(function test_delete() {
+  let handler = httpd_handler(200, "OK", "Got it!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  do_check_eq(request.status, request.NOT_SENT);
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("This function should have been overwritten!");
+  };
+
+  let onProgress_called = false;
+  function onProgress() {
+    onProgress_called = true;
+    do_check_eq(this.status, request.IN_PROGRESS);
+    do_check_true(this.response.body.length > 0);
+  };
+
+  function onComplete(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Got it!");
+    do_check_eq(handler.request.method, "DELETE");
+
+    do_check_true(onProgress_called);
+    CommonUtils.nextTick(function () {
+      do_check_eq(request.onComplete, null);
+      do_check_eq(request.onProgress, null);
+      server.stop(run_next_test);
+    });
+  };
+
+  do_check_eq(request.delete(onComplete, onProgress), request);
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, "DELETE");
+  do_check_throws(function () {
+    request.delete();
+  });
+});
+
+/**
+ * Test an HTTP response with a non-200 status code.
+ */
+add_test(function test_get_404() {
+  let handler = httpd_handler(404, "Not Found", "Cannae find it!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_false(this.response.success);
+    do_check_eq(this.response.status, 404);
+    do_check_eq(this.response.body, "Cannae find it!");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The 'data' argument to PUT, if not a string already, is automatically
+ * stringified as JSON.
+ */
+add_test(function test_put_json() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let sample_data = {
+    some: "sample_data",
+    injson: "format",
+    number: 42
+  };
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.put(sample_data, function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "PUT");
+    do_check_eq(handler.request.body, JSON.stringify(sample_data));
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/json; charset=utf-8");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The 'data' argument to POST, if not a string already, is automatically
+ * stringified as JSON.
+ */
+add_test(function test_post_json() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let sample_data = {
+    some: "sample_data",
+    injson: "format",
+    number: 42
+  };
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.post(sample_data, function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "POST");
+    do_check_eq(handler.request.body, JSON.stringify(sample_data));
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/json; charset=utf-8");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The content-type will be text/plain without a charset if the 'data' argument
+ * to POST is already a string.
+ */
+add_test(function test_post_json() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let sample_data = "hello";
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.post(sample_data, function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "POST");
+    do_check_eq(handler.request.body, sample_data);
+    do_check_eq(handler.request.getHeader("Content-Type"), "text/plain");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * HTTP PUT with a custom Content-Type header.
+ */
+add_test(function test_put_override_content_type() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.setHeader("Content-Type", "application/lolcat");
+  request.put("O HAI!!1!", function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "PUT");
+    do_check_eq(handler.request.body, "O HAI!!1!");
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/lolcat");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * HTTP POST with a custom Content-Type header.
+ */
+add_test(function test_post_override_content_type() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.setHeader("Content-Type", "application/lolcat");
+  request.post("O HAI!!1!", function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "POST");
+    do_check_eq(handler.request.body, "O HAI!!1!");
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/lolcat");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * No special headers are sent by default on a GET request.
+ */
+add_test(function test_get_no_headers() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let ignore_headers = ["host", "user-agent", "accept", "accept-language",
+                        "accept-encoding", "accept-charset", "keep-alive",
+                        "connection", "pragma", "cache-control",
+                        "content-length"];
+
+  new RESTRequest(server.baseURI + "/resource").get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    let server_headers = handler.request.headers;
+    while (server_headers.hasMoreElements()) {
+      let header = server_headers.getNext().toString();
+      if (ignore_headers.indexOf(header) == -1) {
+        do_throw("Got unexpected header!");
+      }
+    }
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test changing the URI after having created the request.
+ */
+add_test(function test_changing_uri() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest("http://localhost:1234/the-wrong-resource");
+  request.uri = CommonUtils.makeURI(server.baseURI + "/resource");
+  request.get(function (error) {
+    do_check_eq(error, null);
+    do_check_eq(this.response.status, 200);
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test setting HTTP request headers.
+ */
+add_test(function test_request_setHeader() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+
+  request.setHeader("X-What-Is-Weave", "awesome");
+  request.setHeader("X-WHAT-is-Weave", "more awesomer");
+  request.setHeader("Another-Header", "Hello World");
+
+  request.get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.getHeader("X-What-Is-Weave"), "more awesomer");
+    do_check_eq(handler.request.getHeader("another-header"), "Hello World");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test receiving HTTP response headers.
+ */
+add_test(function test_response_headers() {
+  function handler(request, response) {
+    response.setHeader("X-What-Is-Weave", "awesome");
+    response.setHeader("Another-Header", "Hello World");
+    response.setStatusLine(request.httpVersion, 200, "OK");
+  }
+  let server = httpd_setup({"/resource": handler});
+  let request = new RESTRequest(server.baseURI + "/resource");
+
+  request.get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(this.response.headers["x-what-is-weave"], "awesome");
+    do_check_eq(this.response.headers["another-header"], "Hello World");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The onComplete() handler gets called in case of any network errors
+ * (e.g. NS_ERROR_CONNECTION_REFUSED).
+ */
+add_test(function test_connection_refused() {
+  let request = new RESTRequest("http://localhost:1234/resource");
+  request.onProgress = function onProgress() {
+    do_throw("Shouldn't have called request.onProgress()!");
+  };
+  request.get(function (error) {
+    do_check_eq(error.result, Cr.NS_ERROR_CONNECTION_REFUSED);
+    do_check_eq(error.message, "NS_ERROR_CONNECTION_REFUSED");
+    do_check_eq(this.status, this.COMPLETED);
+    run_next_test();
+  });
+  do_check_eq(request.status, request.SENT);
+});
+
+/**
+ * Abort a request that just sent off.
+ */
+add_test(function test_abort() {
+  function handler() {
+    do_throw("Shouldn't have gotten here!");
+  }
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+
+  // Aborting a request that hasn't been sent yet is pointless and will throw.
+  do_check_throws(function () {
+    request.abort();
+  });
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("Shouldn't have gotten here!");
+  };
+  request.get();
+  request.abort();
+
+  // Aborting an already aborted request is pointless and will throw.
+  do_check_throws(function () {
+    request.abort();
+  });
+
+  do_check_eq(request.status, request.ABORTED);
+  CommonUtils.nextTick(function () {
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * A non-zero 'timeout' property specifies the amount of seconds to wait after
+ * channel activity until the request is automatically canceled.
+ */
+add_test(function test_timeout() {
+  let server = new HttpServer();
+  let server_connection;
+  server._handler.handleResponse = function(connection) {
+    // This is a handler that doesn't do anything, just keeps the connection
+    // open, thereby mimicking a timing out connection. We keep a reference to
+    // the open connection for later so it can be properly disposed of. That's
+    // why you really only want to make one HTTP request to this server ever.
+    server_connection = connection;
+  };
+  server.start();
+  let identity = server.identity;
+  let uri = identity.primaryScheme + "://" + identity.primaryHost + ":" +
+            identity.primaryPort;
+
+  let request = new RESTRequest(uri + "/resource");
+  request.timeout = 0.1; // 100 milliseconds
+  request.get(function (error) {
+    do_check_eq(error.result, Cr.NS_ERROR_NET_TIMEOUT);
+    do_check_eq(this.status, this.ABORTED);
+
+    // server_connection is undefined on the Android emulator for reasons
+    // unknown. Yet, we still get here. If this test is refactored, we should
+    // investigate the reason why the above callback is behaving differently.
+    if (server_connection) {
+      _("Closing connection.");
+      server_connection.close();
+    }
+
+    _("Shutting down server.");
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * An exception thrown in 'onProgress' propagates to the 'onComplete' handler.
+ */
+add_test(function test_exception_in_onProgress() {
+  let handler = httpd_handler(200, "OK", "Foobar");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.onProgress = function onProgress() {
+    it.does.not.exist();
+  };
+  request.get(function onComplete(error) {
+    do_check_eq(error, "ReferenceError: it is not defined");
+    do_check_eq(this.status, this.ABORTED);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_new_channel() {
+  _("Ensure a redirect to a new channel is handled properly.");
+
+  function checkUA(metadata) {
+    let ua = metadata.getHeader("User-Agent");
+    _("User-Agent is " + ua);
+    do_check_eq("foo bar", ua);
+  }
+
+  let redirectRequested = false;
+  let redirectURL;
+  function redirectHandler(metadata, response) {
+    checkUA(metadata);
+    redirectRequested = true;
+
+    let body = "Redirecting";
+    response.setStatusLine(metadata.httpVersion, 307, "TEMPORARY REDIRECT");
+    response.setHeader("Location", redirectURL);
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let resourceRequested = false;
+  function resourceHandler(metadata, response) {
+    checkUA(metadata);
+    resourceRequested = true;
+
+    let body = "Test";
+    response.setHeader("Content-Type", "text/plain");
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let server1 = httpd_setup({"/redirect": redirectHandler});
+  let server2 = httpd_setup({"/resource": resourceHandler});
+  redirectURL = server2.baseURI + "/resource";
+
+  function advance() {
+    server1.stop(function () {
+      server2.stop(run_next_test);
+    });
+  }
+
+  let request = new RESTRequest(server1.baseURI + "/redirect");
+  request.setHeader("User-Agent", "foo bar");
+
+  // Swizzle in our own fakery, because this redirect is neither
+  // internal nor URI-preserving. RESTRequest's policy is to only
+  // copy headers under certain circumstances.
+  let protoMethod = request.shouldCopyOnRedirect;
+  request.shouldCopyOnRedirect = function wrapped(o, n, f) {
+    // Check the default policy.
+    do_check_false(protoMethod.call(this, o, n, f));
+    return true;
+  };
+
+  request.get(function onComplete(error) {
+    let response = this.response;
+
+    do_check_eq(200, response.status);
+    do_check_eq("Test", response.body);
+    do_check_true(redirectRequested);
+    do_check_true(resourceRequested);
+
+    advance();
+  });
+});
+
+add_test(function test_not_sending_cookie() {
+  function handler(metadata, response) {
+    let body = "COOKIE!";
+    response.setStatusLine(metadata.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(body, body.length);
+    do_check_false(metadata.hasHeader("Cookie"));
+  }
+  let server = httpd_setup({"/test": handler});
+
+  let cookieSer = Cc["@mozilla.org/cookieService;1"]
+                    .getService(Ci.nsICookieService);
+  let uri = CommonUtils.makeURI(server.baseURI);
+  cookieSer.setCookieString(uri, null, "test=test; path=/;", null);
+
+  let res = new RESTRequest(server.baseURI + "/test");
+  res.get(function (error) {
+    do_check_null(error);
+    do_check_true(this.response.success);
+    do_check_eq("COOKIE!", this.response.body);
+    server.stop(run_next_test);
+  });
+});
+

services/common/tests/unit/test_storage_adapter.js

@@ -0,0 +1,269 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/kinto-offline-client.js");
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let gFirefoxAdapter = null;
+
+function do_get_kinto_adapter() {
+  if (gFirefoxAdapter == null) {
+    gFirefoxAdapter = new FirefoxAdapter("test");
+  }
+  return gFirefoxAdapter;
+}
+
+function do_get_kinto_db() {
+  let profile = do_get_profile();
+  let kintoDB = profile.clone();
+  kintoDB.append(kintoFilename);
+  return kintoDB;
+}
+
+function cleanup_kinto() {
+  add_test(function cleanup_kinto_files(){
+    let kintoDB = do_get_kinto_db();
+    // clean up the db
+    kintoDB.remove(false);
+    // force re-creation of the adapter
+    gFirefoxAdapter = null;
+    run_next_test();
+  });
+}
+
+function test_collection_operations() {
+  add_task(function* test_kinto_clear() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.clear();
+    yield adapter.close();
+  });
+
+  // test creating new records... and getting them again
+  add_task(function* test_kinto_create_new_get_existing() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let record = {id:"test-id", foo:"bar"};
+    yield adapter.execute((transaction) => transaction.create(record));
+    let newRecord = yield adapter.get("test-id");
+    // ensure the record is the same as when it was added
+    deepEqual(record, newRecord);
+    yield adapter.close();
+  });
+
+  // test removing records
+  add_task(function* test_kinto_can_remove_some_records() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    // create a second record
+    let record = {id:"test-id-2", foo:"baz"};
+    yield adapter.execute((transaction) => transaction.create(record));
+    let newRecord = yield adapter.get("test-id-2");
+    deepEqual(record, newRecord);
+    // delete the record
+    yield adapter.execute((transaction) => transaction.delete(record.id));
+    newRecord = yield adapter.get(record.id);
+    // ... and ensure it's no longer there
+    do_check_eq(newRecord, undefined);
+    // ensure the other record still exists
+    newRecord = yield adapter.get("test-id");
+    do_check_neq(newRecord, undefined);
+    yield adapter.close();
+  });
+
+  // test getting records that don't exist
+  add_task(function* test_kinto_get_non_existant() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    // Kinto expects adapters to either:
+    let newRecord = yield adapter.get("missing-test-id");
+    // resolve with an undefined record
+    do_check_eq(newRecord, undefined);
+    yield adapter.close();
+  });
+
+  // test updating records... and getting them again
+  add_task(function* test_kinto_update_get_existing() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let originalRecord = {id:"test-id", foo:"bar"};
+    let updatedRecord = {id:"test-id", foo:"baz"};
+    yield adapter.clear();
+    yield adapter.execute((transaction) => transaction.create(originalRecord));
+    yield adapter.execute((transaction) => transaction.update(updatedRecord));
+    // ensure the record exists
+    let newRecord = yield adapter.get("test-id");
+    // ensure the record is the same as when it was added
+    deepEqual(updatedRecord, newRecord);
+    yield adapter.close();
+  });
+
+  // test listing records
+  add_task(function* test_kinto_list() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let originalRecord = {id:"test-id-1", foo:"bar"};
+    let records = yield adapter.list();
+    do_check_eq(records.length, 1);
+    yield adapter.execute((transaction) => transaction.create(originalRecord));
+    records = yield adapter.list();
+    do_check_eq(records.length, 2);
+    yield adapter.close();
+  });
+
+  // test aborting transaction
+  add_task(function* test_kinto_aborting_transaction() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.clear();
+    let record = {id: 1, foo: "bar"};
+    let error = null;
+    try {
+      yield adapter.execute((transaction) => {
+        transaction.create(record);
+        throw new Error("unexpected");
+      });
+    } catch (e) {
+      error = e;
+    }
+    do_check_neq(error, null);
+    records = yield adapter.list();
+    do_check_eq(records.length, 0);
+    yield adapter.close();
+  });
+
+  // test save and get last modified
+  add_task(function* test_kinto_last_modified() {
+    const initialValue = 0;
+    const intendedValue = 12345678;
+
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, initialValue);
+    let result = yield adapter.saveLastModified(intendedValue);
+    do_check_eq(result, intendedValue);
+    lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, intendedValue);
+
+    // test saveLastModified parses values correctly
+    result = yield adapter.saveLastModified(" " + intendedValue + " blah");
+    // should resolve with the parsed int
+    do_check_eq(result, intendedValue);
+    // and should have saved correctly
+    lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, intendedValue);
+    yield adapter.close();
+  });
+
+  // test loadDump(records)
+  add_task(function* test_kinto_import_records() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let record1 = {id: 1, foo: "bar"};
+    let record2 = {id: 2, foo: "baz"};
+    let impactedRecords = yield adapter.loadDump([
+      record1, record2
+    ]);
+    do_check_eq(impactedRecords.length, 2);
+    let newRecord1 = yield adapter.get("1");
+    // ensure the record is the same as when it was added
+    deepEqual(record1, newRecord1);
+    let newRecord2 = yield adapter.get("2");
+    // ensure the record is the same as when it was added
+    deepEqual(record2, newRecord2);
+    yield adapter.close();
+  });
+
+  add_task(function* test_kinto_import_records_should_override_existing() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.clear();
+    records = yield adapter.list();
+    do_check_eq(records.length, 0);
+    let impactedRecords = yield adapter.loadDump([
+      {id: 1, foo: "bar"},
+      {id: 2, foo: "baz"},
+    ]);
+    do_check_eq(impactedRecords.length, 2);
+    yield adapter.loadDump([
+      {id: 1, foo: "baz"},
+      {id: 3, foo: "bab"},
+    ]);
+    records = yield adapter.list();
+    do_check_eq(records.length, 3);
+    let newRecord1 = yield adapter.get("1");
+    deepEqual(newRecord1.foo, "baz");
+    yield adapter.close();
+  });
+
+  add_task(function* test_import_updates_lastModified() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.loadDump([
+      {id: 1, foo: "bar", last_modified: 1457896541},
+      {id: 2, foo: "baz", last_modified: 1458796542},
+    ]);
+    let lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, 1458796542);
+    yield adapter.close();
+  });
+
+  add_task(function* test_import_preserves_older_lastModified() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.saveLastModified(1458796543);
+
+    yield adapter.loadDump([
+      {id: 1, foo: "bar", last_modified: 1457896541},
+      {id: 2, foo: "baz", last_modified: 1458796542},
+    ]);
+    let lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, 1458796543);
+    yield adapter.close();
+  });
+}
+
+// test kinto db setup and operations in various scenarios
+// test from scratch - no current existing database
+add_test(function test_db_creation() {
+  add_test(function test_create_from_scratch() {
+    // ensure the file does not exist in the profile
+    let kintoDB = do_get_kinto_db();
+    do_check_false(kintoDB.exists());
+    run_next_test();
+  });
+
+  test_collection_operations();
+
+  cleanup_kinto();
+  run_next_test();
+});
+
+// this is the closest we can get to a schema version upgrade at v1 - test an
+// existing database
+add_test(function test_creation_from_empty_db() {
+  add_test(function test_create_from_empty_db() {
+    // place an empty kinto db file in the profile
+    let profile = do_get_profile();
+    let kintoDB = do_get_kinto_db();
+
+    let emptyDB = do_get_file("test_storage_adapter/empty.sqlite");
+    emptyDB.copyTo(profile,kintoFilename);
+
+    run_next_test();
+  });
+
+  test_collection_operations();
+
+  cleanup_kinto();
+  run_next_test();
+});
+
+function run_test() {
+  run_next_test();
+}

services/common/tests/unit/test_storage_server.js

@@ -0,0 +1,692 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://testing-common/services/common/storageserver.js");
+
+const DEFAULT_USER = "123";
+const DEFAULT_PASSWORD = "password";
+
+/**
+ * Helper function to prepare a RESTRequest against the server.
+ */
+function localRequest(server, path, user=DEFAULT_USER, password=DEFAULT_PASSWORD) {
+  _("localRequest: " + path);
+  let identity = server.server.identity;
+  let url = identity.primaryScheme + "://" + identity.primaryHost + ":" +
+            identity.primaryPort + path;
+  _("url: " + url);
+  let req = new RESTRequest(url);
+
+  let header = basic_auth_header(user, password);
+  req.setHeader("Authorization", header);
+  req.setHeader("Accept", "application/json");
+
+  return req;
+}
+
+/**
+ * Helper function to validate an HTTP response from the server.
+ */
+function validateResponse(response) {
+  do_check_true("x-timestamp" in response.headers);
+
+  if ("content-length" in response.headers) {
+    let cl = parseInt(response.headers["content-length"]);
+
+    if (cl != 0) {
+      do_check_true("content-type" in response.headers);
+      do_check_eq("application/json", response.headers["content-type"]);
+    }
+  }
+
+  if (response.status == 204 || response.status == 304) {
+    do_check_false("content-type" in response.headers);
+
+    if ("content-length" in response.headers) {
+      do_check_eq(response.headers["content-length"], "0");
+    }
+  }
+
+  if (response.status == 405) {
+    do_check_true("allow" in response.headers);
+  }
+}
+
+/**
+ * Helper function to synchronously wait for a response and validate it.
+ */
+function waitAndValidateResponse(cb, request) {
+  let error = cb.wait();
+
+  if (!error) {
+    validateResponse(request.response);
+  }
+
+  return error;
+}
+
+/**
+ * Helper function to synchronously perform a GET request.
+ *
+ * @return Error instance or null if no error.
+ */
+function doGetRequest(request) {
+  let cb = Async.makeSpinningCallback();
+  request.get(cb);
+
+  return waitAndValidateResponse(cb, request);
+}
+
+/**
+ * Helper function to synchronously perform a PUT request.
+ *
+ * @return Error instance or null if no error.
+ */
+function doPutRequest(request, data) {
+  let cb = Async.makeSpinningCallback();
+  request.put(data, cb);
+
+  return waitAndValidateResponse(cb, request);
+}
+
+/**
+ * Helper function to synchronously perform a DELETE request.
+ *
+ * @return Error or null if no error was encountered.
+ */
+function doDeleteRequest(request) {
+  let cb = Async.makeSpinningCallback();
+  request.delete(cb);
+
+  return waitAndValidateResponse(cb, request);
+}
+
+function run_test() {
+  Log.repository.getLogger("Services.Common.Test.StorageServer").level =
+    Log.Level.Trace;
+  initTestLogging();
+
+  run_next_test();
+}
+
+add_test(function test_creation() {
+  _("Ensure a simple server can be created.");
+
+  // Explicit callback for this one.
+  let server = new StorageServer({
+    __proto__: StorageServerCallback,
+  });
+  do_check_true(!!server);
+
+  server.start(-1, function () {
+    _("Started on " + server.port);
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_synchronous_start() {
+  _("Ensure starting using startSynchronous works.");
+
+  let server = new StorageServer();
+  server.startSynchronous();
+  server.stop(run_next_test);
+});
+
+add_test(function test_url_parsing() {
+  _("Ensure server parses URLs properly.");
+
+  let server = new StorageServer();
+
+  // Check that we can parse a BSO URI.
+  let parts = server.pathRE.exec("/2.0/12345/storage/crypto/keys");
+  let [all, version, user, first, rest] = parts;
+  do_check_eq(all, "/2.0/12345/storage/crypto/keys");
+  do_check_eq(version, "2.0");
+  do_check_eq(user, "12345");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, "crypto/keys");
+  do_check_eq(null, server.pathRE.exec("/nothing/else"));
+
+  // Check that we can parse a collection URI.
+  parts = server.pathRE.exec("/2.0/123/storage/crypto");
+  [all, version, user, first, rest] = parts;
+  do_check_eq(all, "/2.0/123/storage/crypto");
+  do_check_eq(version, "2.0");
+  do_check_eq(user, "123");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, "crypto");
+
+  // We don't allow trailing slash on storage URI.
+  parts = server.pathRE.exec("/2.0/1234/storage/");
+  do_check_eq(parts, undefined);
+
+  // storage alone is a valid request.
+  parts = server.pathRE.exec("/2.0/123456/storage");
+  [all, version, user, first, rest] = parts;
+  do_check_eq(all, "/2.0/123456/storage");
+  do_check_eq(version, "2.0");
+  do_check_eq(user, "123456");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, undefined);
+
+  parts = server.storageRE.exec("storage");
+  let storage, collection, id;
+  [all, storage, collection, id] = parts;
+  do_check_eq(all, "storage");
+  do_check_eq(collection, undefined);
+
+  run_next_test();
+});
+
+add_test(function test_basic_http() {
+  let server = new StorageServer();
+  server.registerUser("345", "password");
+  do_check_true(server.userExists("345"));
+  server.startSynchronous();
+
+  _("Started on " + server.port);
+  do_check_eq(server.requestCount, 0);
+  let req = localRequest(server, "/2.0/storage/crypto/keys");
+  _("req is " + req);
+  req.get(function (err) {
+    do_check_eq(null, err);
+    do_check_eq(server.requestCount, 1);
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_info_collections() {
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let path = "/2.0/123/info/collections";
+
+  _("info/collections on empty server should be empty object.");
+  let request = localRequest(server, path, "123", "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 200);
+  do_check_eq(request.response.body, "{}");
+
+  _("Creating an empty collection should result in collection appearing.");
+  let coll = server.createCollection("123", "col1");
+  request = localRequest(server, path, "123", "password");
+  error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 200);
+  let info = JSON.parse(request.response.body);
+  do_check_attribute_count(info, 1);
+  do_check_true("col1" in info);
+  do_check_eq(info.col1, coll.timestamp);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_get_existing() {
+  _("Ensure that BSO retrieval works.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {"bso": {"foo": "bar"}}
+  });
+  server.startSynchronous();
+
+  let coll = server.user("123").collection("test");
+
+  let request = localRequest(server, "/2.0/123/storage/test/bso", "123",
+                             "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 200);
+  do_check_eq(request.response.headers["content-type"], "application/json");
+  let bso = JSON.parse(request.response.body);
+  do_check_attribute_count(bso, 3);
+  do_check_eq(bso.id, "bso");
+  do_check_eq(bso.modified, coll.bso("bso").modified);
+  let payload = JSON.parse(bso.payload);
+  do_check_attribute_count(payload, 1);
+  do_check_eq(payload.foo, "bar");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_percent_decoding() {
+  _("Ensure query string arguments with percent encoded are handled.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let coll = server.user("123").createCollection("test");
+  coll.insert("001", {foo: "bar"});
+  coll.insert("002", {bar: "foo"});
+
+  let request = localRequest(server, "/2.0/123/storage/test?ids=001%2C002",
+                             "123", "password");
+  let error = doGetRequest(request);
+  do_check_null(error);
+  do_check_eq(request.response.status, 200);
+  let items = JSON.parse(request.response.body).items;
+  do_check_attribute_count(items, 2);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_404() {
+  _("Ensure the server responds with a 404 if a BSO does not exist.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {}
+  });
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage/test/foo");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+
+  do_check_eq(request.response.status, 404);
+  do_check_false("content-type" in request.response.headers);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_if_modified_since_304() {
+  _("Ensure the server responds properly to X-If-Modified-Since for BSOs.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {bso: {foo: "bar"}}
+  });
+  server.startSynchronous();
+
+  let coll = server.user("123").collection("test");
+  do_check_neq(coll, null);
+
+  // Rewind clock just in case.
+  coll.timestamp -= 10000;
+  coll.bso("bso").modified -= 10000;
+
+  let request = localRequest(server, "/2.0/123/storage/test/bso",
+                             "123", "password");
+  request.setHeader("X-If-Modified-Since", "" + server.serverTime());
+  let error = doGetRequest(request);
+  do_check_eq(null, error);
+
+  do_check_eq(request.response.status, 304);
+  do_check_false("content-type" in request.response.headers);
+
+  request = localRequest(server, "/2.0/123/storage/test/bso",
+                             "123", "password");
+  request.setHeader("X-If-Modified-Since", "" + (server.serverTime() - 20000));
+  error = doGetRequest(request);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 200);
+  do_check_eq(request.response.headers["content-type"], "application/json");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_if_unmodified_since() {
+  _("Ensure X-If-Unmodified-Since works properly on BSOs.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {bso: {foo: "bar"}}
+  });
+  server.startSynchronous();
+
+  let coll = server.user("123").collection("test");
+  do_check_neq(coll, null);
+
+  let time = coll.bso("bso").modified;
+
+  _("Ensure we get a 412 for specified times older than server time.");
+  let request = localRequest(server, "/2.0/123/storage/test/bso",
+                             "123", "password");
+  request.setHeader("X-If-Unmodified-Since", time - 5000);
+  request.setHeader("Content-Type", "application/json");
+  let payload = JSON.stringify({"payload": "foobar"});
+  let error = doPutRequest(request, payload);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 412);
+
+  _("Ensure we get a 204 if update goes through.");
+  request = localRequest(server, "/2.0/123/storage/test/bso",
+                         "123", "password");
+  request.setHeader("Content-Type", "application/json");
+  request.setHeader("X-If-Unmodified-Since", time + 1);
+  error = doPutRequest(request, payload);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 204);
+  do_check_true(coll.timestamp > time);
+
+  // Not sure why a client would send X-If-Unmodified-Since if a BSO doesn't
+  // exist. But, why not test it?
+  _("Ensure we get a 201 if creation goes through.");
+  request = localRequest(server, "/2.0/123/storage/test/none",
+                         "123", "password");
+  request.setHeader("Content-Type", "application/json");
+  request.setHeader("X-If-Unmodified-Since", time);
+  error = doPutRequest(request, payload);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 201);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_delete_not_exist() {
+  _("Ensure server behaves properly when deleting a BSO that does not exist.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.user("123").createCollection("empty");
+  server.startSynchronous();
+
+  server.callback.onItemDeleted = function onItemDeleted(username, collection,
+                                                         id) {
+    do_throw("onItemDeleted should not have been called.");
+  };
+
+  let request = localRequest(server, "/2.0/123/storage/empty/nada",
+                             "123", "password");
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 404);
+  do_check_false("content-type" in request.response.headers);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_delete_exists() {
+  _("Ensure proper semantics when deleting a BSO that exists.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let coll = server.user("123").createCollection("test");
+  let bso = coll.insert("myid", {foo: "bar"});
+  let timestamp = coll.timestamp;
+
+  server.callback.onItemDeleted = function onDeleted(username, collection, id) {
+    delete server.callback.onItemDeleted;
+    do_check_eq(username, "123");
+    do_check_eq(collection, "test");
+    do_check_eq(id, "myid");
+  };
+
+  let request = localRequest(server, "/2.0/123/storage/test/myid",
+                             "123", "password");
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 204);
+  do_check_eq(coll.bsos().length, 0);
+  do_check_true(coll.timestamp > timestamp);
+
+  _("On next request the BSO should not exist.");
+  request = localRequest(server, "/2.0/123/storage/test/myid",
+                         "123", "password");
+  error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 404);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_delete_unmodified() {
+  _("Ensure X-If-Unmodified-Since works when deleting BSOs.");
+
+  let server = new StorageServer();
+  server.startSynchronous();
+  server.registerUser("123", "password");
+  let coll = server.user("123").createCollection("test");
+  let bso = coll.insert("myid", {foo: "bar"});
+
+  let modified = bso.modified;
+
+  _("Issuing a DELETE with an older time should fail.");
+  let path = "/2.0/123/storage/test/myid";
+  let request = localRequest(server, path, "123", "password");
+  request.setHeader("X-If-Unmodified-Since", modified - 1000);
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 412);
+  do_check_false("content-type" in request.response.headers);
+  do_check_neq(coll.bso("myid"), null);
+
+  _("Issuing a DELETE with a newer time should work.");
+  request = localRequest(server, path, "123", "password");
+  request.setHeader("X-If-Unmodified-Since", modified + 1000);
+  error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 204);
+  do_check_true(coll.bso("myid").deleted);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_collection_get_unmodified_since() {
+  _("Ensure conditional unmodified get on collection works when it should.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+  let collection = server.user("123").createCollection("testcoll");
+  collection.insert("bso0", {foo: "bar"});
+
+  let serverModified = collection.timestamp;
+
+  let request1 = localRequest(server, "/2.0/123/storage/testcoll",
+                              "123", "password");
+  request1.setHeader("X-If-Unmodified-Since", serverModified);
+  let error = doGetRequest(request1);
+  do_check_null(error);
+  do_check_eq(request1.response.status, 200);
+
+  let request2 = localRequest(server, "/2.0/123/storage/testcoll",
+                              "123", "password");
+  request2.setHeader("X-If-Unmodified-Since", serverModified - 1);
+  error = doGetRequest(request2);
+  do_check_null(error);
+  do_check_eq(request2.response.status, 412);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_get_unmodified_since() {
+  _("Ensure conditional unmodified get on BSO works appropriately.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+  let collection = server.user("123").createCollection("testcoll");
+  let bso = collection.insert("bso0", {foo: "bar"});
+
+  let serverModified = bso.modified;
+
+  let request1 = localRequest(server, "/2.0/123/storage/testcoll/bso0",
+                              "123", "password");
+  request1.setHeader("X-If-Unmodified-Since", serverModified);
+  let error = doGetRequest(request1);
+  do_check_null(error);
+  do_check_eq(request1.response.status, 200);
+
+  let request2 = localRequest(server, "/2.0/123/storage/testcoll/bso0",
+                              "123", "password");
+  request2.setHeader("X-If-Unmodified-Since", serverModified - 1);
+  error = doGetRequest(request2);
+  do_check_null(error);
+  do_check_eq(request2.response.status, 412);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_missing_collection_404() {
+  _("Ensure a missing collection returns a 404.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage/none", "123", "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 404);
+  do_check_false("content-type" in request.response.headers);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_get_storage_405() {
+  _("Ensure that a GET on /storage results in a 405.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage", "123", "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 405);
+  do_check_eq(request.response.headers["allow"], "DELETE");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_delete_storage() {
+  _("Ensure that deleting all of storage works.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    foo: {a: {foo: "bar"}, b: {bar: "foo"}},
+    baz: {c: {bob: "law"}, blah: {law: "blog"}}
+  });
+
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage", "123", "password");
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 204);
+  do_check_attribute_count(server.users["123"].collections, 0);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_x_num_records() {
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+
+  server.createContents("123", {
+    crypto: {foos: {foo: "bar"},
+             bars: {foo: "baz"}}
+  });
+  server.startSynchronous();
+  let bso = localRequest(server, "/2.0/123/storage/crypto/foos");
+  bso.get(function (err) {
+    // BSO fetches don't have one.
+    do_check_false("x-num-records" in this.response.headers);
+    let col = localRequest(server, "/2.0/123/storage/crypto");
+    col.get(function (err) {
+      // Collection fetches do.
+      do_check_eq(this.response.headers["x-num-records"], "2");
+      server.stop(run_next_test);
+    });
+  });
+});
+
+add_test(function test_put_delete_put() {
+  _("Bug 790397: Ensure BSO deleted flag is reset on PUT.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {bso: {foo: "bar"}}
+  });
+  server.startSynchronous();
+
+  _("Ensure we can PUT an existing record.");
+  let request1 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  request1.setHeader("Content-Type", "application/json");
+  let payload1 = JSON.stringify({"payload": "foobar"});
+  let error1 = doPutRequest(request1, payload1);
+  do_check_eq(null, error1);
+  do_check_eq(request1.response.status, 204);
+
+  _("Ensure we can DELETE it.");
+  let request2 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  let error2 = doDeleteRequest(request2);
+  do_check_eq(error2, null);
+  do_check_eq(request2.response.status, 204);
+  do_check_false("content-type" in request2.response.headers);
+
+  _("Ensure we can PUT a previously deleted record.");
+  let request3 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  request3.setHeader("Content-Type", "application/json");
+  let payload3 = JSON.stringify({"payload": "foobar"});
+  let error3 = doPutRequest(request3, payload3);
+  do_check_eq(null, error3);
+  do_check_eq(request3.response.status, 201);
+
+  _("Ensure we can GET the re-uploaded record.");
+  let request4 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  let error4 = doGetRequest(request4);
+  do_check_eq(error4, null);
+  do_check_eq(request4.response.status, 200);
+  do_check_eq(request4.response.headers["content-type"], "application/json");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_collection_get_newer() {
+  _("Ensure get with newer argument on collection works.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let coll = server.user("123").createCollection("test");
+  let bso1 = coll.insert("001", {foo: "bar"});
+  let bso2 = coll.insert("002", {bar: "foo"});
+
+  // Don't want both records to have the same timestamp.
+  bso2.modified = bso1.modified + 1000;
+
+  function newerRequest(newer) {
+    return localRequest(server, "/2.0/123/storage/test?newer=" + newer,
+                        "123", "password");
+  }
+
+  let request1 = newerRequest(0);
+  let error1 = doGetRequest(request1);
+  do_check_null(error1);
+  do_check_eq(request1.response.status, 200);
+  let items1 = JSON.parse(request1.response.body).items;
+  do_check_attribute_count(items1, 2);
+
+  let request2 = newerRequest(bso1.modified + 1);
+  let error2 = doGetRequest(request2);
+  do_check_null(error2);
+  do_check_eq(request2.response.status, 200);
+  let items2 = JSON.parse(request2.response.body).items;
+  do_check_attribute_count(items2, 1);
+
+  let request3 = newerRequest(bso2.modified + 1);
+  let error3 = doGetRequest(request3);
+  do_check_null(error3);
+  do_check_eq(request3.response.status, 200);
+  let items3 = JSON.parse(request3.response.body).items;
+  do_check_attribute_count(items3, 0);
+
+  server.stop(run_next_test);
+});

services/common/tests/unit/test_tokenauthenticatedrequest.js

@@ -0,0 +1,52 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+
+add_test(function test_authenticated_request() {
+  _("Ensure that sending a MAC authenticated GET request works as expected.");
+
+  let message = "Great Success!";
+
+  // TODO: We use a preset key here, but use getTokenFromBrowserIDAssertion()
+  // from TokenServerClient to get a real one when possible. (Bug 745800)
+  let id = "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x";
+  let key = "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=";
+  let method = "GET";
+
+  let nonce = btoa(CryptoUtils.generateRandomBytes(16));
+  let ts = Math.floor(Date.now() / 1000);
+  let extra = {ts: ts, nonce: nonce};
+
+  let auth;
+
+  let server = httpd_setup({"/foo": function(request, response) {
+      do_check_true(request.hasHeader("Authorization"));
+      do_check_eq(auth, request.getHeader("Authorization"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+  let uri = CommonUtils.makeURI(server.baseURI + "/foo");
+  let sig = CryptoUtils.computeHTTPMACSHA1(id, key, method, uri, extra);
+  auth = sig.getHeader();
+
+  let req = new TokenAuthenticatedRESTRequest(uri, {id: id, key: key}, extra);
+  let cb = Async.makeSpinningCallback();
+  req.get(cb);
+  let result = cb.wait();
+
+  do_check_eq(null, result);
+  do_check_eq(message, req.response.body);
+
+  server.stop(run_next_test);
+});

services/common/tests/unit/test_tokenserverclient.js

@@ -0,0 +1,466 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/tokenserverclient.js");
+
+function run_test() {
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+add_test(function test_working_bid_exchange() {
+  _("Ensure that working BrowserID token exchange works as expected.");
+
+  let service = "http://example.com/foo";
+  let duration = 300;
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      do_check_true(request.hasHeader("accept"));
+      do_check_false(request.hasHeader("x-conditions-accepted"));
+      do_check_eq("application/json", request.getHeader("accept"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: service,
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let cb = Async.makeSpinningCallback();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", cb);
+  let result = cb.wait();
+  do_check_eq("object", typeof(result));
+  do_check_attribute_count(result, 6);
+  do_check_eq(service, result.endpoint);
+  do_check_eq("id", result.id);
+  do_check_eq("key", result.key);
+  do_check_eq("uid", result.uid);
+  do_check_eq(duration, result.duration);
+  server.stop(run_next_test);
+});
+
+add_test(function test_invalid_arguments() {
+  _("Ensure invalid arguments to APIs are rejected.");
+
+  let args = [
+    [null, "assertion", function() {}],
+    ["http://example.com/", null, function() {}],
+    ["http://example.com/", "assertion", null]
+  ];
+
+  for (let arg of args) {
+    try {
+      let client = new TokenServerClient();
+      client.getTokenFromBrowserIDAssertion(arg[0], arg[1], arg[2]);
+      do_throw("Should never get here.");
+    } catch (ex) {
+      do_check_true(ex instanceof TokenServerClientError);
+    }
+  }
+
+  run_next_test();
+});
+
+add_test(function test_conditions_required_response_handling() {
+  _("Ensure that a conditions required response is handled properly.");
+
+  let description = "Need to accept conditions";
+  let tosURL = "http://example.com/tos";
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      do_check_false(request.hasHeader("x-conditions-accepted"));
+
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        errors: [{description: description, location: "body", name: ""}],
+        urls: {tos: tosURL}
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "conditions-required");
+    // Check a JSON.stringify works on our errors as our logging will try and use it.
+    do_check_true(JSON.stringify(error), "JSON.stringify worked");
+    do_check_null(token);
+
+    do_check_eq(error.urls.tos, tosURL);
+
+    server.stop(run_next_test);
+  }
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_invalid_403_no_content_type() {
+  _("Ensure that a 403 without content-type is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      // No Content-Type header by design.
+
+      let body = JSON.stringify({
+        errors: [{description: "irrelevant", location: "body", name: ""}],
+        urls: {foo: "http://bar"}
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_null(token);
+
+    do_check_null(error.urls);
+
+    server.stop(run_next_test);
+  }
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_invalid_403_bad_json() {
+  _("Ensure that a 403 with JSON that isn't proper is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = JSON.stringify({
+        foo: "bar"
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_null(token);
+    do_check_null(error.urls);
+
+    server.stop(run_next_test);
+  }
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_403_no_urls() {
+  _("Ensure that a 403 without a urls field is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = "{}";
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion",
+                                        function onResponse(error, result) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_null(result);
+
+    server.stop(run_next_test);
+
+  });
+});
+
+add_test(function test_send_extra_headers() {
+  _("Ensures that the condition acceptance header is sent when asked.");
+
+  let duration = 300;
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      do_check_true(request.hasHeader("x-foo"));
+      do_check_eq(request.getHeader("x-foo"), "42");
+
+      do_check_true(request.hasHeader("x-bar"));
+      do_check_eq(request.getHeader("x-bar"), "17");
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: "http://example.com/",
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_null(error);
+
+    // Other tests validate other things.
+
+    server.stop(run_next_test);
+  }
+
+  let extra = {
+    "X-Foo": 42,
+    "X-Bar": 17
+  };
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse, extra);
+});
+
+add_test(function test_error_404_empty() {
+  _("Ensure that 404 responses without proper response are handled properly.");
+
+  let server = httpd_setup();
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/foo";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+
+    do_check_neq(null, error.response);
+    do_check_null(r);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_error_404_proper_response() {
+  _("Ensure that a Cornice error report for 404 is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 404, "Not Found");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = JSON.stringify({
+        status: 404,
+        errors: [{description: "No service", location: "body", name: ""}],
+      });
+
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "unknown-service");
+    do_check_null(token);
+
+    server.stop(run_next_test);
+  }
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_bad_json() {
+  _("Ensure that malformed JSON is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = '{"id": "id", baz}'
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_neq(null, error.response);
+    do_check_eq(null, r);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_400_response() {
+  _("Ensure HTTP 400 is converted to malformed-request.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 400, "Bad Request");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = "{}"; // Actual content may not be used.
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_neq(null, error.response);
+    do_check_eq(error.cause, "malformed-request");
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_401_with_error_cause() {
+  _("Ensure 401 cause is specified in body.status");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = JSON.stringify({status: "no-soup-for-you"});
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_neq(null, error.response);
+    do_check_eq(error.cause, "no-soup-for-you");
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_unhandled_media_type() {
+  _("Ensure that unhandled media types throw an error.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "text/plain");
+
+      let body = "hello, world";
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let url = server.baseURI + "/1.0/foo/1.0";
+  let client = new TokenServerClient();
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_neq(null, error.response);
+    do_check_eq(null, r);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_rich_media_types() {
+  _("Ensure that extra tokens in the media type aren't rejected.");
+
+  let duration = 300;
+  let server = httpd_setup({
+    "/foo": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json; foo=bar; bar=foo");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: "foo",
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let url = server.baseURI + "/foo";
+  let client = new TokenServerClient();
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_eq(null, error);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_exception_during_callback() {
+  _("Ensure that exceptions thrown during callback handling are handled.");
+
+  let duration = 300;
+  let server = httpd_setup({
+    "/foo": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: "foo",
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let url = server.baseURI + "/foo";
+  let client = new TokenServerClient();
+  let cb = Async.makeSpinningCallback();
+  let callbackCount = 0;
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_eq(null, error);
+
+    cb();
+
+    callbackCount += 1;
+    throw new Error("I am a bad function!");
+  });
+
+  cb.wait();
+  // This relies on some heavy event loop magic. The error in the main
+  // callback should already have been raised at this point.
+  do_check_eq(callbackCount, 1);
+
+  server.stop(run_next_test);
+});

services/common/tests/unit/test_utils_atob.js

@@ -0,0 +1,11 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  let data = ["Zm9vYmE=", "Zm9vYmE==", "Zm9vYmE==="];
+  for (let d in data) {
+    do_check_eq(CommonUtils.safeAtoB(data[d]), "fooba");
+  }
+}

services/common/tests/unit/test_utils_convert_string.js

@@ -0,0 +1,132 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-common/utils.js");
+
+// A wise line of Greek verse, and the utf-8 byte encoding.
+// N.b., Greek begins at utf-8 ce 91
+const TEST_STR = "πόλλ' οἶδ' ἀλώπηξ, ἀλλ' ἐχῖνος ἓν μέγα";
+const TEST_HEX = h("cf 80 cf 8c ce bb ce bb   27 20 ce bf e1 bc b6 ce"+
+                   "b4 27 20 e1 bc 80 ce bb   cf 8e cf 80 ce b7 ce be"+
+                   "2c 20 e1 bc 80 ce bb ce   bb 27 20 e1 bc 90 cf 87"+
+                   "e1 bf 96 ce bd ce bf cf   82 20 e1 bc 93 ce bd 20"+
+                   "ce bc ce ad ce b3 ce b1");
+// Integer byte values for the above
+const TEST_BYTES = [207,128,207,140,206,187,206,187,
+                     39, 32,206,191,225,188,182,206,
+                    180, 39, 32,225,188,128,206,187,
+                    207,142,207,128,206,183,206,190,
+                     44, 32,225,188,128,206,187,206,
+                    187, 39, 32,225,188,144,207,135,
+                    225,191,150,206,189,206,191,207,
+                    130, 32,225,188,147,206,189, 32,
+                    206,188,206,173,206,179,206,177];
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_compress_string() {
+  const INPUT = "hello";
+
+  let result = CommonUtils.convertString(INPUT, "uncompressed", "deflate");
+  do_check_eq(result.length, 13);
+
+  let result2 = CommonUtils.convertString(INPUT, "uncompressed", "deflate");
+  do_check_eq(result, result2);
+
+  let result3 = CommonUtils.convertString(result, "deflate", "uncompressed");
+  do_check_eq(result3, INPUT);
+
+  run_next_test();
+});
+
+add_test(function test_compress_utf8() {
+  const INPUT = "Árvíztűrő tükörfúrógép いろはにほへとちりぬるを Pijamalı hasta, yağız şoföre çabucak güvendi.";
+  let inputUTF8 = CommonUtils.encodeUTF8(INPUT);
+
+  let compressed = CommonUtils.convertString(inputUTF8, "uncompressed", "deflate");
+  let uncompressed = CommonUtils.convertString(compressed, "deflate", "uncompressed");
+
+  do_check_eq(uncompressed, inputUTF8);
+
+  let outputUTF8 = CommonUtils.decodeUTF8(uncompressed);
+  do_check_eq(outputUTF8, INPUT);
+
+  run_next_test();
+});
+
+add_test(function test_bad_argument() {
+  let failed = false;
+  try {
+    CommonUtils.convertString(null, "uncompressed", "deflate");
+  } catch (ex) {
+    failed = true;
+    do_check_true(ex.message.startsWith("Input string must be defined"));
+  } finally {
+    do_check_true(failed);
+  }
+
+  run_next_test();
+});
+
+add_task(function test_stringAsHex() {
+  do_check_eq(TEST_HEX, CommonUtils.stringAsHex(TEST_STR));
+});
+
+add_task(function test_hexAsString() {
+  do_check_eq(TEST_STR, CommonUtils.hexAsString(TEST_HEX));
+});
+
+add_task(function test_hexToBytes() {
+  let bytes = CommonUtils.hexToBytes(TEST_HEX);
+  do_check_eq(TEST_BYTES.length, bytes.length);
+  // Ensure that the decimal values of each byte are correct
+  do_check_true(arraysEqual(TEST_BYTES,
+      CommonUtils.stringToByteArray(bytes)));
+});
+
+add_task(function test_bytesToHex() {
+  // Create a list of our character bytes from the reference int values
+  let bytes = CommonUtils.byteArrayToString(TEST_BYTES);
+  do_check_eq(TEST_HEX, CommonUtils.bytesAsHex(bytes));
+});
+
+add_task(function test_stringToBytes() {
+  do_check_true(arraysEqual(TEST_BYTES,
+      CommonUtils.stringToByteArray(CommonUtils.stringToBytes(TEST_STR))));
+});
+
+add_task(function test_stringRoundTrip() {
+  do_check_eq(TEST_STR,
+    CommonUtils.hexAsString(CommonUtils.stringAsHex(TEST_STR)));
+});
+
+add_task(function test_hexRoundTrip() {
+  do_check_eq(TEST_HEX,
+    CommonUtils.stringAsHex(CommonUtils.hexAsString(TEST_HEX)));
+});
+
+add_task(function test_byteArrayRoundTrip() {
+  do_check_true(arraysEqual(TEST_BYTES,
+    CommonUtils.stringToByteArray(CommonUtils.byteArrayToString(TEST_BYTES))));
+});
+
+// turn formatted test vectors into normal hex strings
+function h(hexStr) {
+  return hexStr.replace(/\s+/g, "");
+}
+
+function arraysEqual(a1, a2) {
+  if (a1.length !== a2.length) {
+    return false;
+  }
+  for (let i = 0; i < a1.length; i++) {
+    if (a1[i] !== a2[i]) {
+      return false;
+    }
+  }
+  return true;
+}

services/common/tests/unit/test_utils_dateprefs.js

@@ -0,0 +1,85 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://services-common/utils.js");
+
+
+var prefs = new Preferences("servicescommon.tests.");
+
+function DummyLogger() {
+  this.messages = [];
+}
+DummyLogger.prototype.warn = function warn(message) {
+  this.messages.push(message);
+};
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_set_basic() {
+  let now = new Date();
+
+  CommonUtils.setDatePref(prefs, "test00", now);
+  let value = prefs.get("test00");
+  do_check_eq(value, "" + now.getTime());
+
+  let now2 = CommonUtils.getDatePref(prefs, "test00");
+
+  do_check_eq(now.getTime(), now2.getTime());
+
+  run_next_test();
+});
+
+add_test(function test_set_bounds_checking() {
+  let d = new Date(2342354);
+
+  let failed = false;
+  try {
+    CommonUtils.setDatePref(prefs, "test01", d);
+  } catch (ex) {
+    do_check_true(ex.message.startsWith("Trying to set"));
+    failed = true;
+  }
+
+  do_check_true(failed);
+  run_next_test();
+});
+
+add_test(function test_get_bounds_checking() {
+  prefs.set("test_bounds_checking", "13241431");
+
+  let log = new DummyLogger();
+  let d = CommonUtils.getDatePref(prefs, "test_bounds_checking", 0, log);
+  do_check_eq(d.getTime(), 0);
+  do_check_eq(log.messages.length, 1);
+
+  run_next_test();
+});
+
+add_test(function test_get_bad_default() {
+  let failed = false;
+  try {
+    CommonUtils.getDatePref(prefs, "get_bad_default", new Date());
+  } catch (ex) {
+    do_check_true(ex.message.startsWith("Default value is not a number"));
+    failed = true;
+  }
+
+  do_check_true(failed);
+  run_next_test();
+});
+
+add_test(function test_get_invalid_number() {
+  prefs.set("get_invalid_number", "hello world");
+
+  let log = new DummyLogger();
+  let d = CommonUtils.getDatePref(prefs, "get_invalid_number", 42, log);
+  do_check_eq(d.getTime(), 42);
+  do_check_eq(log.messages.length, 1);
+
+  run_next_test();
+});

services/common/tests/unit/test_utils_deepCopy.js

@@ -0,0 +1,18 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://testing-common/services/common/utils.js");
+
+function run_test() {
+  let thing = {o: {foo: "foo", bar: ["bar"]}, a: ["foo", {bar: "bar"}]};
+  let ret = TestingUtils.deepCopy(thing);
+  do_check_neq(ret, thing)
+  do_check_neq(ret.o, thing.o);
+  do_check_neq(ret.o.bar, thing.o.bar);
+  do_check_neq(ret.a, thing.a);
+  do_check_neq(ret.a[1], thing.a[1]);
+  do_check_eq(ret.o.foo, thing.o.foo);
+  do_check_eq(ret.o.bar[0], thing.o.bar[0]);
+  do_check_eq(ret.a[0], thing.a[0]);
+  do_check_eq(ret.a[1].bar, thing.a[1].bar);
+}

services/common/tests/unit/test_utils_encodeBase32.js

@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  // Testing byte array manipulation.
+  do_check_eq("FOOBAR", CommonUtils.byteArrayToString([70, 79, 79, 66, 65, 82]));
+  do_check_eq("", CommonUtils.byteArrayToString([]));
+
+  _("Testing encoding...");
+  // Test vectors from RFC 4648
+  do_check_eq(CommonUtils.encodeBase32(""), "");
+  do_check_eq(CommonUtils.encodeBase32("f"), "MY======");
+  do_check_eq(CommonUtils.encodeBase32("fo"), "MZXQ====");
+  do_check_eq(CommonUtils.encodeBase32("foo"), "MZXW6===");
+  do_check_eq(CommonUtils.encodeBase32("foob"), "MZXW6YQ=");
+  do_check_eq(CommonUtils.encodeBase32("fooba"), "MZXW6YTB");
+  do_check_eq(CommonUtils.encodeBase32("foobar"), "MZXW6YTBOI======");
+
+  do_check_eq(CommonUtils.encodeBase32("Bacon is a vegetable."),
+              "IJQWG33OEBUXGIDBEB3GKZ3FORQWE3DFFY======");
+
+  _("Checking assumptions...");
+  for (let i = 0; i <= 255; ++i)
+    do_check_eq(undefined | i, i);
+
+  _("Testing decoding...");
+  do_check_eq(CommonUtils.decodeBase32(""), "");
+  do_check_eq(CommonUtils.decodeBase32("MY======"), "f");
+  do_check_eq(CommonUtils.decodeBase32("MZXQ===="), "fo");
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTB"), "fooba");
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTBOI======"), "foobar");
+
+  // Same with incorrect or missing padding.
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTBOI=="), "foobar");
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTBOI"), "foobar");
+
+  let encoded = CommonUtils.encodeBase32("Bacon is a vegetable.");
+  _("Encoded to " + JSON.stringify(encoded));
+  do_check_eq(CommonUtils.decodeBase32(encoded), "Bacon is a vegetable.");
+
+  // Test failure.
+  let err;
+  try {
+    CommonUtils.decodeBase32("000");
+  } catch (ex) {
+    err = ex;
+  }
+  do_check_eq(err, "Unknown character in base32: 0");
+}

services/common/tests/unit/test_utils_encodeBase64URL.js

@@ -0,0 +1,27 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_simple() {
+  let expected = {
+    hello: "aGVsbG8=",
+    "<>?": "PD4_",
+  };
+
+  for (let [k,v] of Object.entries(expected)) {
+    do_check_eq(CommonUtils.encodeBase64URL(k), v);
+  }
+
+  run_next_test();
+});
+
+add_test(function test_no_padding() {
+  do_check_eq(CommonUtils.encodeBase64URL("hello", false), "aGVsbG8");
+
+  run_next_test();
+});

services/common/tests/unit/test_utils_ensureMillisecondsTimestamp.js

@@ -0,0 +1,23 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp(null));
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp(0));
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp("0"));
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp("000"));
+
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp(999 * 10000000000));
+
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(-1); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(1); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(1.5); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(999 * 10000000000 + 0.5); });
+
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("-1"); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("1"); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("1.5"); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("" + (999 * 10000000000 + 0.5)); });
+}

services/common/tests/unit/test_utils_json.js

@@ -0,0 +1,40 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://gre/modules/osfile.jsm");
+
+function run_test() {
+  initTestLogging();
+  run_next_test();
+}
+
+add_test(function test_writeJSON_readJSON() {
+  _("Round-trip some JSON through the promise-based JSON writer.");
+
+  let contents = {
+    "a": 12345.67,
+    "b": {
+      "c": "héllö",
+    },
+    "d": undefined,
+    "e": null,
+  };
+
+  function checkJSON(json) {
+    do_check_eq(contents.a, json.a);
+    do_check_eq(contents.b.c, json.b.c);
+    do_check_eq(contents.d, json.d);
+    do_check_eq(contents.e, json.e);
+    run_next_test();
+  };
+
+  function doRead() {
+    CommonUtils.readJSON(path)
+               .then(checkJSON, do_throw);
+  }
+
+  let path = OS.Path.join(OS.Constants.Path.profileDir, "bar.json");
+  CommonUtils.writeJSON(contents, path)
+             .then(doRead, do_throw);
+});

services/common/tests/unit/test_utils_makeURI.js

@@ -0,0 +1,66 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_("Make sure uri strings are converted to nsIURIs");
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  _test_makeURI();
+}
+
+function _test_makeURI() {
+  _("Check http uris");
+  let uri1 = "http://mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uri1).spec, uri1);
+  let uri2 = "http://www.mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uri2).spec, uri2);
+  let uri3 = "http://mozillalabs.com/path";
+  do_check_eq(CommonUtils.makeURI(uri3).spec, uri3);
+  let uri4 = "http://mozillalabs.com/multi/path";
+  do_check_eq(CommonUtils.makeURI(uri4).spec, uri4);
+  let uri5 = "http://mozillalabs.com/?query";
+  do_check_eq(CommonUtils.makeURI(uri5).spec, uri5);
+  let uri6 = "http://mozillalabs.com/#hash";
+  do_check_eq(CommonUtils.makeURI(uri6).spec, uri6);
+
+  _("Check https uris");
+  let uris1 = "https://mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uris1).spec, uris1);
+  let uris2 = "https://www.mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uris2).spec, uris2);
+  let uris3 = "https://mozillalabs.com/path";
+  do_check_eq(CommonUtils.makeURI(uris3).spec, uris3);
+  let uris4 = "https://mozillalabs.com/multi/path";
+  do_check_eq(CommonUtils.makeURI(uris4).spec, uris4);
+  let uris5 = "https://mozillalabs.com/?query";
+  do_check_eq(CommonUtils.makeURI(uris5).spec, uris5);
+  let uris6 = "https://mozillalabs.com/#hash";
+  do_check_eq(CommonUtils.makeURI(uris6).spec, uris6);
+
+  _("Check chrome uris");
+  let uric1 = "chrome://browser/content/browser.xul";
+  do_check_eq(CommonUtils.makeURI(uric1).spec, uric1);
+  let uric2 = "chrome://browser/skin/browser.css";
+  do_check_eq(CommonUtils.makeURI(uric2).spec, uric2);
+  let uric3 = "chrome://browser/locale/browser.dtd";
+  do_check_eq(CommonUtils.makeURI(uric3).spec, uric3);
+
+  _("Check about uris");
+  let uria1 = "about:weave";
+  do_check_eq(CommonUtils.makeURI(uria1).spec, uria1);
+  let uria2 = "about:weave/";
+  do_check_eq(CommonUtils.makeURI(uria2).spec, uria2);
+  let uria3 = "about:weave/path";
+  do_check_eq(CommonUtils.makeURI(uria3).spec, uria3);
+  let uria4 = "about:weave/multi/path";
+  do_check_eq(CommonUtils.makeURI(uria4).spec, uria4);
+  let uria5 = "about:weave/?query";
+  do_check_eq(CommonUtils.makeURI(uria5).spec, uria5);
+  let uria6 = "about:weave/#hash";
+  do_check_eq(CommonUtils.makeURI(uria6).spec, uria6);
+
+  _("Invalid uris are undefined");
+  do_check_eq(CommonUtils.makeURI("mozillalabs.com"), undefined);
+  do_check_eq(CommonUtils.makeURI("chrome://badstuff"), undefined);
+  do_check_eq(CommonUtils.makeURI("this is a test"), undefined);
+}

services/common/tests/unit/test_utils_namedTimer.js

@@ -0,0 +1,69 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_required_args() {
+  try {
+    CommonUtils.namedTimer(function callback() {
+      do_throw("Shouldn't fire.");
+    }, 0);
+    do_throw("Should have thrown!");
+  } catch(ex) {
+    run_next_test();
+  }
+});
+
+add_test(function test_simple() {
+  _("Test basic properties of CommonUtils.namedTimer.");
+
+  const delay = 200;
+  let that = {};
+  let t0 = Date.now();
+  CommonUtils.namedTimer(function callback(timer) {
+    do_check_eq(this, that);
+    do_check_eq(this._zetimer, null);
+    do_check_true(timer instanceof Ci.nsITimer);
+    // Difference should be ~delay, but hard to predict on all platforms,
+    // particularly Windows XP.
+    do_check_true(Date.now() > t0);
+    run_next_test();
+  }, delay, that, "_zetimer");
+});
+
+add_test(function test_delay() {
+  _("Test delaying a timer that hasn't fired yet.");
+
+  const delay = 100;
+  let that = {};
+  let t0 = Date.now();
+  function callback(timer) {
+    // Difference should be ~2*delay, but hard to predict on all platforms,
+    // particularly Windows XP.
+    do_check_true((Date.now() - t0) > delay);
+    run_next_test();
+  }
+  CommonUtils.namedTimer(callback, delay, that, "_zetimer");
+  CommonUtils.namedTimer(callback, 2 * delay, that, "_zetimer");
+  run_next_test();
+});
+
+add_test(function test_clear() {
+  _("Test clearing a timer that hasn't fired yet.");
+
+  const delay = 0;
+  let that = {};
+  CommonUtils.namedTimer(function callback(timer) {
+    do_throw("Shouldn't fire!");
+  }, delay, that, "_zetimer");
+
+  that._zetimer.clear();
+  do_check_eq(that._zetimer, null);
+  CommonUtils.nextTick(run_next_test);
+
+  run_next_test();
+});

services/common/tests/unit/test_utils_sets.js

@@ -0,0 +1,72 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-common/utils.js");
+
+const EMPTY = new Set();
+const A = new Set(["a"]);
+const ABC = new Set(["a", "b", "c"]);
+const ABCD = new Set(["a", "b", "c", "d"]);
+const BC = new Set(["b", "c"]);
+const BCD = new Set(["b", "c", "d"]);
+const FGH = new Set(["f", "g", "h"]);
+const BCDFGH = new Set(["b", "c", "d", "f", "g", "h"]);
+
+var union = CommonUtils.union;
+var difference = CommonUtils.difference;
+var intersection = CommonUtils.intersection;
+var setEqual = CommonUtils.setEqual;
+
+function do_check_setEqual(a, b) {
+  do_check_true(setEqual(a, b));
+}
+
+function do_check_not_setEqual(a, b) {
+  do_check_false(setEqual(a, b));
+}
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_setEqual() {
+  do_check_setEqual(EMPTY, EMPTY);
+  do_check_setEqual(EMPTY, new Set());
+  do_check_setEqual(A, A);
+  do_check_setEqual(A, new Set(["a"]));
+  do_check_setEqual(new Set(["a"]), A);
+  do_check_not_setEqual(A, EMPTY);
+  do_check_not_setEqual(EMPTY, A);
+  do_check_not_setEqual(ABC, A);
+  run_next_test();
+});
+
+add_test(function test_union() {
+  do_check_setEqual(EMPTY, union(EMPTY, EMPTY));
+  do_check_setEqual(ABC, union(EMPTY, ABC));
+  do_check_setEqual(ABC, union(ABC, ABC));
+  do_check_setEqual(ABCD, union(ABC, BCD));
+  do_check_setEqual(ABCD, union(BCD, ABC));
+  do_check_setEqual(BCDFGH, union(BCD, FGH));
+  run_next_test();
+});
+
+add_test(function test_difference() {
+  do_check_setEqual(EMPTY, difference(EMPTY, EMPTY));
+  do_check_setEqual(EMPTY, difference(EMPTY, A));
+  do_check_setEqual(EMPTY, difference(A, A));
+  do_check_setEqual(ABC, difference(ABC, EMPTY));
+  do_check_setEqual(ABC, difference(ABC, FGH));
+  do_check_setEqual(A, difference(ABC, BCD));
+  run_next_test();
+});
+
+add_test(function test_intersection() {
+  do_check_setEqual(EMPTY, intersection(EMPTY, EMPTY));
+  do_check_setEqual(EMPTY, intersection(ABC, EMPTY));
+  do_check_setEqual(EMPTY, intersection(ABC, FGH));
+  do_check_setEqual(BC, intersection(ABC, BCD));
+  run_next_test();
+});

services/common/tests/unit/test_utils_utf8.js

@@ -0,0 +1,11 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  let str = "Umlaute: \u00FC \u00E4\n"; // Umlaute: ü ä
+  let encoded = CommonUtils.encodeUTF8(str);
+  let decoded = CommonUtils.decodeUTF8(encoded);
+  do_check_eq(decoded, str);
+}

services/common/tests/unit/test_utils_uuid.js

@@ -0,0 +1,12 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+function run_test() {
+  let uuid = CommonUtils.generateUUID();
+  do_check_eq(uuid.length, 36);
+  do_check_eq(uuid[8], "-");
+
+  run_next_test();
+}

services/common/tests/unit/xpcshell.ini

@@ -0,0 +1,53 @@
+[DEFAULT]
+head = head_global.js head_helpers.js head_http.js
+tail =
+firefox-appdir = browser
+support-files =
+  test_storage_adapter/**
+  test_blocklist_signatures/**
+
+# Test load modules first so syntax failures are caught early.
+[test_load_modules.js]
+
+[test_blocklist_certificates.js]
+[test_blocklist_clients.js]
+[test_blocklist_updater.js]
+
+[test_kinto.js]
+[test_blocklist_signatures.js]
+[test_storage_adapter.js]
+
+[test_utils_atob.js]
+[test_utils_convert_string.js]
+[test_utils_dateprefs.js]
+[test_utils_deepCopy.js]
+[test_utils_encodeBase32.js]
+[test_utils_encodeBase64URL.js]
+[test_utils_ensureMillisecondsTimestamp.js]
+[test_utils_json.js]
+[test_utils_makeURI.js]
+[test_utils_namedTimer.js]
+[test_utils_sets.js]
+[test_utils_utf8.js]
+[test_utils_uuid.js]
+
+[test_async_chain.js]
+[test_async_querySpinningly.js]
+
+[test_hawkclient.js]
+skip-if = os == "android"
+[test_hawkrequest.js]
+skip-if = os == "android"
+
+[test_logmanager.js]
+[test_observers.js]
+[test_restrequest.js]
+
+[test_tokenauthenticatedrequest.js]
+skip-if = os == "android"
+
+[test_tokenserverclient.js]
+skip-if = os == "android"
+
+[test_storage_server.js]
+skip-if = os == "android"