import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

services/cloudsync/CloudSync.jsm

@@ -0,0 +1,89 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["CloudSync"];
+
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "Adapters",
+                                  "resource://gre/modules/CloudSyncAdapters.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Local",
+                                  "resource://gre/modules/CloudSyncLocal.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Bookmarks",
+                                  "resource://gre/modules/CloudSyncBookmarks.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Tabs",
+                                  "resource://gre/modules/CloudSyncTabs.jsm");
+
+var API_VERSION = 1;
+
+var _CloudSync = function () {
+};
+
+_CloudSync.prototype = {
+  _adapters: null,
+
+  get adapters () {
+    if (!this._adapters) {
+      this._adapters = new Adapters();
+    }
+    return this._adapters;
+  },
+
+  _bookmarks: null,
+
+  get bookmarks () {
+    if (!this._bookmarks) {
+      this._bookmarks = new Bookmarks();
+    }
+    return this._bookmarks;
+  },
+
+  _local: null,
+
+  get local () {
+    if (!this._local) {
+      this._local = new Local();
+    }
+    return this._local;
+  },
+
+  _tabs: null,
+
+  get tabs () {
+    if (!this._tabs) {
+      this._tabs = new Tabs();
+    }
+    return this._tabs;
+  },
+
+  get tabsReady () {
+    return this._tabs ? true: false;
+  },
+
+  get version () {
+    return API_VERSION;
+  },
+};
+
+this.CloudSync = function CloudSync () {
+  return _cloudSyncInternal.instance;
+};
+
+Object.defineProperty(CloudSync, "ready", {
+  get: function () {
+    return _cloudSyncInternal.ready;
+  }
+});
+
+var _cloudSyncInternal = {
+  instance: null,
+  ready: false,
+};
+
+XPCOMUtils.defineLazyGetter(_cloudSyncInternal, "instance", function () {
+  _cloudSyncInternal.ready = true;
+  return new _CloudSync();
+}.bind(this));

services/cloudsync/CloudSyncAdapters.jsm

@@ -0,0 +1,88 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["Adapters"];
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource://gre/modules/CloudSyncEventSource.jsm");
+
+this.Adapters = function () {
+  let eventTypes = [
+    "sync",
+  ];
+
+  let suspended = true;
+
+  let suspend = function () {
+    if (!suspended) {
+      Services.obs.removeObserver(observer, "cloudsync:user-sync", false);
+      suspended = true;
+    }
+  }.bind(this);
+
+  let resume = function () {
+    if (suspended) {
+      Services.obs.addObserver(observer, "cloudsync:user-sync", false);
+      suspended = false;
+    }
+  }.bind(this);
+
+  let eventSource = new EventSource(eventTypes, suspend, resume);
+  let registeredAdapters = new Map();
+
+  function register (name, opts) {
+    opts = opts || {};
+    registeredAdapters.set(name, opts);
+  }
+
+  function unregister (name) {
+    if (!registeredAdapters.has(name)) {
+      throw new Error("adapter is not registered: " + name)
+    }
+    registeredAdapters.delete(name);
+  }
+
+  function getAdapterNames () {
+    let result = [];
+    for (let name of registeredAdapters.keys()) {
+      result.push(name);
+    }
+    return result;
+  }
+
+  function getAdapter (name) {
+    if (!registeredAdapters.has(name)) {
+      throw new Error("adapter is not registered: " + name)
+    }
+    return registeredAdapters.get(name);
+  }
+
+  function countAdapters () {
+    return registeredAdapters.size;
+  }
+
+  let observer = {
+    observe: function (subject, topic, data) {
+      switch (topic) {
+        case "cloudsync:user-sync":
+          eventSource.emit("sync");
+          break;
+      }
+    }
+  };
+
+  this.addEventListener = eventSource.addEventListener;
+  this.removeEventListener = eventSource.removeEventListener;
+  this.register = register.bind(this);
+  this.get = getAdapter.bind(this);
+  this.unregister = unregister.bind(this);
+  this.__defineGetter__("names", getAdapterNames);
+  this.__defineGetter__("count", countAdapters);
+};
+
+Adapters.prototype = {
+
+};

services/cloudsync/CloudSyncBookmarks.jsm

@@ -0,0 +1,795 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["Bookmarks"];
+
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource:///modules/PlacesUIUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
+                                  "resource://gre/modules/NetUtil.jsm");
+
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/CloudSyncPlacesWrapper.jsm");
+Cu.import("resource://gre/modules/CloudSyncEventSource.jsm");
+Cu.import("resource://gre/modules/CloudSyncBookmarksFolderCache.jsm");
+
+const ITEM_TYPES = [
+  "NULL",
+  "BOOKMARK",
+  "FOLDER",
+  "SEPARATOR",
+  "DYNAMIC_CONTAINER", // no longer used by Places, but this ID should not be used for future item types
+];
+
+const CS_UNKNOWN = 0x1;
+const CS_FOLDER = 0x1 << 1;
+const CS_SEPARATOR = 0x1 << 2;
+const CS_QUERY = 0x1 << 3;
+const CS_LIVEMARK = 0x1 << 4;
+const CS_BOOKMARK = 0x1 << 5;
+
+const EXCLUDE_BACKUP_ANNO = "places/excludeFromBackup";
+
+const DATA_VERSION = 1;
+
+function asyncCallback(ctx, func, args) {
+  function invoke() {
+    func.apply(ctx, args);
+  }
+  CommonUtils.nextTick(invoke);
+}
+
+var Record = function (params) {
+  this.id = params.guid;
+  this.parent = params.parent || null;
+  this.index = params.position;
+  this.title = params.title;
+  this.dateAdded = Math.floor(params.dateAdded/1000);
+  this.lastModified = Math.floor(params.lastModified/1000);
+  this.uri = params.url;
+
+  let annos = params.annos || {};
+  Object.defineProperty(this, "annos", {
+    get: function () {
+      return annos;
+    },
+    enumerable: false
+  });
+
+  switch (params.type) {
+    case PlacesUtils.bookmarks.TYPE_FOLDER:
+      if (PlacesUtils.LMANNO_FEEDURI in annos) {
+        this.type = CS_LIVEMARK;
+        this.feed = annos[PlacesUtils.LMANNO_FEEDURI];
+        this.site = annos[PlacesUtils.LMANNO_SITEURI];
+      } else {
+        this.type = CS_FOLDER;
+      }
+      break;
+    case PlacesUtils.bookmarks.TYPE_BOOKMARK:
+      if (this.uri.startsWith("place:")) {
+        this.type = CS_QUERY;
+      } else {
+        this.type = CS_BOOKMARK;
+      }
+      break;
+    case PlacesUtils.bookmarks.TYPE_SEPARATOR:
+      this.type = CS_SEPARATOR;
+      break;
+    default:
+      this.type = CS_UNKNOWN;
+  }
+};
+
+Record.prototype = {
+  version: DATA_VERSION,
+};
+
+var Bookmarks = function () {
+  let createRootFolder = function (name) {
+    let ROOT_FOLDER_ANNO = "cloudsync/rootFolder/" + name;
+    let ROOT_SHORTCUT_ANNO = "cloudsync/rootShortcut/" + name;
+
+    let deferred = Promise.defer();
+    let placesRootId = PlacesUtils.placesRootId;
+    let rootFolderId;
+    let rootShortcutId;
+
+    function createAdapterShortcut(result) {
+      rootFolderId = result;
+      let uri = "place:folder=" + rootFolderId;
+      return PlacesWrapper.insertBookmark(PlacesUIUtils.allBookmarksFolderId, uri,
+                                          PlacesUtils.bookmarks.DEFAULT_INDEX, name);
+    }
+
+    function setRootFolderCloudSyncAnnotation(result) {
+      rootShortcutId = result;
+      return PlacesWrapper.setItemAnnotation(rootFolderId, ROOT_FOLDER_ANNO,
+                                             1, 0, PlacesUtils.annotations.EXPIRE_NEVER);
+    }
+
+    function setRootShortcutCloudSyncAnnotation() {
+      return PlacesWrapper.setItemAnnotation(rootShortcutId, ROOT_SHORTCUT_ANNO,
+                                             1, 0, PlacesUtils.annotations.EXPIRE_NEVER);
+    }
+
+    function setRootFolderExcludeFromBackupAnnotation() {
+      return PlacesWrapper.setItemAnnotation(rootFolderId, EXCLUDE_BACKUP_ANNO,
+                                             1, 0, PlacesUtils.annotations.EXPIRE_NEVER);
+    }
+
+    function finish() {
+      deferred.resolve(rootFolderId);
+    }
+
+    Promise.resolve(PlacesUtils.bookmarks.createFolder(placesRootId, name, PlacesUtils.bookmarks.DEFAULT_INDEX))
+      .then(createAdapterShortcut)
+      .then(setRootFolderCloudSyncAnnotation)
+      .then(setRootShortcutCloudSyncAnnotation)
+      .then(setRootFolderExcludeFromBackupAnnotation)
+      .then(finish, deferred.reject);
+
+    return deferred.promise;
+  };
+
+  let getRootFolder = function (name) {
+    let ROOT_FOLDER_ANNO = "cloudsync/rootFolder/" + name;
+    let ROOT_SHORTCUT_ANNO = "cloudsync/rootShortcut/" + name;
+    let deferred = Promise.defer();
+
+    function checkRootFolder(folderIds) {
+      if (!folderIds.length) {
+        return createRootFolder(name);
+      }
+      return Promise.resolve(folderIds[0]);
+    }
+
+    function createFolderObject(folderId) {
+      return new RootFolder(folderId, name);
+    }
+
+    PlacesWrapper.getLocalIdsWithAnnotation(ROOT_FOLDER_ANNO)
+      .then(checkRootFolder, deferred.reject)
+      .then(createFolderObject)
+      .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  };
+
+  let deleteRootFolder = function (name) {
+    let ROOT_FOLDER_ANNO = "cloudsync/rootFolder/" + name;
+    let ROOT_SHORTCUT_ANNO = "cloudsync/rootShortcut/" + name;
+
+    let deferred = Promise.defer();
+    let placesRootId = PlacesUtils.placesRootId;
+
+    function getRootShortcutId() {
+      return PlacesWrapper.getLocalIdsWithAnnotation(ROOT_SHORTCUT_ANNO);
+    }
+
+    function deleteShortcut(shortcutIds) {
+      if (!shortcutIds.length) {
+        return Promise.resolve();
+      }
+      return PlacesWrapper.removeItem(shortcutIds[0]);
+    }
+
+    function getRootFolderId() {
+      return PlacesWrapper.getLocalIdsWithAnnotation(ROOT_FOLDER_ANNO);
+    }
+
+    function deleteFolder(folderIds) {
+      let deleteFolderDeferred = Promise.defer();
+
+      if (!folderIds.length) {
+        return Promise.resolve();
+      }
+
+      let rootFolderId = folderIds[0];
+      PlacesWrapper.removeFolderChildren(rootFolderId).then(
+        function () {
+          return PlacesWrapper.removeItem(rootFolderId);
+        }
+      ).then(deleteFolderDeferred.resolve, deleteFolderDeferred.reject);
+
+      return deleteFolderDeferred.promise;
+    }
+
+    getRootShortcutId().then(deleteShortcut)
+                       .then(getRootFolderId)
+                       .then(deleteFolder)
+                       .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  };
+
+  /* PUBLIC API */
+  this.getRootFolder = getRootFolder.bind(this);
+  this.deleteRootFolder = deleteRootFolder.bind(this);
+
+};
+
+this.Bookmarks = Bookmarks;
+
+var RootFolder = function (rootId, rootName) {
+  let suspended = true;
+  let ignoreAll = false;
+
+  let suspend = function () {
+    if (!suspended) {
+      PlacesUtils.bookmarks.removeObserver(observer);
+      suspended = true;
+    }
+  }.bind(this);
+
+  let resume = function () {
+    if (suspended) {
+      PlacesUtils.bookmarks.addObserver(observer, false);
+      suspended = false;
+    }
+  }.bind(this);
+
+  let eventTypes = [
+    "add",
+    "remove",
+    "change",
+    "move",
+  ];
+
+  let eventSource = new EventSource(eventTypes, suspend, resume);
+
+  let folderCache = new FolderCache;
+  folderCache.insert(rootId, null);
+
+  let getCachedFolderIds = function (cache, roots) {
+    let nodes = [...roots];
+    let results = [];
+
+    while (nodes.length) {
+      let node = nodes.shift();
+      results.push(node);
+      let children = cache.getChildren(node);
+      nodes = nodes.concat([...children]);
+    }
+    return results;
+  };
+
+  let getLocalItems = function () {
+    let deferred = Promise.defer();
+
+    let folders = getCachedFolderIds(folderCache, folderCache.getChildren(rootId));
+
+    function getFolders(ids) {
+      let types = [
+        PlacesUtils.bookmarks.TYPE_FOLDER,
+      ];
+      return PlacesWrapper.getItemsById(ids, types);
+    }
+
+    function getContents(parents) {
+      parents.push(rootId);
+      let types = [
+        PlacesUtils.bookmarks.TYPE_BOOKMARK,
+        PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      ];
+      return PlacesWrapper.getItemsByParentId(parents, types)
+    }
+
+    function getParentGuids(results) {
+      results = Array.prototype.concat.apply([], results);
+      let promises = [];
+      results.map(function (result) {
+        let promise = PlacesWrapper.localIdToGuid(result.parent).then(
+          function (guidResult) {
+            result.parent = guidResult;
+            return Promise.resolve(result);
+          },
+          Promise.reject.bind(Promise)
+        );
+        promises.push(promise);
+      });
+      return Promise.all(promises);
+    }
+
+    function getAnnos(results) {
+      results = Array.prototype.concat.apply([], results);
+      let promises = [];
+      results.map(function (result) {
+        let promise = PlacesWrapper.getItemAnnotationsForLocalId(result.id).then(
+          function (annos) {
+            result.annos = annos;
+            return Promise.resolve(result);
+          },
+          Promise.reject.bind(Promise)
+        );
+        promises.push(promise);
+      });
+      return Promise.all(promises);
+    }
+
+    let promises = [
+      getFolders(folders),
+      getContents(folders),
+    ];
+
+    Promise.all(promises)
+           .then(getParentGuids)
+           .then(getAnnos)
+           .then(function (results) {
+                   results = results.map((result) => new Record(result));
+                   deferred.resolve(results);
+                 },
+                 deferred.reject);
+
+    return deferred.promise;
+  };
+
+  let getLocalItemsById = function (guids) {
+    let deferred = Promise.defer();
+
+    let types = [
+      PlacesUtils.bookmarks.TYPE_BOOKMARK,
+      PlacesUtils.bookmarks.TYPE_FOLDER,
+      PlacesUtils.bookmarks.TYPE_SEPARATOR,
+      PlacesUtils.bookmarks.TYPE_DYNAMIC_CONTAINER,
+    ];
+
+    function getParentGuids(results) {
+      let promises = [];
+      results.map(function (result) {
+        let promise = PlacesWrapper.localIdToGuid(result.parent).then(
+          function (guidResult) {
+            result.parent = guidResult;
+            return Promise.resolve(result);
+          },
+          Promise.reject.bind(Promise)
+        );
+        promises.push(promise);
+      });
+      return Promise.all(promises);
+    }
+
+    PlacesWrapper.getItemsByGuid(guids, types)
+                 .then(getParentGuids)
+                 .then(function (results) {
+                         results = results.map((result) => new Record(result));
+                         deferred.resolve(results);
+                       },
+                       deferred.reject);
+
+    return deferred.promise;
+  };
+
+  let _createItem = function (item) {
+    let deferred = Promise.defer();
+
+    function getFolderId() {
+      if (item.parent) {
+        return PlacesWrapper.guidToLocalId(item.parent);
+      }
+      return Promise.resolve(rootId);
+    }
+
+    function create(folderId) {
+      let deferred = Promise.defer();
+
+      if (!folderId) {
+        folderId = rootId;
+      }
+      let index = item.hasOwnProperty("index") ? item.index : PlacesUtils.bookmarks.DEFAULT_INDEX;
+
+      function complete(localId) {
+        folderCache.insert(localId, folderId);
+        deferred.resolve(localId);
+      }
+
+      switch (item.type) {
+        case CS_BOOKMARK:
+        case CS_QUERY:
+          PlacesWrapper.insertBookmark(folderId, item.uri, index, item.title, item.id)
+                       .then(complete, deferred.reject);
+          break;
+        case CS_FOLDER:
+          PlacesWrapper.createFolder(folderId, item.title, index, item.id)
+                       .then(complete, deferred.reject);
+          break;
+        case CS_SEPARATOR:
+          PlacesWrapper.insertSeparator(folderId, index, item.id)
+                       .then(complete, deferred.reject);
+          break;
+        case CS_LIVEMARK:
+          let livemark = {
+            title: item.title,
+            parentId: folderId,
+            index: item.index,
+            feedURI: item.feed,
+            siteURI: item.site,
+            guid: item.id,
+          };
+          PlacesUtils.livemarks.addLivemark(livemark)
+                               .then(complete, deferred.reject);
+          break;
+        default:
+          deferred.reject("invalid item type: " + item.type);
+      }
+
+      return deferred.promise;
+    }
+
+    getFolderId().then(create)
+                 .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  };
+
+  let _deleteItem = function (item) {
+    let deferred = Promise.defer();
+
+    PlacesWrapper.guidToLocalId(item.id).then(
+      function (localId) {
+        folderCache.remove(localId);
+        return PlacesWrapper.removeItem(localId);
+      }
+    ).then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  };
+
+  let _updateItem = function (item) {
+    let deferred = Promise.defer();
+
+    PlacesWrapper.guidToLocalId(item.id).then(
+      function (localId) {
+        let promises = [];
+
+        if (item.hasOwnProperty("dateAdded")) {
+          promises.push(PlacesWrapper.setItemDateAdded(localId, item.dateAdded));
+        }
+
+        if (item.hasOwnProperty("lastModified")) {
+          promises.push(PlacesWrapper.setItemLastModified(localId, item.lastModified));
+        }
+
+        if ((CS_BOOKMARK | CS_FOLDER) & item.type && item.hasOwnProperty("title")) {
+          promises.push(PlacesWrapper.setItemTitle(localId, item.title));
+        }
+
+        if (CS_BOOKMARK & item.type && item.hasOwnProperty("uri")) {
+          promises.push(PlacesWrapper.changeBookmarkURI(localId, item.uri));
+        }
+
+        if (item.hasOwnProperty("parent")) {
+          let deferred = Promise.defer();
+          PlacesWrapper.guidToLocalId(item.parent)
+            .then(
+                function (parent) {
+                  let index = item.hasOwnProperty("index") ? item.index : PlacesUtils.bookmarks.DEFAULT_INDEX;
+                  if (CS_FOLDER & item.type) {
+                    folderCache.setParent(localId, parent);
+                  }
+                  return PlacesWrapper.moveItem(localId, parent, index);
+                }
+              )
+            .then(deferred.resolve, deferred.reject);
+          promises.push(deferred.promise);
+        }
+
+        if (item.hasOwnProperty("index") && !item.hasOwnProperty("parent")) {
+          promises.push(Task.spawn(function* () {
+            let localItem = (yield getLocalItemsById([item.id]))[0];
+            let parent = yield PlacesWrapper.guidToLocalId(localItem.parent);
+            let index = item.index;
+            if (CS_FOLDER & item.type) {
+              folderCache.setParent(localId, parent);
+            }
+            yield PlacesWrapper.moveItem(localId, parent, index);
+          }));
+        }
+
+        Promise.all(promises)
+               .then(deferred.resolve, deferred.reject);
+      }
+    );
+
+    return deferred.promise;
+  };
+
+  let mergeRemoteItems = function (items) {
+    ignoreAll = true;
+    let deferred = Promise.defer();
+
+    let newFolders = {};
+    let newItems = [];
+    let updatedItems = [];
+    let deletedItems = [];
+
+    let sortItems = function () {
+      let promises = [];
+
+      let exists = function (item) {
+        let existsDeferred = Promise.defer();
+        if (!item.id) {
+          Object.defineProperty(item, "__exists__", {
+            value: false,
+            enumerable: false
+          });
+          existsDeferred.resolve(item);
+        } else {
+          PlacesWrapper.guidToLocalId(item.id).then(
+            function (localId) {
+              Object.defineProperty(item, "__exists__", {
+                value: localId ? true : false,
+                enumerable: false
+              });
+              existsDeferred.resolve(item);
+            },
+            existsDeferred.reject
+          );
+        }
+        return existsDeferred.promise;
+      }
+
+      let handleSortedItem = function (item) {
+        if (!item.__exists__ && !item.deleted) {
+          if (CS_FOLDER == item.type) {
+            newFolders[item.id] = item;
+            item._children = [];
+          } else {
+            newItems.push(item);
+          }
+        } else if (item.__exists__ && item.deleted) {
+          deletedItems.push(item);
+        } else if (item.__exists__) {
+          updatedItems.push(item);
+        }
+      }
+
+      for (let item of items) {
+        if (!item || 'object' !== typeof(item)) {
+          continue;
+        }
+
+        let promise = exists(item).then(handleSortedItem, Promise.reject.bind(Promise));
+        promises.push(promise);
+      }
+
+      return Promise.all(promises);
+    }
+
+    let processNewFolders = function () {
+      let newFolderGuids = Object.keys(newFolders);
+      let newFolderRoots = [];
+
+      for (let guid of newFolderGuids) {
+        let item = newFolders[guid];
+        if (item.parent && newFolderGuids.indexOf(item.parent) >= 0) {
+          let parent = newFolders[item.parent];
+          parent._children.push(item.id);
+        } else {
+          newFolderRoots.push(guid);
+        }
+      };
+
+      let promises = [];
+      for (let guid of newFolderRoots) {
+        let root = newFolders[guid];
+        let promise = Promise.resolve();
+        promise = promise.then(
+          function () {
+            return _createItem(root);
+          },
+          Promise.reject.bind(Promise)
+        );
+        let items = [].concat(root._children);
+
+        while (items.length) {
+          let item = newFolders[items.shift()];
+          items = items.concat(item._children);
+          promise = promise.then(
+            function () {
+              return _createItem(item);
+            },
+            Promise.reject.bind(Promise)
+          );
+        }
+        promises.push(promise);
+      }
+
+      return Promise.all(promises);
+    }
+
+    let processItems = function () {
+      let promises = [];
+
+      for (let item of newItems) {
+        promises.push(_createItem(item));
+      }
+
+      for (let item of updatedItems) {
+        promises.push(_updateItem(item));
+      }
+
+      for (let item of deletedItems) {
+        _deleteItem(item);
+      }
+
+      return Promise.all(promises);
+    }
+
+    sortItems().then(processNewFolders)
+               .then(processItems)
+               .then(function () {
+                       ignoreAll = false;
+                       deferred.resolve(items);
+                     },
+                     function (err) {
+                       ignoreAll = false;
+                       deferred.reject(err);
+                     });
+
+    return deferred.promise;
+  };
+
+  let ignore = function (id, parent) {
+    if (ignoreAll) {
+      return true;
+    }
+
+    if (rootId == parent || folderCache.has(parent)) {
+      return false;
+    }
+
+    return true;
+  };
+
+  let handleItemAdded = function (id, parent, index, type, uri, title, dateAdded, guid, parentGuid) {
+    let deferred = Promise.defer();
+
+    if (PlacesUtils.bookmarks.TYPE_FOLDER == type) {
+      folderCache.insert(id, parent);
+    }
+
+    eventSource.emit("add", guid);
+    deferred.resolve();
+
+    return deferred.promise;
+  };
+
+  let handleItemRemoved = function (id, parent, index, type, uri, guid, parentGuid) {
+    let deferred = Promise.defer();
+
+    if (PlacesUtils.bookmarks.TYPE_FOLDER == type) {
+      folderCache.remove(id);
+    }
+
+    eventSource.emit("remove", guid);
+    deferred.resolve();
+
+    return deferred.promise;
+  };
+
+  let handleItemChanged = function (id, property, isAnnotation, newValue, lastModified, type, parent, guid, parentGuid) {
+    let deferred = Promise.defer();
+
+    eventSource.emit('change', guid);
+    deferred.resolve();
+
+    return deferred.promise;
+  };
+
+  let handleItemMoved = function (id, oldParent, oldIndex, newParent, newIndex, type, guid, oldParentGuid, newParentGuid) {
+    let deferred = Promise.defer();
+
+    function complete() {
+      eventSource.emit('move', guid);
+      deferred.resolve();
+    }
+
+    if (PlacesUtils.bookmarks.TYPE_FOLDER != type) {
+      complete();
+      return deferred.promise;
+    }
+
+    if (folderCache.has(oldParent) && folderCache.has(newParent)) {
+      // Folder move inside cloudSync root, so just update parents/children.
+      folderCache.setParent(id, newParent);
+      complete();
+    } else if (!folderCache.has(oldParent)) {
+      // Folder moved in from ouside cloudSync root.
+      PlacesWrapper.updateCachedFolderIds(folderCache, newParent)
+                   .then(complete, complete);
+    } else if (!folderCache.has(newParent)) {
+      // Folder moved out from inside cloudSync root.
+      PlacesWrapper.updateCachedFolderIds(folderCache, oldParent)
+                   .then(complete, complete);
+    }
+
+    return deferred.promise;
+  };
+
+  let observer = {
+    onBeginBatchUpdate: function () {
+    },
+
+    onEndBatchUpdate: function () {
+    },
+
+    onItemAdded: function (id, parent, index, type, uri, title, dateAdded, guid, parentGuid) {
+      if (ignore(id, parent)) {
+        return;
+      }
+
+      asyncCallback(this, handleItemAdded, Array.prototype.slice.call(arguments));
+    },
+
+    onItemRemoved: function (id, parent, index, type, uri, guid, parentGuid) {
+      if (ignore(id, parent)) {
+        return;
+      }
+
+      asyncCallback(this, handleItemRemoved, Array.prototype.slice.call(arguments));
+    },
+
+    onItemChanged: function (id, property, isAnnotation, newValue, lastModified, type, parent, guid, parentGuid) {
+      if (ignore(id, parent)) {
+        return;
+      }
+
+      asyncCallback(this, handleItemChanged, Array.prototype.slice.call(arguments));
+    },
+
+    onItemMoved: function (id, oldParent, oldIndex, newParent, newIndex, type, guid, oldParentGuid, newParentGuid) {
+      if (ignore(id, oldParent) && ignore(id, newParent)) {
+        return;
+      }
+
+      asyncCallback(this, handleItemMoved, Array.prototype.slice.call(arguments));
+    }
+  };
+
+  /* PUBLIC API */
+  this.addEventListener = eventSource.addEventListener;
+  this.removeEventListener = eventSource.removeEventListener;
+  this.getLocalItems = getLocalItems.bind(this);
+  this.getLocalItemsById = getLocalItemsById.bind(this);
+  this.mergeRemoteItems = mergeRemoteItems.bind(this);
+
+  let rootGuid = null; // resolved before becoming ready (below)
+  this.__defineGetter__("id", function () {
+    return rootGuid;
+  });
+  this.__defineGetter__("name", function () {
+    return rootName;
+  });
+
+  let deferred = Promise.defer();
+  let getGuidForRootFolder = function () {
+    return PlacesWrapper.localIdToGuid(rootId);
+  }
+  PlacesWrapper.updateCachedFolderIds(folderCache, rootId)
+               .then(getGuidForRootFolder, getGuidForRootFolder)
+               .then(function (guid) {
+                       rootGuid = guid;
+                       deferred.resolve(this);
+                     }.bind(this),
+                     deferred.reject);
+  return deferred.promise;
+};
+
+RootFolder.prototype = {
+  BOOKMARK: CS_BOOKMARK,
+  FOLDER: CS_FOLDER,
+  SEPARATOR: CS_SEPARATOR,
+  QUERY: CS_QUERY,
+  LIVEMARK: CS_LIVEMARK,
+};

services/cloudsync/CloudSyncBookmarksFolderCache.jsm

@@ -0,0 +1,105 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["FolderCache"];
+
+// Cache for bookmarks folder heirarchy.
+var FolderCache = function () {
+  this.cache = new Map();
+}
+
+FolderCache.prototype = {
+  has: function (id) {
+    return this.cache.has(id);
+  },
+
+  insert: function (id, parentId) {
+    if (this.cache.has(id)) {
+      return;
+    }
+
+    if (parentId && !(this.cache.has(parentId))) {
+      throw new Error("insert :: parentId not found in cache: " + parentId);
+    }
+
+    this.cache.set(id, {
+      parent: parentId || null,
+      children: new Set(),
+    });
+
+    if (parentId) {
+      this.cache.get(parentId).children.add(id);
+    }
+  },
+
+  remove: function (id) {
+    if (!(this.cache.has(id))) {
+      throw new Error("remote :: id not found in cache: " + id);
+    }
+
+    let parentId = this.cache.get(id).parent;
+    if (parentId) {
+      this.cache.get(parentId).children.delete(id);
+    }
+
+    for (let child of this.cache.get(id).children) {
+      this.cache.get(child).parent = null;
+    }
+
+    this.cache.delete(id);
+  },
+
+  setParent: function (id, parentId) {
+    if (!(this.cache.has(id))) {
+      throw new Error("setParent :: id not found in cache: " + id);
+    }
+
+    if (parentId && !(this.cache.has(parentId))) {
+      throw new Error("setParent :: parentId not found in cache: " + parentId);
+    }
+
+    let oldParent = this.cache.get(id).parent;
+    if (oldParent) {
+      this.cache.get(oldParent).children.delete(id);
+    }
+    this.cache.get(id).parent = parentId;
+    this.cache.get(parentId).children.add(id);
+
+    return true;
+  },
+
+  getParent: function (id) {
+    if (this.cache.has(id)) {
+      return this.cache.get(id).parent;
+    }
+
+    throw new Error("getParent :: id not found in cache: " + id);
+  },
+
+  getChildren: function (id) {
+    if (this.cache.has(id)) {
+      return this.cache.get(id).children;
+    }
+
+    throw new Error("getChildren :: id not found in cache: " + id);
+  },
+
+  setChildren: function (id, children) {
+    for (let child of children) {
+      if (!this.cache.has(child)) {
+        this.insert(child, id);
+      } else {
+        this.setParent(child, id);
+      }
+    }
+  },
+
+  dump: function () {
+    dump("FolderCache: " + JSON.stringify(this.cache) + "\n");
+  },
+};
+
+this.FolderCache = FolderCache;

services/cloudsync/CloudSyncEventSource.jsm

@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["EventSource"];
+
+Components.utils.import("resource://services-common/utils.js");
+
+var EventSource = function (types, suspendFunc, resumeFunc) {
+  this.listeners = new Map();
+  for (let type of types) {
+    this.listeners.set(type, new Set());
+  }
+
+  this.suspend = suspendFunc || function () {};
+  this.resume = resumeFunc || function () {};
+
+  this.addEventListener = this.addEventListener.bind(this);
+  this.removeEventListener = this.removeEventListener.bind(this);
+};
+
+EventSource.prototype = {
+  addEventListener: function (type, listener) {
+    if (!this.listeners.has(type)) {
+      return;
+    }
+    this.listeners.get(type).add(listener);
+    this.resume();
+  },
+
+  removeEventListener: function (type, listener) {
+    if (!this.listeners.has(type)) {
+      return;
+    }
+    this.listeners.get(type).delete(listener);
+    if (!this.hasListeners()) {
+      this.suspend();
+    }
+  },
+
+  hasListeners: function () {
+    for (let l of this.listeners.values()) {
+      if (l.size > 0) {
+        return true;
+      }
+    }
+    return false;
+  },
+
+  emit: function (type, arg) {
+    if (!this.listeners.has(type)) {
+      return;
+    }
+    CommonUtils.nextTick(
+      function () {
+        for (let listener of this.listeners.get(type)) {
+          listener.call(undefined, arg);
+        }
+      },
+      this
+    );
+  },
+};
+
+this.EventSource = EventSource;

services/cloudsync/CloudSyncLocal.jsm

@@ -0,0 +1,87 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["Local"];
+
+const Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/stringbundle.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+function lazyStrings(name) {
+  let bundle = "chrome://weave/locale/services/" + name + ".properties";
+  return () => new StringBundle(bundle);
+}
+
+this.Str = {};
+XPCOMUtils.defineLazyGetter(Str, "errors", lazyStrings("errors"));
+XPCOMUtils.defineLazyGetter(Str, "sync", lazyStrings("sync"));
+
+function makeGUID() {
+  return CommonUtils.encodeBase64URL(CryptoUtils.generateRandomBytes(9));
+}
+
+this.Local = function () {
+  let prefs = new Preferences("services.cloudsync.");
+  this.__defineGetter__("prefs", function () {
+    return prefs;
+  });
+};
+
+Local.prototype = {
+  get id() {
+    let clientId = this.prefs.get("client.GUID", "");
+    return clientId == "" ? this.id = makeGUID(): clientId;
+  },
+
+  set id(value) {
+    this.prefs.set("client.GUID", value);
+  },
+
+  get name() {
+    let clientName = this.prefs.get("client.name", "");
+
+    if (clientName != "") {
+      return clientName;
+    }
+
+    // Generate a client name if we don't have a useful one yet
+    let env = Cc["@mozilla.org/process/environment;1"]
+                .getService(Ci.nsIEnvironment);
+    let user = env.get("USER") || env.get("USERNAME");
+    let appName;
+    let brand = new StringBundle("chrome://branding/locale/brand.properties");
+    let brandName = brand.get("brandShortName");
+
+    try {
+      let syncStrings = new StringBundle("chrome://browser/locale/sync.properties");
+      appName = syncStrings.getFormattedString("sync.defaultAccountApplication", [brandName]);
+    } catch (ex) {
+    }
+
+    appName = appName || brandName;
+
+    let system =
+      // 'device' is defined on unix systems
+      Cc["@mozilla.org/system-info;1"].getService(Ci.nsIPropertyBag2).get("device") ||
+      // hostname of the system, usually assigned by the user or admin
+      Cc["@mozilla.org/system-info;1"].getService(Ci.nsIPropertyBag2).get("host") ||
+      // fall back on ua info string
+      Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler).oscpu;
+
+    return this.name = Str.sync.get("client.name2", [user, appName, system]);
+  },
+
+  set name(value) {
+    this.prefs.set("client.name", value);
+  },
+};
+

services/cloudsync/CloudSyncPlacesWrapper.jsm

@@ -0,0 +1,375 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["PlacesWrapper"];
+
+const {interfaces: Ci, utils: Cu} = Components;
+const REASON_ERROR = Ci.mozIStorageStatementCallback.REASON_ERROR;
+
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource:///modules/PlacesUIUtils.jsm");
+Cu.import("resource://services-common/utils.js");
+
+var PlacesQueries = function () {
+}
+
+PlacesQueries.prototype = {
+  cachedStmts: {},
+
+  getQuery: function (queryString) {
+    if (queryString in this.cachedStmts) {
+      return this.cachedStmts[queryString];
+    }
+
+    let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection;
+    return this.cachedStmts[queryString] = db.createAsyncStatement(queryString);
+  }
+};
+
+var PlacesWrapper = function () {
+}
+
+PlacesWrapper.prototype = {
+  placesQueries: new PlacesQueries(),
+
+  guidToLocalId: function (guid) {
+    let deferred = Promise.defer();
+
+    let stmt = "SELECT id AS item_id " +
+               "FROM moz_bookmarks " +
+               "WHERE guid = :guid";
+    let query = this.placesQueries.getQuery(stmt);
+
+    function getLocalId(results) {
+      let result = results[0] && results[0]["item_id"];
+      return Promise.resolve(result);
+    }
+
+    query.params.guid = guid.toString();
+
+    this.asyncQuery(query, ["item_id"])
+        .then(getLocalId, deferred.reject)
+        .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  },
+
+  localIdToGuid: function (id) {
+    let deferred = Promise.defer();
+
+    let stmt = "SELECT guid " +
+               "FROM moz_bookmarks " +
+               "WHERE id = :item_id";
+    let query = this.placesQueries.getQuery(stmt);
+
+    function getGuid(results) {
+      let result = results[0] && results[0]["guid"];
+      return Promise.resolve(result);
+    }
+
+    query.params.item_id = id;
+
+    this.asyncQuery(query, ["guid"])
+        .then(getGuid, deferred.reject)
+        .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  },
+
+  getItemsById: function (ids, types) {
+    let deferred = Promise.defer();
+    let stmt = "SELECT b.id, b.type, b.parent, b.position, b.title, b.guid, b.dateAdded, b.lastModified, p.url " +
+               "FROM moz_bookmarks b " +
+               "LEFT JOIN moz_places p ON b.fk = p.id " +
+               "WHERE b.id in (" + ids.join(",") + ") AND b.type in (" + types.join(",") + ")";
+    let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection;
+    let query = db.createAsyncStatement(stmt);
+
+    this.asyncQuery(query, ["id", "type", "parent", "position", "title", "guid", "dateAdded", "lastModified", "url"])
+        .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  },
+
+  getItemsByParentId: function (parents, types) {
+    let deferred = Promise.defer();
+    let stmt = "SELECT b.id, b.type, b.parent, b.position, b.title, b.guid, b.dateAdded, b.lastModified, p.url " +
+               "FROM moz_bookmarks b " +
+               "LEFT JOIN moz_places p ON b.fk = p.id " +
+               "WHERE b.parent in (" + parents.join(",") + ") AND b.type in (" + types.join(",") + ")";
+    let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection;
+    let query = db.createAsyncStatement(stmt);
+
+    this.asyncQuery(query, ["id", "type", "parent", "position", "title", "guid", "dateAdded", "lastModified", "url"])
+        .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  },
+
+  getItemsByGuid: function (guids, types) {
+    let deferred = Promise.defer();
+    guids = guids.map(JSON.stringify);
+    let stmt = "SELECT b.id, b.type, b.parent, b.position, b.title, b.guid, b.dateAdded, b.lastModified, p.url " +
+               "FROM moz_bookmarks b " +
+               "LEFT JOIN moz_places p ON b.fk = p.id " +
+               "WHERE b.guid in (" + guids.join(",") + ") AND b.type in (" + types.join(",") + ")";
+    let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase).DBConnection;
+    let query = db.createAsyncStatement(stmt);
+
+    this.asyncQuery(query, ["id", "type", "parent", "position", "title", "guid", "dateAdded", "lastModified", "url"])
+        .then(deferred.resolve, deferred.reject);
+
+    return deferred.promise;
+  },
+
+  updateCachedFolderIds: function (folderCache, folder) {
+    let deferred = Promise.defer();
+    let stmt = "SELECT id, guid " +
+               "FROM moz_bookmarks " +
+               "WHERE parent = :parent_id AND type = :item_type";
+    let query = this.placesQueries.getQuery(stmt);
+
+    query.params.parent_id = folder;
+    query.params.item_type = PlacesUtils.bookmarks.TYPE_FOLDER;
+
+    this.asyncQuery(query, ["id", "guid"]).then(
+      function (items) {
+        let previousIds = folderCache.getChildren(folder);
+        let currentIds = new Set();
+        for (let item of items) {
+          currentIds.add(item.id);
+        }
+        let newIds = new Set();
+        let missingIds = new Set();
+
+        for (let currentId of currentIds) {
+          if (!previousIds.has(currentId)) {
+            newIds.add(currentId);
+          }
+        }
+        for (let previousId of previousIds) {
+          if (!currentIds.has(previousId)) {
+            missingIds.add(previousId);
+          }
+        }
+
+        folderCache.setChildren(folder, currentIds);
+
+        let promises = [];
+        for (let newId of newIds) {
+          promises.push(this.updateCachedFolderIds(folderCache, newId));
+        }
+        Promise.all(promises)
+               .then(deferred.resolve, deferred.reject);
+
+        for (let missingId of missingIds) {
+          folderCache.remove(missingId);
+        }
+      }.bind(this)
+    );
+
+    return deferred.promise;
+  },
+
+  getLocalIdsWithAnnotation: function (anno) {
+    let deferred = Promise.defer();
+    let stmt = "SELECT a.item_id " +
+               "FROM moz_anno_attributes n " +
+               "JOIN moz_items_annos a ON n.id = a.anno_attribute_id " +
+               "WHERE n.name = :anno_name";
+    let query = this.placesQueries.getQuery(stmt);
+
+    query.params.anno_name = anno.toString();
+
+    this.asyncQuery(query, ["item_id"])
+        .then(function (items) {
+                let results = [];
+                for (let item of items) {
+                  results.push(item.item_id);
+                }
+                deferred.resolve(results);
+              },
+              deferred.reject);
+
+    return deferred.promise;
+  },
+
+  getItemAnnotationsForLocalId: function (id) {
+    let deferred = Promise.defer();
+    let stmt = "SELECT a.name, b.content " +
+               "FROM moz_anno_attributes a " +
+               "JOIN moz_items_annos b ON a.id = b.anno_attribute_id " +
+               "WHERE b.item_id = :item_id";
+    let query = this.placesQueries.getQuery(stmt);
+
+    query.params.item_id = id;
+
+    this.asyncQuery(query, ["name", "content"])
+        .then(function (results) {
+                let annos = {};
+                for (let result of results) {
+                  annos[result.name] = result.content;
+                }
+                deferred.resolve(annos);
+              },
+              deferred.reject);
+
+    return deferred.promise;
+  },
+
+  insertBookmark: function (parent, uri, index, title, guid) {
+    let parsedURI;
+    try {
+      parsedURI = CommonUtils.makeURI(uri)
+    } catch (e) {
+      return Promise.reject("unable to parse URI '" + uri + "': " + e);
+    }
+
+    try {
+      let id = PlacesUtils.bookmarks.insertBookmark(parent, parsedURI, index, title, guid);
+      return Promise.resolve(id);
+    } catch (e) {
+      return Promise.reject("unable to insert bookmark " + JSON.stringify(arguments) + ": " + e);
+    }
+  },
+
+  setItemAnnotation: function (item, anno, value, flags, exp) {
+    try {
+      return Promise.resolve(PlacesUtils.annotations.setItemAnnotation(item, anno, value, flags, exp));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  itemHasAnnotation: function (item, anno) {
+    try {
+      return Promise.resolve(PlacesUtils.annotations.itemHasAnnotation(item, anno));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  createFolder: function (parent, name, index, guid) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.createFolder(parent, name, index, guid));
+    } catch (e) {
+      return Promise.reject("unable to create folder ['" + name + "']: " + e);
+    }
+  },
+
+  removeFolderChildren: function (folder) {
+    try {
+      PlacesUtils.bookmarks.removeFolderChildren(folder);
+      return Promise.resolve();
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  insertSeparator: function (parent, index, guid) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.insertSeparator(parent, index, guid));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  removeItem: function (item) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.removeItem(item));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  setItemDateAdded: function (item, dateAdded) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.setItemDateAdded(item, dateAdded));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  setItemLastModified: function (item, lastModified) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.setItemLastModified(item, lastModified));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  setItemTitle: function (item, title) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.setItemTitle(item, title));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  changeBookmarkURI: function (item, uri) {
+    try {
+      uri = CommonUtils.makeURI(uri);
+      return Promise.resolve(PlacesUtils.bookmarks.changeBookmarkURI(item, uri));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  moveItem: function (item, parent, index) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.moveItem(item, parent, index));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  setItemIndex: function (item, index) {
+    try {
+      return Promise.resolve(PlacesUtils.bookmarks.setItemIndex(item, index));
+    } catch (e) {
+      return Promise.reject(e);
+    }
+  },
+
+  asyncQuery: function (query, names) {
+    let deferred = Promise.defer();
+    let storageCallback = {
+      results: [],
+      handleResult: function (results) {
+        if (!names) {
+          return;
+        }
+
+        let row;
+        while ((row = results.getNextRow()) != null) {
+          let item = {};
+          for (let name of names) {
+            item[name] = row.getResultByName(name);
+          }
+          this.results.push(item);
+        }
+      },
+
+      handleError: function (error) {
+        deferred.reject(error);
+      },
+
+      handleCompletion: function (reason) {
+        if (REASON_ERROR == reason) {
+          return;
+        }
+
+        deferred.resolve(this.results);
+      }
+    };
+
+    query.executeAsync(storageCallback);
+    return deferred.promise;
+  },
+};
+
+this.PlacesWrapper = new PlacesWrapper();

services/cloudsync/CloudSyncTabs.jsm

@@ -0,0 +1,318 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["Tabs"];
+
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/CloudSyncEventSource.jsm");
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://services-common/observers.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils", "resource://gre/modules/PrivateBrowsingUtils.jsm");
+XPCOMUtils.defineLazyServiceGetter(this, "Session", "@mozilla.org/browser/sessionstore;1", "nsISessionStore");
+
+const DATA_VERSION = 1;
+
+var ClientRecord = function (params) {
+  this.id = params.id;
+  this.name = params.name || "?";
+  this.tabs = new Set();
+}
+
+ClientRecord.prototype = {
+  version: DATA_VERSION,
+
+  update: function (params) {
+    if (this.id !== params.id) {
+      throw new Error("expected " + this.id + " to equal " + params.id);
+    }
+
+    this.name = params.name;
+  }
+};
+
+var TabRecord = function (params) {
+  this.url = params.url || "";
+  this.update(params);
+};
+
+TabRecord.prototype = {
+  version: DATA_VERSION,
+
+  update: function (params) {
+    if (this.url && this.url !== params.url) {
+      throw new Error("expected " + this.url + " to equal " + params.url);
+    }
+
+    if (params.lastUsed && params.lastUsed < this.lastUsed) {
+      return;
+    }
+
+    this.title = params.title || "";
+    this.icon = params.icon || "";
+    this.lastUsed = params.lastUsed || 0;
+  },
+};
+
+var TabCache = function () {
+  this.tabs = new Map();
+  this.clients = new Map();
+};
+
+TabCache.prototype = {
+  merge: function (client, tabs) {
+    if (!client || !client.id) {
+      return;
+    }
+
+    if (!tabs) {
+      return;
+    }
+
+    let cRecord;
+    if (this.clients.has(client.id)) {
+      try {
+        cRecord = this.clients.get(client.id);
+      } catch (e) {
+        throw new Error("unable to update client: " + e);
+      }
+    } else {
+      cRecord = new ClientRecord(client);
+      this.clients.set(cRecord.id, cRecord);
+    }
+
+    for (let tab of tabs) {
+      if (!tab || 'object' !== typeof(tab)) {
+        continue;
+      }
+
+      let tRecord;
+      if (this.tabs.has(tab.url)) {
+        tRecord = this.tabs.get(tab.url);
+        try {
+          tRecord.update(tab);
+        } catch (e) {
+          throw new Error("unable to update tab: " + e);
+        }
+      } else {
+        tRecord = new TabRecord(tab);
+        this.tabs.set(tRecord.url, tRecord);
+      }
+
+      if (tab.deleted) {
+        cRecord.tabs.delete(tRecord);
+      } else {
+        cRecord.tabs.add(tRecord);
+      }
+    }
+  },
+
+  clear: function (client) {
+    if (client) {
+      this.clients.delete(client.id);
+    } else {
+      this.clients = new Map();
+      this.tabs = new Map();
+    }
+  },
+
+  get: function () {
+    let results = [];
+    for (let client of this.clients.values()) {
+      results.push(client);
+    }
+    return results;
+  },
+
+  isEmpty: function () {
+    return 0 == this.clients.size;
+  },
+
+};
+
+this.Tabs = function () {
+  let suspended = true;
+
+  let topics = [
+    "pageshow",
+    "TabOpen",
+    "TabClose",
+    "TabSelect",
+  ];
+
+  let update = function (event) {
+    if (event.originalTarget.linkedBrowser) {
+      if (PrivateBrowsingUtils.isBrowserPrivate(event.originalTarget.linkedBrowser) &&
+          !PrivateBrowsingUtils.permanentPrivateBrowsing) {
+        return;
+      }
+    }
+
+    eventSource.emit("change");
+  };
+
+  let registerListenersForWindow = function (window) {
+    for (let topic of topics) {
+      window.addEventListener(topic, update, false);
+    }
+    window.addEventListener("unload", unregisterListeners, false);
+  };
+
+  let unregisterListenersForWindow = function (window) {
+    window.removeEventListener("unload", unregisterListeners, false);
+    for (let topic of topics) {
+      window.removeEventListener(topic, update, false);
+    }
+  };
+
+  let unregisterListeners = function (event) {
+    unregisterListenersForWindow(event.target);
+  };
+
+  let observer = {
+    observe: function (subject, topic, data) {
+      switch (topic) {
+        case "domwindowopened":
+          let onLoad = () => {
+            subject.removeEventListener("load", onLoad, false);
+            // Only register after the window is done loading to avoid unloads.
+            registerListenersForWindow(subject);
+          };
+
+          // Add tab listeners now that a window has opened.
+          subject.addEventListener("load", onLoad, false);
+          break;
+      }
+    }
+  };
+
+  let resume = function () {
+    if (suspended) {
+      Observers.add("domwindowopened", observer);
+      let wins = Services.wm.getEnumerator("navigator:browser");
+      while (wins.hasMoreElements()) {
+        registerListenersForWindow(wins.getNext());
+      }
+    }
+  }.bind(this);
+
+  let suspend = function () {
+    if (!suspended) {
+      Observers.remove("domwindowopened", observer);
+      let wins = Services.wm.getEnumerator("navigator:browser");
+      while (wins.hasMoreElements()) {
+        unregisterListenersForWindow(wins.getNext());
+      }
+    }
+  }.bind(this);
+
+  let eventTypes = [
+    "change",
+  ];
+
+  let eventSource = new EventSource(eventTypes, suspend, resume);
+
+  let tabCache = new TabCache();
+
+  let getWindowEnumerator = function () {
+    return Services.wm.getEnumerator("navigator:browser");
+  };
+
+  let shouldSkipWindow = function (win) {
+    return win.closed ||
+           PrivateBrowsingUtils.isWindowPrivate(win);
+  };
+
+  let getTabState = function (tab) {
+    return JSON.parse(Session.getTabState(tab));
+  };
+
+  let getLocalTabs = function (filter) {
+    let deferred = Promise.defer();
+
+    filter = (undefined === filter) ? true : filter;
+    let filteredUrls = new RegExp("^(about:.*|chrome://weave/.*|wyciwyg:.*|file:.*)$"); // FIXME: should be a pref (B#1044304)
+
+    let allTabs = [];
+
+    let currentState = JSON.parse(Session.getBrowserState());
+    currentState.windows.forEach(function (window) {
+      if (window.isPrivate) {
+        return;
+      }
+      window.tabs.forEach(function (tab) {
+        if (!tab.entries.length) {
+          return;
+        }
+
+        // Get only the latest entry
+        // FIXME: support full history (B#1044306)
+        let entry = tab.entries[tab.index - 1];
+
+        if (!entry.url || filter && filteredUrls.test(entry.url)) {
+          return;
+        }
+
+        allTabs.push(new TabRecord({
+          title: entry.title,
+          url: entry.url,
+          icon: tab.attributes && tab.attributes.image || "",
+          lastUsed: tab.lastAccessed,
+        }));
+      });
+    });
+
+    deferred.resolve(allTabs);
+
+    return deferred.promise;
+  };
+
+  let mergeRemoteTabs = function (client, tabs) {
+    let deferred = Promise.defer();
+
+    deferred.resolve(tabCache.merge(client, tabs));
+    Observers.notify("cloudsync:tabs:update");
+
+    return deferred.promise;
+  };
+
+  let clearRemoteTabs = function (client) {
+    let deferred = Promise.defer();
+
+    deferred.resolve(tabCache.clear(client));
+    Observers.notify("cloudsync:tabs:update");
+
+    return deferred.promise;
+  };
+
+  let getRemoteTabs = function () {
+    let deferred = Promise.defer();
+
+    deferred.resolve(tabCache.get());
+
+    return deferred.promise;
+  };
+
+  let hasRemoteTabs = function () {
+    return !tabCache.isEmpty();
+  };
+
+  /* PUBLIC API */
+  this.addEventListener = eventSource.addEventListener;
+  this.removeEventListener = eventSource.removeEventListener;
+  this.getLocalTabs = getLocalTabs.bind(this);
+  this.mergeRemoteTabs = mergeRemoteTabs.bind(this);
+  this.clearRemoteTabs = clearRemoteTabs.bind(this);
+  this.getRemoteTabs = getRemoteTabs.bind(this);
+  this.hasRemoteTabs = hasRemoteTabs.bind(this);
+};
+
+Tabs.prototype = {
+};
+this.Tabs = Tabs;

services/cloudsync/docs/api.md

@@ -0,0 +1,234 @@
+### Importing the JS module
+
+````
+Cu.import("resource://gre/modules/CloudSync.jsm");
+
+let cloudSync = CloudSync();
+console.log(cloudSync); // Module is imported
+````
+
+### cloudSync.local
+
+#### id
+
+Local device ID. Is unique.
+
+````
+let localId = cloudSync.local.id;
+````
+
+#### name
+
+Local device name.
+
+````
+let localName = cloudSync.local.name;
+````
+
+### CloudSync.tabs
+
+#### addEventListener(type, callback)
+
+Add an event handler for Tabs events. Valid type is `change`. The callback receives no arguments.
+
+````
+function handleTabChange() {
+  // Tabs have changed.
+}
+
+cloudSync.tabs.addEventListener("change", handleTabChange);
+````
+
+Change events are emitted when a tab is opened or closed, when a tab is selected, or when the page changes for an open tab.
+
+#### removeEventListener(type, callback)
+
+Remove an event handler. Pass the type and function that were passed to addEventListener.
+
+````
+cloudSync.tabs.removeEventListener("change", handleTabChange);
+````
+
+#### mergeRemoteTabs(client, tabs)
+
+Merge remote tabs from upstream by updating existing items, adding new tabs, and deleting existing tabs. Accepts a client and a list of tabs. Returns a promise.
+
+````
+let remoteClient = {
+  id: "fawe78",
+  name: "My Firefox client",
+};
+
+let remoteTabs = [
+    {title: "Google",
+     url: "https://www.google.com",
+     icon: "https://www.google.com/favicon.ico",
+     lastUsed: 1400799296192},
+    {title: "Reddit",
+     url: "http://www.reddit.com",
+     icon: "http://www.reddit.com/favicon.ico",
+     lastUsed: 1400799296192
+     deleted: true},
+];
+
+cloudSync.tabs.mergeRemoteTabs(client, tabs).then(
+  function() {
+    console.log("merge complete");
+  }
+);
+````
+
+#### getLocalTabs()
+
+Returns a promise. Passes a list of local tabs when complete.
+
+````
+cloudSync.tabs.getLocalTabs().then(
+  function(tabs) {
+    console.log(JSON.stringify(tabs));
+  }
+);
+````
+
+#### clearRemoteTabs(client)
+
+Clears all tabs for a remote client.
+
+````
+let remoteClient = {
+  id: "fawe78",
+  name: "My Firefox client",
+};
+
+cloudSync.tabs.clearRemoteTabs(client);
+````
+
+### cloudSync.bookmarks
+
+#### getRootFolder(name)
+
+Gets the named root folder, creating it if it doesn't exist. The root folder object has a number of methods (see the next section for details).
+
+````
+cloudSync.bookmarks.getRootFolder("My Bookmarks").then(
+  function(rootFolder) {
+    console.log(rootFolder);
+  }
+);
+````
+
+### cloudSync.bookmarks.RootFolder
+
+This is a root folder object for bookmarks, created by `cloudSync.bookmarks.getRootFolder`.
+
+#### BOOKMARK
+
+Bookmark type. Used in results objects.
+
+````
+let bookmarkType = rootFolder.BOOKMARK;
+````
+
+#### FOLDER
+
+Folder type. Used in results objects.
+
+````
+let folderType = rootFolder.FOLDER;
+````
+
+#### SEPARATOR
+
+Separator type. Used in results objects.
+
+````
+let separatorType = rootFolder.SEPARATOR;
+````
+
+#### addEventListener(type, callback)
+
+Add an event handler for Tabs events. Valid types are `add, remove, change, move`. The callback receives an ID corresponding to the target item.
+
+````
+function handleBoookmarkEvent(id) {
+  console.log("event for id:", id);
+}
+
+rootFolder.addEventListener("add", handleBookmarkEvent);
+rootFolder.addEventListener("remove", handleBookmarkEvent);
+rootFolder.addEventListener("change", handleBookmarkEvent);
+rootFolder.addEventListener("move", handleBookmarkEvent);
+````
+
+#### removeEventListener(type, callback)
+
+Remove an event handler. Pass the type and function that were passed to addEventListener.
+
+````
+rootFolder.removeEventListener("add", handleBookmarkEvent);
+rootFolder.removeEventListener("remove", handleBookmarkEvent);
+rootFolder.removeEventListener("change", handleBookmarkEvent);
+rootFolder.removeEventListener("move", handleBookmarkEvent);
+````
+
+#### getLocalItems()
+
+Callback receives a list of items on the local client. Results have the following form:
+
+````
+{
+  id: "faw8e7f", // item guid
+  parent: "f7sydf87y", // parent folder guid
+  dateAdded: 1400799296192, // timestamp
+  lastModified: 1400799296192, // timestamp
+  uri: "https://www.google.ca", // null for FOLDER and SEPARATOR
+  title: "Google"
+  type: rootFolder.BOOKMARK, // should be one of rootFolder.{BOOKMARK, FOLDER, SEPARATOR},
+  index: 0 // must be unique among folder items
+}
+````
+
+````
+rootFolder.getLocalItems().then(
+  function(items) {
+    console.log(JSON.stringify(items));
+  }
+);
+````
+
+#### getLocalItemsById([...])
+
+Callback receives a list of items, specified by ID, on the local client. Results have the same form as `getLocalItems()` above.
+
+````
+rootFolder.getLocalItemsById(["213r23f", "f22fy3f3"]).then(
+  function(items) {
+    console.log(JSON.stringify(items));
+  }
+);
+````
+
+#### mergeRemoteItems([...])
+
+Merge remote items from upstream by updating existing items, adding new items, and deleting existing items. Folders are created first so that subsequent operations will succeed. Items have the same form as `getLocalItems()` above. Items that do not have an ID will have an ID generated for them. The results structure will contain this generated ID.
+
+````
+rootFolder.mergeRemoteItems([
+        {
+          id: 'f2398f23',
+          type: rootFolder.FOLDER,
+          title: 'Folder 1',
+          parent: '9f8237f928'
+        },
+        {
+          id: '9f8237f928',
+          type: rootFolder.FOLDER,
+          title: 'Folder 0',
+        }
+      ]).then(
+  function(items) {
+    console.log(items); // any generated IDs are filled in now
+    console.log("merge completed");
+  }
+);
+````

services/cloudsync/docs/architecture.rst

@@ -0,0 +1,54 @@
+.. _cloudsync_architecture:
+
+============
+Architecture
+============
+
+CloudSync offers functionality similar to Firefox Sync for data sources. Third-party addons
+(sync adapters) consume local data, send and receive updates from the cloud, and merge remote data.
+
+
+Files
+=====
+
+CloudSync.jsm
+    Main module; Includes other modules and exposes them.
+
+CloudSyncAdapters.jsm
+    Provides an API for addons to register themselves. Will be used to
+    list available adapters and to notify adapters when sync operations
+    are requested manually by the user.
+
+CloudSyncBookmarks.jsm
+    Provides operations for interacting with bookmarks.
+
+CloudSyncBookmarksFolderCache.jsm
+    Implements a cache used to store folder hierarchy for filtering bookmark events.
+
+CloudSyncEventSource.jsm
+    Implements an event emitter. Used to provide addEventListener and removeEventListener
+    for tabs and bookmarks.
+
+CloudSyncLocal.jsm
+    Provides information about the local device, such as name and a unique id.
+
+CloudSyncPlacesWrapper.jsm
+    Wraps parts of the Places API in promises. Some methods are implemented to be asynchronous
+    where they are not in the places API.
+
+CloudSyncTabs.jsm
+    Provides operations for fetching local tabs and for populating the about:sync-tabs page.
+
+
+Data Sources
+============
+
+CloudSync provides data for tabs and bookmarks. For tabs, local open pages can be enumerated and
+remote tabs can be merged for displaying in about:sync-tabs. For bookmarks, updates are tracked
+for a named folder (given by each adapter) and handled by callbacks registered using addEventListener,
+and remote changes can be merged into the local database.
+
+Versioning
+==========
+
+The API carries an integer version number (clouySync.version). Data records are versioned separately and individually.

services/cloudsync/docs/dataformat.rst

@@ -0,0 +1,77 @@
+.. _cloudsync_dataformat:
+
+===========
+Data Format
+===========
+
+All fields are required unless noted otherwise.
+
+Bookmarks
+=========
+
+Record
+------
+
+type:
+    record type; one of CloudSync.bookmarks.{BOOKMARK, FOLDER, SEPARATOR, QUERY, LIVEMARK}
+
+id:
+    GUID for this bookmark item
+
+parent:
+    id of parent folder
+
+index:
+    item index in parent folder; should be unique and contiguous, or they will be adjusted internally
+
+title:
+    bookmark or folder title; not meaningful for separators
+
+dateAdded:
+    timestamp (in milliseconds) for item added
+
+lastModified:
+    timestamp (in milliseconds) for last modification
+
+uri:
+    bookmark URI; not meaningful for folders or separators
+
+version:
+    data layout version
+
+Tabs
+====
+
+ClientRecord
+------------
+
+id:
+    GUID for this client
+
+name:
+    name for this client; not guaranteed to be unique
+
+tabs:
+    list of tabs open on this client; see TabRecord
+
+version:
+    data layout version
+
+
+TabRecord
+---------
+
+title:
+    name for this tab
+
+url:
+    URL for this tab; only one tab for each URL is stored
+
+icon:
+    favicon URL for this tab; optional
+
+lastUsed:
+    timetamp (in milliseconds) for last use
+
+version:
+    data layout version

services/cloudsync/docs/example.rst

@@ -0,0 +1,132 @@
+.. _cloudsync_example:
+
+=======
+Example
+=======
+
+.. code-block:: javascript
+
+	Cu.import("resource://gre/modules/CloudSync.jsm");
+
+	let HelloWorld = {
+	  onLoad: function() {
+	    let cloudSync = CloudSync();
+	    console.log("CLOUDSYNC -- hello world", cloudSync.local.id, cloudSync.local.name, cloudSync.adapters);
+	    cloudSync.adapters.register('helloworld', {});
+	    console.log("CLOUDSYNC -- " + JSON.stringify(cloudSync.adapters.getAdapterNames()));
+
+
+	    cloudSync.tabs.addEventListener("change", function() {
+	      console.log("tab change");
+	      cloudSync.tabs.getLocalTabs().then(
+	        function(records) {
+	          console.log(JSON.stringify(records));
+	        }
+	      );
+	    });
+
+	    cloudSync.tabs.getLocalTabs().then(
+	      function(records) {
+	        console.log(JSON.stringify(records));
+	      }
+	    );
+
+	    let remoteClient = {
+	      id: "001",
+	      name: "FakeClient",
+	    };
+	    let remoteTabs1 = [
+	      {url:"https://www.google.ca",title:"Google",icon:"https://www.google.ca/favicon.ico",lastUsed:Date.now()},
+	    ];
+	    let remoteTabs2 = [
+	      {url:"https://www.google.ca",title:"Google Canada",icon:"https://www.google.ca/favicon.ico",lastUsed:Date.now()},
+	      {url:"http://www.reddit.com",title:"Reddit",icon:"http://www.reddit.com/favicon.ico",lastUsed:Date.now()},
+	    ];
+	    cloudSync.tabs.mergeRemoteTabs(remoteClient, remoteTabs1).then(
+	      function() {
+	        return cloudSync.tabs.mergeRemoteTabs(remoteClient, remoteTabs2);
+	      }
+	    ).then(
+	      function() {
+	        return cloudSync.tabs.getRemoteTabs();
+	      }
+	    ).then(
+	      function(tabs) {
+	        console.log("remote tabs:", tabs);
+	      }
+	    );
+
+	    cloudSync.bookmarks.getRootFolder("Hello World").then(
+	      function(rootFolder) {
+	        console.log(rootFolder.name, rootFolder.id);
+	        rootFolder.addEventListener("add", function(guid) {
+	          console.log("CLOUDSYNC -- bookmark item added: " + guid);
+	          rootFolder.getLocalItemsById([guid]).then(
+	            function(items) {
+	              console.log("CLOUDSYNC -- items: " + JSON.stringify(items));
+	            }
+	          );
+	        });
+	        rootFolder.addEventListener("remove", function(guid) {
+	          console.log("CLOUDSYNC -- bookmark item removed: " + guid);
+	          rootFolder.getLocalItemsById([guid]).then(
+	            function(items) {
+	              console.log("CLOUDSYNC -- items: " + JSON.stringify(items));
+	            }
+	          );
+	        });
+	        rootFolder.addEventListener("change", function(guid) {
+	          console.log("CLOUDSYNC -- bookmark item changed: " + guid);
+	          rootFolder.getLocalItemsById([guid]).then(
+	            function(items) {
+	              console.log("CLOUDSYNC -- items: " + JSON.stringify(items));
+	            }
+	          );
+	        });
+	        rootFolder.addEventListener("move", function(guid) {
+	          console.log("CLOUDSYNC -- bookmark item moved: " + guid);
+	          rootFolder.getLocalItemsById([guid]).then(
+	            function(items) {
+	              console.log("CLOUDSYNC -- items: " + JSON.stringify(items));
+	            }
+	          );
+	        });
+
+	        function logLocalItems() {
+	          return rootFolder.getLocalItems().then(
+	            function(items) {
+	              console.log("CLOUDSYNC -- local items: " + JSON.stringify(items));
+	            }
+	          );
+	        }
+
+	        let items = [
+	            {"id":"9fdoci2KOME6","type":rootFolder.FOLDER,"parent":rootFolder.id,"title":"My Bookmarks 1"},
+	            {"id":"1fdoci2KOME5","type":rootFolder.FOLDER,"parent":rootFolder.id,"title":"My Bookmarks 2"},
+	            {"id":"G_UL4ZhOyX8m","type":rootFolder.BOOKMARK,"parent":"1fdoci2KOME5","title":"reddit: the front page of the internet","uri":"http://www.reddit.com/"},
+	          ];
+	        function mergeSomeItems() {
+	          return rootFolder.mergeRemoteItems(items);
+	        }
+
+	        logLocalItems().then(
+	          mergeSomeItems
+	        ).then(
+	          function(processedItems) {
+	            console.log("!!!", processedItems);
+	            console.log("merge complete");
+	          },
+	          function(error) {
+	            console.log("merge failed:", error);
+	          }
+	        ).then(
+	          logLocalItems
+	        );
+	      }
+	    );
+
+
+	  },
+	};
+
+	window.addEventListener("load", function(e) { HelloWorld.onLoad(e); }, false);

services/cloudsync/docs/index.rst

@@ -0,0 +1,19 @@
+.. _cloudsync:
+
+=====================
+CloudSync
+=====================
+
+CloudSync is a service that provides access to tabs and bookmarks data
+for third-party sync addons. Addons can read local bookmarks and tabs.
+Bookmarks and tab data can be merged from remote devices.
+
+Addons are responsible for maintaining an upstream representation, as
+well as sending and receiving data over the network.
+
+.. toctree::
+   :maxdepth: 1
+
+   architecture
+   dataformat
+   example

services/cloudsync/moz.build

@@ -0,0 +1,21 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+SPHINX_TREES['cloudsync'] = 'docs'
+
+EXTRA_JS_MODULES += [
+  'CloudSync.jsm',
+  'CloudSyncAdapters.jsm',
+  'CloudSyncBookmarks.jsm',
+  'CloudSyncBookmarksFolderCache.jsm',
+  'CloudSyncEventSource.jsm',
+  'CloudSyncLocal.jsm',
+  'CloudSyncPlacesWrapper.jsm',
+  'CloudSyncTabs.jsm',
+]
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini']
+BROWSER_CHROME_MANIFESTS += ['tests/mochitest/browser.ini']

services/cloudsync/tests/mochitest/browser.ini

@@ -0,0 +1,5 @@
+[DEFAULT]
+support-files=
+  other_window.html
+
+[browser_tabEvents.js]

services/cloudsync/tests/mochitest/browser_tabEvents.js

@@ -0,0 +1,79 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+function test() {
+
+  let local = {};
+
+  Components.utils.import("resource://gre/modules/CloudSync.jsm", local);
+  Components.utils.import("resource:///modules/sessionstore/TabStateFlusher.jsm", local);
+
+  let cloudSync = local.CloudSync();
+  let opentabs = [];
+
+  waitForExplicitFinish();
+
+  let testURL = "chrome://mochitests/content/browser/services/cloudsync/tests/mochitest/other_window.html";
+  let expected = [
+    testURL,
+    testURL+"?x=1",
+    testURL+"?x=%20a",
+    // testURL+"?x=å",
+  ];
+
+  let nevents = 0;
+  let nflushed = 0;
+  function handleTabChangeEvent () {
+    cloudSync.tabs.removeEventListener("change", handleTabChangeEvent);
+    ++ nevents;
+    info("tab change event " + nevents);
+    next();
+  }
+
+  function getLocalTabs() {
+    cloudSync.tabs.getLocalTabs().then(
+      function (tabs) {
+        for (let tab of tabs) {
+          ok(expected.indexOf(tab.url) >= 0, "found an expected tab");
+        }
+
+        is(tabs.length, expected.length, "found the right number of tabs");
+
+        opentabs.forEach(function (tab) {
+          gBrowser.removeTab(tab);
+        });
+
+        is(nevents, 1, "expected number of change events");
+
+        finish();
+      }
+    )
+  }
+
+  cloudSync.tabs.addEventListener("change", handleTabChangeEvent);
+
+  expected.forEach(function(url) {
+    let tab = gBrowser.addTab(url);
+
+    function flush() {
+      tab.linkedBrowser.removeEventListener("load", flush, true);
+      local.TabStateFlusher.flush(tab.linkedBrowser).then(() => {
+        ++ nflushed;
+        info("flushed " + nflushed);
+        next();
+      });
+    }
+
+    tab.linkedBrowser.addEventListener("load", flush, true);
+
+    opentabs.push(tab);
+  });
+
+  function next() {
+    if (nevents == 1 && nflushed == expected.length) {
+      getLocalTabs();
+    }
+  }
+
+}

services/cloudsync/tests/mochitest/other_window.html

@@ -0,0 +1,7 @@
+<!--
+     Any copyright is dedicated to the Public Domain.
+     http://creativecommons.org/publicdomain/zero/1.0/
+-->
+<!DOCTYPE HTML>
+<html>
+</html>

services/cloudsync/tests/xpcshell/head.js

@@ -0,0 +1,10 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+"use strict";
+
+(function initCloudSyncTestingInfrastructure () {
+  do_get_profile();
+}).call(this);

services/cloudsync/tests/xpcshell/test_bookmarks.js

@@ -0,0 +1,73 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/CloudSync.jsm");
+
+function run_test () {
+  run_next_test();
+}
+
+function cleanup () {
+
+}
+
+add_task(function* test_merge_bookmarks_flat () {
+  try {
+	  let rootFolder = yield CloudSync().bookmarks.getRootFolder("TEST");
+	  ok(rootFolder.id, "root folder id is ok");
+
+	  let items = [
+		{"id":"G_UL4ZhOyX8m","type":rootFolder.BOOKMARK,"title":"reddit: the front page of the internet 1","uri":"http://www.reddit.com",index:2},
+		{"id":"G_UL4ZhOyX8n","type":rootFolder.BOOKMARK,"title":"reddit: the front page of the internet 2","uri":"http://www.reddit.com?1",index:1},
+	  ];
+	  yield rootFolder.mergeRemoteItems(items);
+
+	  let localItems = yield rootFolder.getLocalItems();
+	  equal(Object.keys(localItems).length, items.length, "found merged items");
+  } finally {
+  	yield CloudSync().bookmarks.deleteRootFolder("TEST");
+  }
+});
+
+add_task(function* test_merge_bookmarks_in_folders () {
+  try {
+	  let rootFolder = yield CloudSync().bookmarks.getRootFolder("TEST");
+	  ok(rootFolder.id, "root folder id is ok");
+
+	  let items = [
+	  	{"id":"G_UL4ZhOyX8m","type":rootFolder.BOOKMARK,"title":"reddit: the front page of the internet 1","uri":"http://www.reddit.com",index:2},
+	    {"id":"G_UL4ZhOyX8n","type":rootFolder.BOOKMARK,parent:"G_UL4ZhOyX8x","title":"reddit: the front page of the internet 2","uri":"http://www.reddit.com/?a=å%20ä%20ö",index:1},
+	    {"id":"G_UL4ZhOyX8x","type":rootFolder.FOLDER},
+	  ];
+	  yield rootFolder.mergeRemoteItems(items);
+
+	  let localItems = yield rootFolder.getLocalItems();
+	  equal(localItems.length, items.length, "found merged items");
+
+	  localItems.forEach(function(item) {
+	    ok(item.id == "G_UL4ZhOyX8m" ||
+	       item.id == "G_UL4ZhOyX8n" ||
+	       item.id == "G_UL4ZhOyX8x");
+	    if (item.id == "G_UL4ZhOyX8n") {
+	    	equal(item.parent, "G_UL4ZhOyX8x")
+	    } else {
+	    	equal(item.parent, rootFolder.id);
+	    }
+	  });
+
+	  let folder = (yield rootFolder.getLocalItemsById(["G_UL4ZhOyX8x"]))[0];
+	  equal(folder.id, "G_UL4ZhOyX8x");
+	  equal(folder.type, rootFolder.FOLDER);
+
+	  let bookmark = (yield rootFolder.getLocalItemsById(["G_UL4ZhOyX8n"]))[0];
+	  equal(bookmark.id, "G_UL4ZhOyX8n");
+	  equal(bookmark.parent, "G_UL4ZhOyX8x");
+	  equal(bookmark.title, "reddit: the front page of the internet 2");
+	  equal(bookmark.index, 0);
+	  equal(bookmark.uri, "http://www.reddit.com/?a=%C3%A5%20%C3%A4%20%C3%B6");
+  } finally {
+	yield CloudSync().bookmarks.deleteRootFolder("TEST");
+  }
+});

services/cloudsync/tests/xpcshell/test_lazyload.js

@@ -0,0 +1,18 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/CloudSync.jsm");
+
+function run_test() {
+  run_next_test();
+}
+
+add_task(function test_lazyload() {
+  ok(!CloudSync.ready, "CloudSync.ready is false before CloudSync() invoked");
+  let cs1 = CloudSync();
+  ok(CloudSync.ready, "CloudSync.ready is true after CloudSync() invoked");
+  let cs2 = CloudSync();
+  ok(cs1 === cs2, "CloudSync() returns the same instance on multiple invocations");
+});

services/cloudsync/tests/xpcshell/test_module.js

@@ -0,0 +1,19 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/CloudSync.jsm");
+
+function run_test () {
+  run_next_test();
+}
+
+add_task(function test_module_load () {
+  ok(CloudSync);
+  let cloudSync = CloudSync();
+  ok(cloudSync.adapters);
+  ok(cloudSync.bookmarks);
+  ok(cloudSync.local);
+  ok(cloudSync.tabs);
+});

services/cloudsync/tests/xpcshell/test_tabs.js

@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/CloudSync.jsm");
+
+function run_test () {
+  run_next_test();
+}
+
+add_task(function* test_get_remote_tabs () {
+  let cloudSync = CloudSync();
+  let clients = yield cloudSync.tabs.getRemoteTabs();
+  equal(clients.length, 0);
+
+  yield cloudSync.tabs.mergeRemoteTabs({
+      id: "001",
+      name: "FakeClient",
+    },[
+      {url:"https://www.google.ca?a=å%20ä%20ö",title:"Google Canada",icon:"https://www.google.ca/favicon.ico",lastUsed:0},
+      {url:"http://www.reddit.com",title:"Reddit",icon:"http://www.reddit.com/favicon.ico",lastUsed:1},
+    ]);
+  ok(cloudSync.tabs.hasRemoteTabs());
+
+  clients = yield cloudSync.tabs.getRemoteTabs();
+  equal(clients.length, 1);
+  equal(clients[0].tabs.size, 2);
+});

services/cloudsync/tests/xpcshell/xpcshell.ini

@@ -0,0 +1,10 @@
+[DEFAULT]
+head = head.js
+tail =
+firefox-appdir = browser
+skip-if = toolkit == 'android'
+
+[test_module.js]
+[test_tabs.js]
+[test_bookmarks.js]
+[test_lazyload.js]

services/common/async.js

@@ -0,0 +1,220 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["Async"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+// Constants for makeSyncCallback, waitForSyncCallback.
+const CB_READY = {};
+const CB_COMPLETE = {};
+const CB_FAIL = {};
+
+const REASON_ERROR = Ci.mozIStorageStatementCallback.REASON_ERROR;
+
+Cu.import("resource://gre/modules/Services.jsm");
+
+/*
+ * Helpers for various async operations.
+ */
+this.Async = {
+
+  /**
+   * Execute an arbitrary number of asynchronous functions one after the
+   * other, passing the callback arguments on to the next one.  All functions
+   * must take a callback function as their last argument.  The 'this' object
+   * will be whatever chain()'s is.
+   *
+   * @usage this._chain = Async.chain;
+   *        this._chain(this.foo, this.bar, this.baz)(args, for, foo)
+   *
+   * This is equivalent to:
+   *
+   *   let self = this;
+   *   self.foo(args, for, foo, function (bars, args) {
+   *     self.bar(bars, args, function (baz, params) {
+   *       self.baz(baz, params);
+   *     });
+   *   });
+   */
+  chain: function chain() {
+    let funcs = Array.slice(arguments);
+    let thisObj = this;
+    return function callback() {
+      if (funcs.length) {
+        let args = Array.slice(arguments).concat(callback);
+        let f = funcs.shift();
+        f.apply(thisObj, args);
+      }
+    };
+  },
+
+  /**
+   * Helpers for making asynchronous calls within a synchronous API possible.
+   *
+   * If you value your sanity, do not look closely at the following functions.
+   */
+
+  /**
+   * Create a sync callback that remembers state, in particular whether it has
+   * been called.
+   * The returned callback can be called directly passing an optional arg which
+   * will be returned by waitForSyncCallback().  The callback also has a
+   * .throw() method, which takes an error object and will cause
+   * waitForSyncCallback to fail with the error object thrown as an exception
+   * (but note that the .throw method *does not* itself throw - it just causes
+   * the wait function to throw).
+   */
+  makeSyncCallback: function makeSyncCallback() {
+    // The main callback remembers the value it was passed, and that it got data.
+    let onComplete = function onComplete(data) {
+      onComplete.state = CB_COMPLETE;
+      onComplete.value = data;
+    };
+
+    // Initialize private callback data in preparation for being called.
+    onComplete.state = CB_READY;
+    onComplete.value = null;
+
+    // Allow an alternate callback to trigger an exception to be thrown.
+    onComplete.throw = function onComplete_throw(data) {
+      onComplete.state = CB_FAIL;
+      onComplete.value = data;
+    };
+
+    return onComplete;
+  },
+
+  /**
+   * Wait for a sync callback to finish.
+   */
+  waitForSyncCallback: function waitForSyncCallback(callback) {
+    // Grab the current thread so we can make it give up priority.
+    let thread = Cc["@mozilla.org/thread-manager;1"].getService().currentThread;
+
+    // Keep waiting until our callback is triggered (unless the app is quitting).
+    while (Async.checkAppReady() && callback.state == CB_READY) {
+      thread.processNextEvent(true);
+    }
+
+    // Reset the state of the callback to prepare for another call.
+    let state = callback.state;
+    callback.state = CB_READY;
+
+    // Throw the value the callback decided to fail with.
+    if (state == CB_FAIL) {
+      throw callback.value;
+    }
+
+    // Return the value passed to the callback.
+    return callback.value;
+  },
+
+  /**
+   * Check if the app is still ready (not quitting).
+   */
+  checkAppReady: function checkAppReady() {
+    // Watch for app-quit notification to stop any sync calls
+    Services.obs.addObserver(function onQuitApplication() {
+      Services.obs.removeObserver(onQuitApplication, "quit-application");
+      Async.checkAppReady = function() {
+        let exception = Components.Exception("App. Quitting", Cr.NS_ERROR_ABORT);
+        exception.appIsShuttingDown = true;
+        throw exception;
+      };
+    }, "quit-application", false);
+    // In the common case, checkAppReady just returns true
+    return (Async.checkAppReady = function() { return true; })();
+  },
+
+  /**
+   * Check if the passed exception is one raised by checkAppReady. Typically
+   * this will be used in exception handlers to allow such exceptions to
+   * make their way to the top frame and allow the app to actually terminate.
+   */
+  isShutdownException(exception) {
+    return exception && exception.appIsShuttingDown === true;
+  },
+
+  /**
+   * Return the two things you need to make an asynchronous call synchronous
+   * by spinning the event loop.
+   */
+  makeSpinningCallback: function makeSpinningCallback() {
+    let cb = Async.makeSyncCallback();
+    function callback(error, ret) {
+      if (error)
+        cb.throw(error);
+      else
+        cb(ret);
+    }
+    callback.wait = () => Async.waitForSyncCallback(cb);
+    return callback;
+  },
+
+  // Prototype for mozIStorageCallback, used in querySpinningly.
+  // This allows us to define the handle* functions just once rather
+  // than on every querySpinningly invocation.
+  _storageCallbackPrototype: {
+    results: null,
+
+    // These are set by queryAsync.
+    names: null,
+    syncCb: null,
+
+    handleResult: function handleResult(results) {
+      if (!this.names) {
+        return;
+      }
+      if (!this.results) {
+        this.results = [];
+      }
+      let row;
+      while ((row = results.getNextRow()) != null) {
+        let item = {};
+        for (let name of this.names) {
+          item[name] = row.getResultByName(name);
+        }
+        this.results.push(item);
+      }
+    },
+    handleError: function handleError(error) {
+      this.syncCb.throw(error);
+    },
+    handleCompletion: function handleCompletion(reason) {
+
+      // If we got an error, handleError will also have been called, so don't
+      // call the callback! We never cancel statements, so we don't need to
+      // address that quandary.
+      if (reason == REASON_ERROR)
+        return;
+
+      // If we were called with column names but didn't find any results,
+      // the calling code probably still expects an array as a return value.
+      if (this.names && !this.results) {
+        this.results = [];
+      }
+      this.syncCb(this.results);
+    }
+  },
+
+  querySpinningly: function querySpinningly(query, names) {
+    // 'Synchronously' asyncExecute, fetching all results by name.
+    let storageCallback = Object.create(Async._storageCallbackPrototype);
+    storageCallback.names = names;
+    storageCallback.syncCb = Async.makeSyncCallback();
+    query.executeAsync(storageCallback);
+    return Async.waitForSyncCallback(storageCallback.syncCb);
+  },
+
+  promiseSpinningly(promise) {
+    let cb = Async.makeSpinningCallback();
+    promise.then(result =>  {
+      cb(null, result);
+    }, err => {
+      cb(err || new Error("Promise rejected without explicit error"));
+    });
+    return cb.wait();
+  },
+};

services/common/blocklist-clients.js

@@ -0,0 +1,310 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["AddonBlocklistClient",
+                         "GfxBlocklistClient",
+                         "OneCRLBlocklistClient",
+                         "PluginBlocklistClient",
+                         "FILENAME_ADDONS_JSON",
+                         "FILENAME_GFX_JSON",
+                         "FILENAME_PLUGINS_JSON"];
+
+const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
+
+Cu.import("resource://gre/modules/Services.jsm");
+const { Task } = Cu.import("resource://gre/modules/Task.jsm");
+const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
+Cu.importGlobalProperties(["fetch"]);
+
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { KintoHttpClient } = Cu.import("resource://services-common/kinto-http-client.js");
+const { CanonicalJSON } = Components.utils.import("resource://gre/modules/CanonicalJSON.jsm");
+
+const PREF_SETTINGS_SERVER                   = "services.settings.server";
+const PREF_BLOCKLIST_BUCKET                  = "services.blocklist.bucket";
+const PREF_BLOCKLIST_ONECRL_COLLECTION       = "services.blocklist.onecrl.collection";
+const PREF_BLOCKLIST_ONECRL_CHECKED_SECONDS  = "services.blocklist.onecrl.checked";
+const PREF_BLOCKLIST_ADDONS_COLLECTION       = "services.blocklist.addons.collection";
+const PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS  = "services.blocklist.addons.checked";
+const PREF_BLOCKLIST_PLUGINS_COLLECTION      = "services.blocklist.plugins.collection";
+const PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS = "services.blocklist.plugins.checked";
+const PREF_BLOCKLIST_GFX_COLLECTION          = "services.blocklist.gfx.collection";
+const PREF_BLOCKLIST_GFX_CHECKED_SECONDS     = "services.blocklist.gfx.checked";
+const PREF_BLOCKLIST_ENFORCE_SIGNING         = "services.blocklist.signing.enforced";
+
+const INVALID_SIGNATURE = "Invalid content/signature";
+
+this.FILENAME_ADDONS_JSON  = "blocklist-addons.json";
+this.FILENAME_GFX_JSON     = "blocklist-gfx.json";
+this.FILENAME_PLUGINS_JSON = "blocklist-plugins.json";
+
+function mergeChanges(localRecords, changes) {
+  // Kinto.js adds attributes to local records that aren't present on server.
+  // (e.g. _status)
+  const stripPrivateProps = (obj) => {
+    return Object.keys(obj).reduce((current, key) => {
+      if (!key.startsWith("_")) {
+        current[key] = obj[key];
+      }
+      return current;
+    }, {});
+  };
+
+  const records = {};
+  // Local records by id.
+  localRecords.forEach((record) => records[record.id] = stripPrivateProps(record));
+  // All existing records are replaced by the version from the server.
+  changes.forEach((record) => records[record.id] = record);
+
+  return Object.values(records)
+    // Filter out deleted records.
+    .filter((record) => record.deleted != true)
+    // Sort list by record id.
+    .sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
+}
+
+
+function fetchCollectionMetadata(collection) {
+  const client = new KintoHttpClient(collection.api.remote);
+  return client.bucket(collection.bucket).collection(collection.name).getData()
+    .then(result => {
+      return result.signature;
+    });
+}
+
+function fetchRemoteCollection(collection) {
+  const client = new KintoHttpClient(collection.api.remote);
+  return client.bucket(collection.bucket)
+           .collection(collection.name)
+           .listRecords({sort: "id"});
+}
+
+/**
+ * Helper to instantiate a Kinto client based on preferences for remote server
+ * URL and bucket name. It uses the `FirefoxAdapter` which relies on SQLite to
+ * persist the local DB.
+ */
+function kintoClient() {
+  let base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+  let bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
+
+  let Kinto = loadKinto();
+
+  let FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+
+  let config = {
+    remote: base,
+    bucket: bucket,
+    adapter: FirefoxAdapter,
+  };
+
+  return new Kinto(config);
+}
+
+
+class BlocklistClient {
+
+  constructor(collectionName, lastCheckTimePref, processCallback, signerName) {
+    this.collectionName = collectionName;
+    this.lastCheckTimePref = lastCheckTimePref;
+    this.processCallback = processCallback;
+    this.signerName = signerName;
+  }
+
+  validateCollectionSignature(payload, collection, ignoreLocal) {
+    return Task.spawn((function* () {
+      // this is a content-signature field from an autograph response.
+      const {x5u, signature} = yield fetchCollectionMetadata(collection);
+      const certChain = yield fetch(x5u).then((res) => res.text());
+
+      const verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
+                         .createInstance(Ci.nsIContentSignatureVerifier);
+
+      let toSerialize;
+      if (ignoreLocal) {
+        toSerialize = {
+          last_modified: `${payload.last_modified}`,
+          data: payload.data
+        };
+      } else {
+        const localRecords = (yield collection.list()).data;
+        const records = mergeChanges(localRecords, payload.changes);
+        toSerialize = {
+          last_modified: `${payload.lastModified}`,
+          data: records
+        };
+      }
+
+      const serialized = CanonicalJSON.stringify(toSerialize);
+
+      if (verifier.verifyContentSignature(serialized, "p384ecdsa=" + signature,
+                                          certChain,
+                                          this.signerName)) {
+        // In case the hash is valid, apply the changes locally.
+        return payload;
+      }
+      throw new Error(INVALID_SIGNATURE);
+    }).bind(this));
+  }
+
+  /**
+   * Synchronize from Kinto server, if necessary.
+   *
+   * @param {int}  lastModified the lastModified date (on the server) for
+                                the remote collection.
+   * @param {Date} serverTime   the current date return by the server.
+   * @return {Promise}          which rejects on sync or process failure.
+   */
+  maybeSync(lastModified, serverTime) {
+    let db = kintoClient();
+    let opts = {};
+    let enforceCollectionSigning =
+      Services.prefs.getBoolPref(PREF_BLOCKLIST_ENFORCE_SIGNING);
+
+    // if there is a signerName and collection signing is enforced, add a
+    // hook for incoming changes that validates the signature
+    if (this.signerName && enforceCollectionSigning) {
+      opts.hooks = {
+        "incoming-changes": [this.validateCollectionSignature.bind(this)]
+      }
+    }
+
+    let collection = db.collection(this.collectionName, opts);
+
+    return Task.spawn((function* syncCollection() {
+      try {
+        yield collection.db.open();
+
+        let collectionLastModified = yield collection.db.getLastModified();
+        // If the data is up to date, there's no need to sync. We still need
+        // to record the fact that a check happened.
+        if (lastModified <= collectionLastModified) {
+          this.updateLastCheck(serverTime);
+          return;
+        }
+        // Fetch changes from server.
+        try {
+          let syncResult = yield collection.sync();
+          if (!syncResult.ok) {
+            throw new Error("Sync failed");
+          }
+        } catch (e) {
+          if (e.message == INVALID_SIGNATURE) {
+            // if sync fails with a signature error, it's likely that our
+            // local data has been modified in some way.
+            // We will attempt to fix this by retrieving the whole
+            // remote collection.
+            let payload = yield fetchRemoteCollection(collection);
+            yield this.validateCollectionSignature(payload, collection, true);
+            // if the signature is good (we haven't thrown), and the remote
+            // last_modified is newer than the local last_modified, replace the
+            // local data
+            const localLastModified = yield collection.db.getLastModified();
+            if (payload.last_modified >= localLastModified) {
+              yield collection.clear();
+              yield collection.loadDump(payload.data);
+            }
+          } else {
+            throw e;
+          }
+        }
+        // Read local collection of records.
+        let list = yield collection.list();
+
+        yield this.processCallback(list.data);
+
+        // Track last update.
+        this.updateLastCheck(serverTime);
+      } finally {
+        collection.db.close();
+      }
+    }).bind(this));
+  }
+
+  /**
+   * Save last time server was checked in users prefs.
+   *
+   * @param {Date} serverTime   the current date return by server.
+   */
+  updateLastCheck(serverTime) {
+    let checkedServerTimeInSeconds = Math.round(serverTime / 1000);
+    Services.prefs.setIntPref(this.lastCheckTimePref, checkedServerTimeInSeconds);
+  }
+}
+
+/**
+ * Revoke the appropriate certificates based on the records from the blocklist.
+ *
+ * @param {Object} records   current records in the local db.
+ */
+function* updateCertBlocklist(records) {
+  let certList = Cc["@mozilla.org/security/certblocklist;1"]
+                   .getService(Ci.nsICertBlocklist);
+  for (let item of records) {
+    try {
+      if (item.issuerName && item.serialNumber) {
+        certList.revokeCertByIssuerAndSerial(item.issuerName,
+                                            item.serialNumber);
+      } else if (item.subject && item.pubKeyHash) {
+        certList.revokeCertBySubjectAndPubKey(item.subject,
+                                              item.pubKeyHash);
+      }
+    } catch (e) {
+      // prevent errors relating to individual blocklist entries from
+      // causing sync to fail. At some point in the future, we may want to
+      // accumulate telemetry on these failures.
+      Cu.reportError(e);
+    }
+  }
+  certList.saveEntries();
+}
+
+/**
+ * Write list of records into JSON file, and notify nsBlocklistService.
+ *
+ * @param {String} filename  path relative to profile dir.
+ * @param {Object} records   current records in the local db.
+ */
+function* updateJSONBlocklist(filename, records) {
+  // Write JSON dump for synchronous load at startup.
+  const path = OS.Path.join(OS.Constants.Path.profileDir, filename);
+  const serialized = JSON.stringify({data: records}, null, 2);
+  try {
+    yield OS.File.writeAtomic(path, serialized, {tmpPath: path + ".tmp"});
+
+    // Notify change to `nsBlocklistService`
+    const eventData = {filename: filename};
+    Services.cpmm.sendAsyncMessage("Blocklist:reload-from-disk", eventData);
+  } catch(e) {
+    Cu.reportError(e);
+  }
+}
+
+
+this.OneCRLBlocklistClient = new BlocklistClient(
+  Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION),
+  PREF_BLOCKLIST_ONECRL_CHECKED_SECONDS,
+  updateCertBlocklist,
+  "onecrl.content-signature.mozilla.org"
+);
+
+this.AddonBlocklistClient = new BlocklistClient(
+  Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS_COLLECTION),
+  PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS,
+  updateJSONBlocklist.bind(undefined, FILENAME_ADDONS_JSON)
+);
+
+this.GfxBlocklistClient = new BlocklistClient(
+  Services.prefs.getCharPref(PREF_BLOCKLIST_GFX_COLLECTION),
+  PREF_BLOCKLIST_GFX_CHECKED_SECONDS,
+  updateJSONBlocklist.bind(undefined, FILENAME_GFX_JSON)
+);
+
+this.PluginBlocklistClient = new BlocklistClient(
+  Services.prefs.getCharPref(PREF_BLOCKLIST_PLUGINS_COLLECTION),
+  PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS,
+  updateJSONBlocklist.bind(undefined, FILENAME_PLUGINS_JSON)
+);

services/common/blocklist-updater.js

@@ -0,0 +1,117 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["checkVersions", "addTestBlocklistClient"];
+
+const { classes: Cc, Constructor: CC, interfaces: Ci, utils: Cu } = Components;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.importGlobalProperties(['fetch']);
+const BlocklistClients = Cu.import("resource://services-common/blocklist-clients.js", {});
+
+const PREF_SETTINGS_SERVER              = "services.settings.server";
+const PREF_BLOCKLIST_CHANGES_PATH       = "services.blocklist.changes.path";
+const PREF_BLOCKLIST_BUCKET             = "services.blocklist.bucket";
+const PREF_BLOCKLIST_LAST_UPDATE        = "services.blocklist.last_update_seconds";
+const PREF_BLOCKLIST_LAST_ETAG          = "services.blocklist.last_etag";
+const PREF_BLOCKLIST_CLOCK_SKEW_SECONDS = "services.blocklist.clock_skew_seconds";
+
+
+const gBlocklistClients = {
+  [BlocklistClients.OneCRLBlocklistClient.collectionName]: BlocklistClients.OneCRLBlocklistClient,
+  [BlocklistClients.AddonBlocklistClient.collectionName]: BlocklistClients.AddonBlocklistClient,
+  [BlocklistClients.GfxBlocklistClient.collectionName]: BlocklistClients.GfxBlocklistClient,
+  [BlocklistClients.PluginBlocklistClient.collectionName]: BlocklistClients.PluginBlocklistClient
+};
+
+// Add a blocklist client for testing purposes. Do not use for any other purpose
+this.addTestBlocklistClient = (name, client) => { gBlocklistClients[name] = client; }
+
+// This is called by the ping mechanism.
+// returns a promise that rejects if something goes wrong
+this.checkVersions = function() {
+  return Task.spawn(function* syncClients() {
+    // Fetch a versionInfo object that looks like:
+    // {"data":[
+    //   {
+    //     "host":"kinto-ota.dev.mozaws.net",
+    //     "last_modified":1450717104423,
+    //     "bucket":"blocklists",
+    //     "collection":"certificates"
+    //    }]}
+    // Right now, we only use the collection name and the last modified info
+    let kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+    let changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_BLOCKLIST_CHANGES_PATH);
+    let blocklistsBucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
+
+    // Use ETag to obtain a `304 Not modified` when no change occurred.
+    const headers = {};
+    if (Services.prefs.prefHasUserValue(PREF_BLOCKLIST_LAST_ETAG)) {
+      const lastEtag = Services.prefs.getCharPref(PREF_BLOCKLIST_LAST_ETAG);
+      if (lastEtag) {
+        headers["If-None-Match"] = lastEtag;
+      }
+    }
+
+    let response = yield fetch(changesEndpoint, {headers});
+
+    let versionInfo;
+    // No changes since last time. Go on with empty list of changes.
+    if (response.status == 304) {
+      versionInfo = {data: []};
+    } else {
+      versionInfo = yield response.json();
+    }
+
+    // If the server is failing, the JSON response might not contain the
+    // expected data (e.g. error response - Bug 1259145)
+    if (!versionInfo.hasOwnProperty("data")) {
+      throw new Error("Polling for changes failed.");
+    }
+
+    // Record new update time and the difference between local and server time
+    let serverTimeMillis = Date.parse(response.headers.get("Date"));
+
+    // negative clockDifference means local time is behind server time
+    // by the absolute of that value in seconds (positive means it's ahead)
+    let clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
+    Services.prefs.setIntPref(PREF_BLOCKLIST_CLOCK_SKEW_SECONDS, clockDifference);
+    Services.prefs.setIntPref(PREF_BLOCKLIST_LAST_UPDATE, serverTimeMillis / 1000);
+
+    let firstError;
+    for (let collectionInfo of versionInfo.data) {
+      // Skip changes that don't concern configured blocklist bucket.
+      if (collectionInfo.bucket != blocklistsBucket) {
+        continue;
+      }
+
+      let collection = collectionInfo.collection;
+      let client = gBlocklistClients[collection];
+      if (client && client.maybeSync) {
+        let lastModified = 0;
+        if (collectionInfo.last_modified) {
+          lastModified = collectionInfo.last_modified;
+        }
+        try {
+          yield client.maybeSync(lastModified, serverTimeMillis);
+        } catch (e) {
+          if (!firstError) {
+            firstError = e;
+          }
+        }
+      }
+    }
+    if (firstError) {
+      // cause the promise to reject by throwing the first observed error
+      throw firstError;
+    }
+
+    // Save current Etag for next poll.
+    if (response.headers.has("ETag")) {
+      const currentEtag = response.headers.get("ETag");
+      Services.prefs.setCharPref(PREF_BLOCKLIST_LAST_ETAG, currentEtag);
+    }
+  });
+};

services/common/hawkclient.js

@@ -0,0 +1,346 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/*
+ * HAWK is an HTTP authentication scheme using a message authentication code
+ * (MAC) algorithm to provide partial HTTP request cryptographic verification.
+ *
+ * For details, see: https://github.com/hueniverse/hawk
+ *
+ * With HAWK, it is essential that the clocks on clients and server not have an
+ * absolute delta of greater than one minute, as the HAWK protocol uses
+ * timestamps to reduce the possibility of replay attacks.  However, it is
+ * likely that some clients' clocks will be more than a little off, especially
+ * in mobile devices, which would break HAWK-based services (like sync and
+ * firefox accounts) for those clients.
+ *
+ * This library provides a stateful HAWK client that calculates (roughly) the
+ * clock delta on the client vs the server.  The library provides an interface
+ * for deriving HAWK credentials and making HAWK-authenticated REST requests to
+ * a single remote server.  Therefore, callers who want to interact with
+ * multiple HAWK services should instantiate one HawkClient per service.
+ */
+
+this.EXPORTED_SYMBOLS = ["HawkClient"];
+
+var {interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-common/hawkrequest.js");
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+
+// log.appender.dump should be one of "Fatal", "Error", "Warn", "Info", "Config",
+// "Debug", "Trace" or "All". If none is specified, "Error" will be used by
+// default.
+// Note however that Sync will also add this log to *its* DumpAppender, so
+// in a Sync context it shouldn't be necessary to adjust this - however, that
+// also means error logs are likely to be dump'd twice but that's OK.
+const PREF_LOG_LEVEL = "services.common.hawk.log.appender.dump";
+
+// A pref that can be set so "sensitive" information (eg, personally
+// identifiable info, credentials, etc) will be logged.
+const PREF_LOG_SENSITIVE_DETAILS = "services.common.hawk.log.sensitive";
+
+XPCOMUtils.defineLazyGetter(this, "log", function() {
+  let log = Log.repository.getLogger("Hawk");
+  // We set the log itself to "debug" and set the level from the preference to
+  // the appender.  This allows other things to send the logs to different
+  // appenders, while still allowing the pref to control what is seen via dump()
+  log.level = Log.Level.Debug;
+  let appender = new Log.DumpAppender();
+  log.addAppender(appender);
+  appender.level = Log.Level.Error;
+  try {
+    let level =
+      Services.prefs.getPrefType(PREF_LOG_LEVEL) == Ci.nsIPrefBranch.PREF_STRING
+      && Services.prefs.getCharPref(PREF_LOG_LEVEL);
+    appender.level = Log.Level[level] || Log.Level.Error;
+  } catch (e) {
+    log.error(e);
+  }
+
+  return log;
+});
+
+// A boolean to indicate if personally identifiable information (or anything
+// else sensitive, such as credentials) should be logged.
+XPCOMUtils.defineLazyGetter(this, 'logPII', function() {
+  try {
+    return Services.prefs.getBoolPref(PREF_LOG_SENSITIVE_DETAILS);
+  } catch (_) {
+    return false;
+  }
+});
+
+/*
+ * A general purpose client for making HAWK authenticated requests to a single
+ * host.  Keeps track of the clock offset between the client and the host for
+ * computation of the timestamp in the HAWK Authorization header.
+ *
+ * Clients should create one HawkClient object per each server they wish to
+ * interact with.
+ *
+ * @param host
+ *        The url of the host
+ */
+this.HawkClient = function(host) {
+  this.host = host;
+
+  // Clock offset in milliseconds between our client's clock and the date
+  // reported in responses from our host.
+  this._localtimeOffsetMsec = 0;
+}
+
+this.HawkClient.prototype = {
+
+  /*
+   * A boolean for feature detection.
+   */
+  willUTF8EncodeRequests: HAWKAuthenticatedRESTRequest.prototype.willUTF8EncodeObjectRequests,
+
+  /*
+   * Construct an error message for a response.  Private.
+   *
+   * @param restResponse
+   *        A RESTResponse object from a RESTRequest
+   *
+   * @param error
+   *        A string or object describing the error
+   */
+  _constructError: function(restResponse, error) {
+    let errorObj = {
+      error: error,
+      // This object is likely to be JSON.stringify'd, but neither Error()
+      // objects nor Components.Exception objects do the right thing there,
+      // so we add a new element which is simply the .toString() version of
+      // the error object, so it does appear in JSON'd values.
+      errorString: error.toString(),
+      message: restResponse.statusText,
+      code: restResponse.status,
+      errno: restResponse.status,
+      toString() {
+        return this.code + ": " + this.message;
+      },
+    };
+    let retryAfter = restResponse.headers && restResponse.headers["retry-after"];
+    retryAfter = retryAfter ? parseInt(retryAfter) : retryAfter;
+    if (retryAfter) {
+      errorObj.retryAfter = retryAfter;
+      // and notify observers of the retry interval
+      if (this.observerPrefix) {
+        Observers.notify(this.observerPrefix + ":backoff:interval", retryAfter);
+      }
+    }
+    return errorObj;
+  },
+
+  /*
+   *
+   * Update clock offset by determining difference from date gives in the (RFC
+   * 1123) Date header of a server response.  Because HAWK tolerates a window
+   * of one minute of clock skew (so two minutes total since the skew can be
+   * positive or negative), the simple method of calculating offset here is
+   * probably good enough.  We keep the value in milliseconds to make life
+   * easier, even though the value will not have millisecond accuracy.
+   *
+   * @param dateString
+   *        An RFC 1123 date string (e.g., "Mon, 13 Jan 2014 21:45:06 GMT")
+   *
+   * For HAWK clock skew and replay protection, see
+   * https://github.com/hueniverse/hawk#replay-protection
+   */
+  _updateClockOffset: function(dateString) {
+    try {
+      let serverDateMsec = Date.parse(dateString);
+      this._localtimeOffsetMsec = serverDateMsec - this.now();
+      log.debug("Clock offset vs " + this.host + ": " + this._localtimeOffsetMsec);
+    } catch(err) {
+      log.warn("Bad date header in server response: " + dateString);
+    }
+  },
+
+  /*
+   * Get the current clock offset in milliseconds.
+   *
+   * The offset is the number of milliseconds that must be added to the client
+   * clock to make it equal to the server clock.  For example, if the client is
+   * five minutes ahead of the server, the localtimeOffsetMsec will be -300000.
+   */
+  get localtimeOffsetMsec() {
+    return this._localtimeOffsetMsec;
+  },
+
+  /*
+   * return current time in milliseconds
+   */
+  now: function() {
+    return Date.now();
+  },
+
+  /* A general method for sending raw RESTRequest calls authorized using HAWK
+   *
+   * @param path
+   *        API endpoint path
+   * @param method
+   *        The HTTP request method
+   * @param credentials
+   *        Hawk credentials
+   * @param payloadObj
+   *        An object that can be encodable as JSON as the payload of the
+   *        request
+   * @param extraHeaders
+   *        An object with header/value pairs to send with the request.
+   * @return Promise
+   *        Returns a promise that resolves to the response of the API call,
+   *        or is rejected with an error.  If the server response can be parsed
+   *        as JSON and contains an 'error' property, the promise will be
+   *        rejected with this JSON-parsed response.
+   */
+  request: function(path, method, credentials=null, payloadObj={}, extraHeaders = {},
+                    retryOK=true) {
+    method = method.toLowerCase();
+
+    let deferred = Promise.defer();
+    let uri = this.host + path;
+    let self = this;
+
+    function _onComplete(error) {
+      // |error| can be either a normal caught error or an explicitly created
+      // Components.Exception() error. Log it now as it might not end up
+      // correctly in the logs by the time it's passed through _constructError.
+      if (error) {
+        log.warn("hawk request error", error);
+      }
+      // If there's no response there's nothing else to do.
+      if (!this.response) {
+        deferred.reject(error);
+        return;
+      }
+      let restResponse = this.response;
+      let status = restResponse.status;
+
+      log.debug("(Response) " + path + ": code: " + status +
+                " - Status text: " + restResponse.statusText);
+      if (logPII) {
+        log.debug("Response text: " + restResponse.body);
+      }
+
+      // All responses may have backoff headers, which are a server-side safety
+      // valve to allow slowing down clients without hurting performance.
+      self._maybeNotifyBackoff(restResponse, "x-weave-backoff");
+      self._maybeNotifyBackoff(restResponse, "x-backoff");
+
+      if (error) {
+        // When things really blow up, reconstruct an error object that follows
+        // the general format of the server on error responses.
+        return deferred.reject(self._constructError(restResponse, error));
+      }
+
+      self._updateClockOffset(restResponse.headers["date"]);
+
+      if (status === 401 && retryOK && !("retry-after" in restResponse.headers)) {
+        // Retry once if we were rejected due to a bad timestamp.
+        // Clock offset is adjusted already in the top of this function.
+        log.debug("Received 401 for " + path + ": retrying");
+        return deferred.resolve(
+            self.request(path, method, credentials, payloadObj, extraHeaders, false));
+      }
+
+      // If the server returned a json error message, use it in the rejection
+      // of the promise.
+      //
+      // In the case of a 401, in which we are probably being rejected for a
+      // bad timestamp, retry exactly once, during which time clock offset will
+      // be adjusted.
+
+      let jsonResponse = {};
+      try {
+        jsonResponse = JSON.parse(restResponse.body);
+      } catch(notJSON) {}
+
+      let okResponse = (200 <= status && status < 300);
+      if (!okResponse || jsonResponse.error) {
+        if (jsonResponse.error) {
+          return deferred.reject(jsonResponse);
+        }
+        return deferred.reject(self._constructError(restResponse, "Request failed"));
+      }
+      // It's up to the caller to know how to decode the response.
+      // We just return the whole response.
+      deferred.resolve(this.response);
+    };
+
+    function onComplete(error) {
+      try {
+        // |this| is the RESTRequest object and we need to ensure _onComplete
+        // gets the same one.
+        _onComplete.call(this, error);
+      } catch (ex) {
+        log.error("Unhandled exception processing response", ex);
+        deferred.reject(ex);
+      }
+    }
+
+    let extra = {
+      now: this.now(),
+      localtimeOffsetMsec: this.localtimeOffsetMsec,
+      headers: extraHeaders
+    };
+
+    let request = this.newHAWKAuthenticatedRESTRequest(uri, credentials, extra);
+    try {
+      if (method == "post" || method == "put" || method == "patch") {
+        request[method](payloadObj, onComplete);
+      } else {
+        request[method](onComplete);
+      }
+    } catch (ex) {
+      log.error("Failed to make hawk request", ex);
+      deferred.reject(ex);
+    }
+
+    return deferred.promise;
+  },
+
+  /*
+   * The prefix used for all notifications sent by this module.  This
+   * allows the handler of notifications to be sure they are handling
+   * notifications for the service they expect.
+   *
+   * If not set, no notifications will be sent.
+   */
+  observerPrefix: null,
+
+  // Given an optional header value, notify that a backoff has been requested.
+  _maybeNotifyBackoff: function (response, headerName) {
+    if (!this.observerPrefix || !response.headers) {
+      return;
+    }
+    let headerVal = response.headers[headerName];
+    if (!headerVal) {
+      return;
+    }
+    let backoffInterval;
+    try {
+      backoffInterval = parseInt(headerVal, 10);
+    } catch (ex) {
+      log.error("hawkclient response had invalid backoff value in '" +
+                headerName + "' header: " + headerVal);
+      return;
+    }
+    Observers.notify(this.observerPrefix + ":backoff:interval", backoffInterval);
+  },
+
+  // override points for testing.
+  newHAWKAuthenticatedRESTRequest: function(uri, credentials, extra) {
+    return new HAWKAuthenticatedRESTRequest(uri, credentials, extra);
+  },
+
+}

services/common/hawkrequest.js

@@ -0,0 +1,198 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+this.EXPORTED_SYMBOLS = [
+  "HAWKAuthenticatedRESTRequest",
+  "deriveHawkCredentials"
+];
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://gre/modules/Credentials.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "CryptoUtils",
+                                  "resource://services-crypto/utils.js");
+
+const Prefs = new Preferences("services.common.rest.");
+
+/**
+ * Single-use HAWK-authenticated HTTP requests to RESTish resources.
+ *
+ * @param uri
+ *        (String) URI for the RESTRequest constructor
+ *
+ * @param credentials
+ *        (Object) Optional credentials for computing HAWK authentication
+ *        header.
+ *
+ * @param payloadObj
+ *        (Object) Optional object to be converted to JSON payload
+ *
+ * @param extra
+ *        (Object) Optional extra params for HAWK header computation.
+ *        Valid properties are:
+ *
+ *          now:                 <current time in milliseconds>,
+ *          localtimeOffsetMsec: <local clock offset vs server>,
+ *          headers:             <An object with header/value pairs to be sent
+ *                                as headers on the request>
+ *
+ * extra.localtimeOffsetMsec is the value in milliseconds that must be added to
+ * the local clock to make it agree with the server's clock.  For instance, if
+ * the local clock is two minutes ahead of the server, the time offset in
+ * milliseconds will be -120000.
+ */
+
+this.HAWKAuthenticatedRESTRequest =
+ function HawkAuthenticatedRESTRequest(uri, credentials, extra={}) {
+  RESTRequest.call(this, uri);
+
+  this.credentials = credentials;
+  this.now = extra.now || Date.now();
+  this.localtimeOffsetMsec = extra.localtimeOffsetMsec || 0;
+  this._log.trace("local time, offset: " + this.now + ", " + (this.localtimeOffsetMsec));
+  this.extraHeaders = extra.headers || {};
+
+  // Expose for testing
+  this._intl = getIntl();
+};
+HAWKAuthenticatedRESTRequest.prototype = {
+  __proto__: RESTRequest.prototype,
+
+  dispatch: function dispatch(method, data, onComplete, onProgress) {
+    let contentType = "text/plain";
+    if (method == "POST" || method == "PUT" || method == "PATCH") {
+      contentType = "application/json";
+    }
+    if (this.credentials) {
+      let options = {
+        now: this.now,
+        localtimeOffsetMsec: this.localtimeOffsetMsec,
+        credentials: this.credentials,
+        payload: data && JSON.stringify(data) || "",
+        contentType: contentType,
+      };
+      let header = CryptoUtils.computeHAWK(this.uri, method, options);
+      this.setHeader("Authorization", header.field);
+      this._log.trace("hawk auth header: " + header.field);
+    }
+
+    for (let header in this.extraHeaders) {
+      this.setHeader(header, this.extraHeaders[header]);
+    }
+
+    this.setHeader("Content-Type", contentType);
+
+    this.setHeader("Accept-Language", this._intl.accept_languages);
+
+    return RESTRequest.prototype.dispatch.call(
+      this, method, data, onComplete, onProgress
+    );
+  }
+};
+
+
+/**
+  * Generic function to derive Hawk credentials.
+  *
+  * Hawk credentials are derived using shared secrets, which depend on the token
+  * in use.
+  *
+  * @param tokenHex
+  *        The current session token encoded in hex
+  * @param context
+  *        A context for the credentials. A protocol version will be prepended
+  *        to the context, see Credentials.keyWord for more information.
+  * @param size
+  *        The size in bytes of the expected derived buffer,
+  *        defaults to 3 * 32.
+  * @return credentials
+  *        Returns an object:
+  *        {
+  *          algorithm: sha256
+  *          id: the Hawk id (from the first 32 bytes derived)
+  *          key: the Hawk key (from bytes 32 to 64)
+  *          extra: size - 64 extra bytes (if size > 64)
+  *        }
+  */
+this.deriveHawkCredentials = function deriveHawkCredentials(tokenHex,
+                                                            context,
+                                                            size = 96,
+                                                            hexKey = false) {
+  let token = CommonUtils.hexToBytes(tokenHex);
+  let out = CryptoUtils.hkdf(token, undefined, Credentials.keyWord(context), size);
+
+  let result = {
+    algorithm: "sha256",
+    key: hexKey ? CommonUtils.bytesAsHex(out.slice(32, 64)) : out.slice(32, 64),
+    id: CommonUtils.bytesAsHex(out.slice(0, 32))
+  };
+  if (size > 64) {
+    result.extra = out.slice(64);
+  }
+
+  return result;
+}
+
+// With hawk request, we send the user's accepted-languages with each request.
+// To keep the number of times we read this pref at a minimum, maintain the
+// preference in a stateful object that notices and updates itself when the
+// pref is changed.
+this.Intl = function Intl() {
+  // We won't actually query the pref until the first time we need it
+  this._accepted = "";
+  this._everRead = false;
+  this._log = Log.repository.getLogger("Services.common.RESTRequest");
+  this._log.level = Log.Level[Prefs.get("log.logger.rest.request")];
+  this.init();
+};
+
+this.Intl.prototype = {
+  init: function() {
+    Services.prefs.addObserver("intl.accept_languages", this, false);
+  },
+
+  uninit: function() {
+    Services.prefs.removeObserver("intl.accept_languages", this);
+  },
+
+  observe: function(subject, topic, data) {
+    this.readPref();
+  },
+
+  readPref: function() {
+    this._everRead = true;
+    try {
+      this._accepted = Services.prefs.getComplexValue(
+        "intl.accept_languages", Ci.nsIPrefLocalizedString).data;
+    } catch (err) {
+      this._log.error("Error reading intl.accept_languages pref", err);
+    }
+  },
+
+  get accept_languages() {
+    if (!this._everRead) {
+      this.readPref();
+    }
+    return this._accepted;
+  },
+};
+
+// Singleton getter for Intl, creating an instance only when we first need it.
+var intl = null;
+function getIntl() {
+  if (!intl) {
+    intl = new Intl();
+  }
+  return intl;
+}
+

services/common/logmanager.js

@@ -0,0 +1,331 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict;"
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Services",
+  "resource://gre/modules/Services.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "FileUtils",
+  "resource://gre/modules/FileUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Log",
+  "resource://gre/modules/Log.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "OS",
+  "resource://gre/modules/osfile.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "CommonUtils",
+  "resource://services-common/utils.js");
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+
+this.EXPORTED_SYMBOLS = [
+  "LogManager",
+];
+
+const DEFAULT_MAX_ERROR_AGE = 20 * 24 * 60 * 60; // 20 days
+
+// "shared" logs (ie, where the same log name is used by multiple LogManager
+// instances) are a fact of life here - eg, FirefoxAccounts logs are used by
+// both Sync and Reading List.
+// However, different instances have different pref branches, so we need to
+// handle when one pref branch says "Debug" and the other says "Error"
+// So we (a) keep singleton console and dump appenders and (b) keep track
+// of the minimum (ie, most verbose) level and use that.
+// This avoids (a) the most recent setter winning (as that is indeterminate)
+// and (b) multiple dump/console appenders being added to the same log multiple
+// times, which would cause messages to appear twice.
+
+// Singletons used by each instance.
+var formatter;
+var dumpAppender;
+var consoleAppender;
+
+// A set of all preference roots used by all instances.
+var allBranches = new Set();
+
+// A storage appender that is flushable to a file on disk.  Policies for
+// when to flush, to what file, log rotation etc are up to the consumer
+// (although it does maintain a .sawError property to help the consumer decide
+// based on its policies)
+function FlushableStorageAppender(formatter) {
+  Log.StorageStreamAppender.call(this, formatter);
+  this.sawError = false;
+}
+
+FlushableStorageAppender.prototype = {
+  __proto__: Log.StorageStreamAppender.prototype,
+
+  append(message) {
+    if (message.level >= Log.Level.Error) {
+      this.sawError = true;
+    }
+    Log.StorageStreamAppender.prototype.append.call(this, message);
+  },
+
+  reset() {
+    Log.StorageStreamAppender.prototype.reset.call(this);
+    this.sawError = false;
+  },
+
+  // Flush the current stream to a file. Somewhat counter-intuitively, you
+  // must pass a log which will be written to with details of the operation.
+  flushToFile: Task.async(function* (subdirArray, filename, log) {
+    let inStream = this.getInputStream();
+    this.reset();
+    if (!inStream) {
+      log.debug("Failed to flush log to a file - no input stream");
+      return;
+    }
+    log.debug("Flushing file log");
+    log.trace("Beginning stream copy to " + filename + ": " + Date.now());
+    try {
+      yield this._copyStreamToFile(inStream, subdirArray, filename, log);
+      log.trace("onCopyComplete", Date.now());
+    } catch (ex) {
+      log.error("Failed to copy log stream to file", ex);
+    }
+  }),
+
+  /**
+   * Copy an input stream to the named file, doing everything off the main
+   * thread.
+   * subDirArray is an array of path components, relative to the profile
+   * directory, where the file will be created.
+   * outputFileName is the filename to create.
+   * Returns a promise that is resolved on completion or rejected with an error.
+   */
+  _copyStreamToFile: Task.async(function* (inputStream, subdirArray, outputFileName, log) {
+    // The log data could be large, so we don't want to pass it all in a single
+    // message, so use BUFFER_SIZE chunks.
+    const BUFFER_SIZE = 8192;
+
+    // get a binary stream
+    let binaryStream = Cc["@mozilla.org/binaryinputstream;1"].createInstance(Ci.nsIBinaryInputStream);
+    binaryStream.setInputStream(inputStream);
+
+    let outputDirectory = OS.Path.join(OS.Constants.Path.profileDir, ...subdirArray);
+    yield OS.File.makeDir(outputDirectory, { ignoreExisting: true, from: OS.Constants.Path.profileDir });
+    let fullOutputFileName = OS.Path.join(outputDirectory, outputFileName);
+    let output = yield OS.File.open(fullOutputFileName, { write: true} );
+    try {
+      while (true) {
+        let available = binaryStream.available();
+        if (!available) {
+          break;
+        }
+        let chunk = binaryStream.readByteArray(Math.min(available, BUFFER_SIZE));
+        yield output.write(new Uint8Array(chunk));
+      }
+    } finally {
+      try {
+        binaryStream.close(); // inputStream is closed by the binaryStream
+        yield output.close();
+      } catch (ex) {
+        log.error("Failed to close the input stream", ex);
+      }
+    }
+    log.trace("finished copy to", fullOutputFileName);
+  }),
+}
+
+// The public LogManager object.
+function LogManager(prefRoot, logNames, logFilePrefix) {
+  this._prefObservers = [];
+  this.init(prefRoot, logNames, logFilePrefix);
+}
+
+LogManager.prototype = {
+  _cleaningUpFileLogs: false,
+
+  init(prefRoot, logNames, logFilePrefix) {
+    if (prefRoot instanceof Preferences) {
+      this._prefs = prefRoot;
+    } else {
+      this._prefs = new Preferences(prefRoot);
+    }
+
+    this.logFilePrefix = logFilePrefix;
+    if (!formatter) {
+      // Create a formatter and various appenders to attach to the logs.
+      formatter = new Log.BasicFormatter();
+      consoleAppender = new Log.ConsoleAppender(formatter);
+      dumpAppender = new Log.DumpAppender(formatter);
+    }
+
+    allBranches.add(this._prefs._branchStr);
+    // We create a preference observer for all our prefs so they are magically
+    // reflected if the pref changes after creation.
+    let setupAppender = (appender, prefName, defaultLevel, findSmallest = false) => {
+      let observer = newVal => {
+        let level = Log.Level[newVal] || defaultLevel;
+        if (findSmallest) {
+          // As some of our appenders have global impact (ie, there is only one
+          // place 'dump' goes to), we need to find the smallest value from all
+          // prefs controlling this appender.
+          // For example, if consumerA has dump=Debug then consumerB sets
+          // dump=Error, we need to keep dump=Debug so consumerA is respected.
+          for (let branch of allBranches) {
+            let lookPrefBranch = new Preferences(branch);
+            let lookVal = Log.Level[lookPrefBranch.get(prefName)];
+            if (lookVal && lookVal < level) {
+              level = lookVal;
+            }
+          }
+        }
+        appender.level = level;
+      }
+      this._prefs.observe(prefName, observer, this);
+      this._prefObservers.push([prefName, observer]);
+      // and call the observer now with the current pref value.
+      observer(this._prefs.get(prefName));
+      return observer;
+    }
+
+    this._observeConsolePref = setupAppender(consoleAppender, "log.appender.console", Log.Level.Fatal, true);
+    this._observeDumpPref = setupAppender(dumpAppender, "log.appender.dump", Log.Level.Error, true);
+
+    // The file appender doesn't get the special singleton behaviour.
+    let fapp = this._fileAppender = new FlushableStorageAppender(formatter);
+    // the stream gets a default of Debug as the user must go out of their way
+    // to see the stuff spewed to it.
+    this._observeStreamPref = setupAppender(fapp, "log.appender.file.level", Log.Level.Debug);
+
+    // now attach the appenders to all our logs.
+    for (let logName of logNames) {
+      let log = Log.repository.getLogger(logName);
+      for (let appender of [fapp, dumpAppender, consoleAppender]) {
+        log.addAppender(appender);
+      }
+    }
+    // and use the first specified log as a "root" for our log.
+    this._log = Log.repository.getLogger(logNames[0] + ".LogManager");
+  },
+
+  /**
+   * Cleanup this instance
+   */
+  finalize() {
+    for (let [name, pref] of this._prefObservers) {
+      this._prefs.ignore(name, pref, this);
+    }
+    this._prefObservers = [];
+    try {
+      allBranches.delete(this._prefs._branchStr);
+    } catch (e) {}
+    this._prefs = null;
+  },
+
+  get _logFileSubDirectoryEntries() {
+    // At this point we don't allow a custom directory for the logs, nor allow
+    // it to be outside the profile directory.
+    // This returns an array of the the relative directory entries below the
+    // profile dir, and is the directory about:sync-log uses.
+    return ["weave", "logs"];
+  },
+
+  get sawError() {
+    return this._fileAppender.sawError;
+  },
+
+  // Result values for resetFileLog.
+  SUCCESS_LOG_WRITTEN: "success-log-written",
+  ERROR_LOG_WRITTEN: "error-log-written",
+
+  /**
+   * Possibly generate a log file for all accumulated log messages and refresh
+   * the input & output streams.
+   * Whether a "success" or "error" log is written is determined based on
+   * whether an "Error" log entry was written to any of the logs.
+   * Returns a promise that resolves on completion with either null (for no
+   * file written or on error), SUCCESS_LOG_WRITTEN if a "success" log was
+   * written, or ERROR_LOG_WRITTEN if an "error" log was written.
+   */
+  resetFileLog: Task.async(function* () {
+    try {
+      let flushToFile;
+      let reasonPrefix;
+      let reason;
+      if (this._fileAppender.sawError) {
+        reason = this.ERROR_LOG_WRITTEN;
+        flushToFile = this._prefs.get("log.appender.file.logOnError", true);
+        reasonPrefix = "error";
+      } else {
+        reason = this.SUCCESS_LOG_WRITTEN;
+        flushToFile = this._prefs.get("log.appender.file.logOnSuccess", false);
+        reasonPrefix = "success";
+      }
+
+      // might as well avoid creating an input stream if we aren't going to use it.
+      if (!flushToFile) {
+        this._fileAppender.reset();
+        return null;
+      }
+
+      // We have reasonPrefix at the start of the filename so all "error"
+      // logs are grouped in about:sync-log.
+      let filename = reasonPrefix + "-" + this.logFilePrefix + "-" + Date.now() + ".txt";
+      yield this._fileAppender.flushToFile(this._logFileSubDirectoryEntries, filename, this._log);
+
+      // It's not completely clear to markh why we only do log cleanups
+      // for errors, but for now the Sync semantics have been copied...
+      // (one theory is that only cleaning up on error makes it less
+      // likely old error logs would be removed, but that's not true if
+      // there are occasional errors - let's address this later!)
+      if (reason == this.ERROR_LOG_WRITTEN && !this._cleaningUpFileLogs) {
+        this._log.trace("Scheduling cleanup.");
+        // Note we don't return/yield or otherwise wait on this promise - it
+        // continues in the background
+        this.cleanupLogs().catch(err => {
+          this._log.error("Failed to cleanup logs", err);
+        });
+      }
+      return reason;
+    } catch (ex) {
+      this._log.error("Failed to resetFileLog", ex);
+      return null;
+    }
+  }),
+
+  /**
+   * Finds all logs older than maxErrorAge and deletes them using async I/O.
+   */
+  cleanupLogs: Task.async(function* () {
+    this._cleaningUpFileLogs = true;
+    let logDir = FileUtils.getDir("ProfD", this._logFileSubDirectoryEntries);
+    let iterator = new OS.File.DirectoryIterator(logDir.path);
+    let maxAge = this._prefs.get("log.appender.file.maxErrorAge", DEFAULT_MAX_ERROR_AGE);
+    let threshold = Date.now() - 1000 * maxAge;
+
+    this._log.debug("Log cleanup threshold time: " + threshold);
+    yield iterator.forEach(Task.async(function* (entry) {
+      // Note that we don't check this.logFilePrefix is in the name - we cleanup
+      // all files in this directory regardless of that prefix so old logfiles
+      // for prefixes no longer in use are still cleaned up. See bug 1279145.
+      if (!entry.name.startsWith("error-") &&
+          !entry.name.startsWith("success-")) {
+        return;
+      }
+      try {
+        // need to call .stat() as the enumerator doesn't give that to us on *nix.
+        let info = yield OS.File.stat(entry.path);
+        if (info.lastModificationDate.getTime() >= threshold) {
+          return;
+        }
+        this._log.trace(" > Cleanup removing " + entry.name +
+                        " (" + info.lastModificationDate.getTime() + ")");
+        yield OS.File.remove(entry.path);
+        this._log.trace("Deleted " + entry.name);
+      } catch (ex) {
+        this._log.debug("Encountered error trying to clean up old log file "
+                        + entry.name, ex);
+      }
+    }.bind(this)));
+    iterator.close();
+    this._cleaningUpFileLogs = false;
+    this._log.debug("Done deleting files.");
+    // This notification is used only for tests.
+    Services.obs.notifyObservers(null, "services-tests:common:log-manager:cleanup-logs", null);
+  }),
+}

services/common/modules-testing/logging.js

@@ -0,0 +1,54 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "getTestLogger",
+  "initTestLogging",
+];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+
+this.initTestLogging = function initTestLogging(level) {
+  function LogStats() {
+    this.errorsLogged = 0;
+  }
+  LogStats.prototype = {
+    format: function format(message) {
+      if (message.level == Log.Level.Error) {
+        this.errorsLogged += 1;
+      }
+
+      return message.time + "\t" + message.loggerName + "\t" + message.levelDesc + "\t" +
+        this.formatText(message) + "\n";
+    }
+  };
+  LogStats.prototype.__proto__ = new Log.BasicFormatter();
+
+  let log = Log.repository.rootLogger;
+  let logStats = new LogStats();
+  let appender = new Log.DumpAppender(logStats);
+
+  if (typeof(level) == "undefined") {
+    level = "Debug";
+  }
+  getTestLogger().level = Log.Level[level];
+  Log.repository.getLogger("Services").level = Log.Level[level];
+
+  log.level = Log.Level.Trace;
+  appender.level = Log.Level.Trace;
+  // Overwrite any other appenders (e.g. from previous incarnations)
+  log.ownAppenders = [appender];
+  log.updateAppenders();
+
+  return logStats;
+}
+
+this.getTestLogger = function getTestLogger(component) {
+  return Log.repository.getLogger("Testing");
+}
+

services/common/modules-testing/utils.js

@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "TestingUtils",
+];
+
+this.TestingUtils = {
+  /**
+   * Perform a deep copy of an Array or Object.
+   */
+  deepCopy: function deepCopy(thing, noSort) {
+    if (typeof(thing) != "object" || thing == null) {
+      return thing;
+    }
+
+    if (Array.isArray(thing)) {
+      let ret = [];
+      for (let element of thing) {
+        ret.push(this.deepCopy(element, noSort));
+      }
+
+      return ret;
+    }
+
+    let ret = {};
+    let props = Object.keys(thing);
+
+    if (!noSort) {
+      props = props.sort();
+    }
+
+    for (let prop of props) {
+      ret[prop] = this.deepCopy(thing[prop], noSort);
+    }
+
+    return ret;
+  },
+};

services/common/moz.build

@@ -0,0 +1,48 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('**'):
+    BUG_COMPONENT = ('Mozilla Services', 'Firefox: Common')
+
+TEST_DIRS += ['tests']
+
+EXTRA_COMPONENTS += [
+    'servicesComponents.manifest',
+]
+
+EXTRA_JS_MODULES['services-common'] += [
+    'async.js',
+    'blocklist-clients.js',
+    'blocklist-updater.js',
+    'kinto-http-client.js',
+    'kinto-offline-client.js',
+    'logmanager.js',
+    'observers.js',
+    'rest.js',
+    'stringbundle.js',
+    'utils.js',
+]
+
+if CONFIG['MOZ_WIDGET_TOOLKIT'] != 'android':
+    EXTRA_JS_MODULES['services-common'] += [
+        'hawkclient.js',
+        'hawkrequest.js',
+        'tokenserverclient.js',
+    ]
+
+    TESTING_JS_MODULES.services.common += [
+        'modules-testing/storageserver.js',
+    ]
+
+TESTING_JS_MODULES.services.common += [
+    'modules-testing/logging.js',
+    'modules-testing/utils.js',
+]
+
+JS_PREFERENCE_FILES += [
+    'services-common.js',
+]
+

services/common/observers.js

@@ -0,0 +1,150 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["Observers"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+/**
+ * A service for adding, removing and notifying observers of notifications.
+ * Wraps the nsIObserverService interface.
+ *
+ * @version 0.2
+ */
+this.Observers = {
+  /**
+   * Register the given callback as an observer of the given topic.
+   *
+   * @param   topic       {String}
+   *          the topic to observe
+   *
+   * @param   callback    {Object}
+   *          the callback; an Object that implements nsIObserver or a Function
+   *          that gets called when the notification occurs
+   *
+   * @param   thisObject  {Object}  [optional]
+   *          the object to use as |this| when calling a Function callback
+   *
+   * @returns the observer
+   */
+  add: function(topic, callback, thisObject) {
+    let observer = new Observer(topic, callback, thisObject);
+    this._cache.push(observer);
+    this._service.addObserver(observer, topic, true);
+
+    return observer;
+  },
+
+  /**
+   * Unregister the given callback as an observer of the given topic.
+   *
+   * @param topic       {String}
+   *        the topic being observed
+   *
+   * @param callback    {Object}
+   *        the callback doing the observing
+   *
+   * @param thisObject  {Object}  [optional]
+   *        the object being used as |this| when calling a Function callback
+   */
+  remove: function(topic, callback, thisObject) {
+    // This seems fairly inefficient, but I'm not sure how much better
+    // we can make it.  We could index by topic, but we can't index by callback
+    // or thisObject, as far as I know, since the keys to JavaScript hashes
+    // (a.k.a. objects) can apparently only be primitive values.
+    let [observer] = this._cache.filter(v => v.topic      == topic    &&
+                                             v.callback   == callback &&
+                                             v.thisObject == thisObject);
+    if (observer) {
+      this._service.removeObserver(observer, topic);
+      this._cache.splice(this._cache.indexOf(observer), 1);
+    }
+  },
+
+  /**
+   * Notify observers about something.
+   *
+   * @param topic   {String}
+   *        the topic to notify observers about
+   *
+   * @param subject {Object}  [optional]
+   *        some information about the topic; can be any JS object or primitive
+   *
+   * @param data    {String}  [optional] [deprecated]
+   *        some more information about the topic; deprecated as the subject
+   *        is sufficient to pass all needed information to the JS observers
+   *        that this module targets; if you have multiple values to pass to
+   *        the observer, wrap them in an object and pass them via the subject
+   *        parameter (i.e.: { foo: 1, bar: "some string", baz: myObject })
+   */
+  notify: function(topic, subject, data) {
+    subject = (typeof subject == "undefined") ? null : new Subject(subject);
+       data = (typeof    data == "undefined") ? null : data;
+    this._service.notifyObservers(subject, topic, data);
+  },
+
+  _service: Cc["@mozilla.org/observer-service;1"].
+            getService(Ci.nsIObserverService),
+
+  /**
+   * A cache of observers that have been added.
+   *
+   * We use this to remove observers when a caller calls |remove|.
+   *
+   * XXX This might result in reference cycles, causing memory leaks,
+   * if we hold a reference to an observer that holds a reference to us.
+   * Could we fix that by making this an independent top-level object
+   * rather than a property of this object?
+   */
+  _cache: []
+};
+
+
+function Observer(topic, callback, thisObject) {
+  this.topic = topic;
+  this.callback = callback;
+  this.thisObject = thisObject;
+}
+
+Observer.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver, Ci.nsISupportsWeakReference]),
+  observe: function(subject, topic, data) {
+    // Extract the wrapped object for subjects that are one of our wrappers
+    // around a JS object.  This way we support both wrapped subjects created
+    // using this module and those that are real XPCOM components.
+    if (subject && typeof subject == "object" &&
+        ("wrappedJSObject" in subject) &&
+        ("observersModuleSubjectWrapper" in subject.wrappedJSObject))
+      subject = subject.wrappedJSObject.object;
+
+    if (typeof this.callback == "function") {
+      if (this.thisObject)
+        this.callback.call(this.thisObject, subject, data);
+      else
+        this.callback(subject, data);
+    }
+    else // typeof this.callback == "object" (nsIObserver)
+      this.callback.observe(subject, topic, data);
+  }
+}
+
+
+function Subject(object) {
+  // Double-wrap the object and set a property identifying the wrappedJSObject
+  // as one of our wrappers to distinguish between subjects that are one of our
+  // wrappers (which we should unwrap when notifying our observers) and those
+  // that are real JS XPCOM components (which we should pass through unaltered).
+  this.wrappedJSObject = { observersModuleSubjectWrapper: true, object: object };
+}
+
+Subject.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([]),
+  getScriptableHelper: function() {},
+  getInterfaces: function() {}
+};

services/common/rest.js

@@ -0,0 +1,764 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+this.EXPORTED_SYMBOLS = [
+  "RESTRequest",
+  "RESTResponse",
+  "TokenAuthenticatedRESTRequest",
+];
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/NetUtil.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "CryptoUtils",
+                                  "resource://services-crypto/utils.js");
+
+const Prefs = new Preferences("services.common.");
+
+/**
+ * Single use HTTP requests to RESTish resources.
+ *
+ * @param uri
+ *        URI for the request. This can be an nsIURI object or a string
+ *        that can be used to create one. An exception will be thrown if
+ *        the string is not a valid URI.
+ *
+ * Examples:
+ *
+ * (1) Quick GET request:
+ *
+ *   new RESTRequest("http://server/rest/resource").get(function (error) {
+ *     if (error) {
+ *       // Deal with a network error.
+ *       processNetworkErrorCode(error.result);
+ *       return;
+ *     }
+ *     if (!this.response.success) {
+ *       // Bail out if we're not getting an HTTP 2xx code.
+ *       processHTTPError(this.response.status);
+ *       return;
+ *     }
+ *     processData(this.response.body);
+ *   });
+ *
+ * (2) Quick PUT request (non-string data is automatically JSONified)
+ *
+ *   new RESTRequest("http://server/rest/resource").put(data, function (error) {
+ *     ...
+ *   });
+ *
+ * (3) Streaming GET
+ *
+ *   let request = new RESTRequest("http://server/rest/resource");
+ *   request.setHeader("Accept", "application/newlines");
+ *   request.onComplete = function (error) {
+ *     if (error) {
+ *       // Deal with a network error.
+ *       processNetworkErrorCode(error.result);
+ *       return;
+ *     }
+ *     callbackAfterRequestHasCompleted()
+ *   });
+ *   request.onProgress = function () {
+ *     if (!this.response.success) {
+ *       // Bail out if we're not getting an HTTP 2xx code.
+ *       return;
+ *     }
+ *     // Process body data and reset it so we don't process the same data twice.
+ *     processIncrementalData(this.response.body);
+ *     this.response.body = "";
+ *   });
+ *   request.get();
+ */
+this.RESTRequest = function RESTRequest(uri) {
+  this.status = this.NOT_SENT;
+
+  // If we don't have an nsIURI object yet, make one. This will throw if
+  // 'uri' isn't a valid URI string.
+  if (!(uri instanceof Ci.nsIURI)) {
+    uri = Services.io.newURI(uri, null, null);
+  }
+  this.uri = uri;
+
+  this._headers = {};
+  this._log = Log.repository.getLogger(this._logName);
+  this._log.level =
+    Log.Level[Prefs.get("log.logger.rest.request")];
+}
+RESTRequest.prototype = {
+
+  _logName: "Services.Common.RESTRequest",
+
+  QueryInterface: XPCOMUtils.generateQI([
+    Ci.nsIBadCertListener2,
+    Ci.nsIInterfaceRequestor,
+    Ci.nsIChannelEventSink
+  ]),
+
+  /*** Public API: ***/
+
+  /**
+   * A constant boolean that indicates whether this object will automatically
+   * utf-8 encode request bodies passed as an object. Used for feature detection
+   * so, eg, loop can use the same source code for old and new Firefox versions.
+   */
+  willUTF8EncodeObjectRequests: true,
+
+  /**
+   * URI for the request (an nsIURI object).
+   */
+  uri: null,
+
+  /**
+   * HTTP method (e.g. "GET")
+   */
+  method: null,
+
+  /**
+   * RESTResponse object
+   */
+  response: null,
+
+  /**
+   * nsIRequest load flags. Don't do any caching by default. Don't send user
+   * cookies and such over the wire (Bug 644734).
+   */
+  loadFlags: Ci.nsIRequest.LOAD_BYPASS_CACHE | Ci.nsIRequest.INHIBIT_CACHING | Ci.nsIRequest.LOAD_ANONYMOUS,
+
+  /**
+   * nsIHttpChannel
+   */
+  channel: null,
+
+  /**
+   * Flag to indicate the status of the request.
+   *
+   * One of NOT_SENT, SENT, IN_PROGRESS, COMPLETED, ABORTED.
+   */
+  status: null,
+
+  NOT_SENT:    0,
+  SENT:        1,
+  IN_PROGRESS: 2,
+  COMPLETED:   4,
+  ABORTED:     8,
+
+  /**
+   * HTTP status text of response
+   */
+  statusText: null,
+
+  /**
+   * Request timeout (in seconds, though decimal values can be used for
+   * up to millisecond granularity.)
+   *
+   * 0 for no timeout.
+   */
+  timeout: null,
+
+  /**
+   * The encoding with which the response to this request must be treated.
+   * If a charset parameter is available in the HTTP Content-Type header for
+   * this response, that will always be used, and this value is ignored. We
+   * default to UTF-8 because that is a reasonable default.
+   */
+  charset: "utf-8",
+
+  /**
+   * Called when the request has been completed, including failures and
+   * timeouts.
+   *
+   * @param error
+   *        Error that occurred while making the request, null if there
+   *        was no error.
+   */
+  onComplete: function onComplete(error) {
+  },
+
+  /**
+   * Called whenever data is being received on the channel. If this throws an
+   * exception, the request is aborted and the exception is passed as the
+   * error to onComplete().
+   */
+  onProgress: function onProgress() {
+  },
+
+  /**
+   * Set a request header.
+   */
+  setHeader: function setHeader(name, value) {
+    this._headers[name.toLowerCase()] = value;
+  },
+
+  /**
+   * Perform an HTTP GET.
+   *
+   * @param onComplete
+   *        Short-circuit way to set the 'onComplete' method. Optional.
+   * @param onProgress
+   *        Short-circuit way to set the 'onProgress' method. Optional.
+   *
+   * @return the request object.
+   */
+  get: function get(onComplete, onProgress) {
+    return this.dispatch("GET", null, onComplete, onProgress);
+  },
+
+  /**
+   * Perform an HTTP PATCH.
+   *
+   * @param data
+   *        Data to be used as the request body. If this isn't a string
+   *        it will be JSONified automatically.
+   * @param onComplete
+   *        Short-circuit way to set the 'onComplete' method. Optional.
+   * @param onProgress
+   *        Short-circuit way to set the 'onProgress' method. Optional.
+   *
+   * @return the request object.
+   */
+  patch: function patch(data, onComplete, onProgress) {
+    return this.dispatch("PATCH", data, onComplete, onProgress);
+  },
+
+  /**
+   * Perform an HTTP PUT.
+   *
+   * @param data
+   *        Data to be used as the request body. If this isn't a string
+   *        it will be JSONified automatically.
+   * @param onComplete
+   *        Short-circuit way to set the 'onComplete' method. Optional.
+   * @param onProgress
+   *        Short-circuit way to set the 'onProgress' method. Optional.
+   *
+   * @return the request object.
+   */
+  put: function put(data, onComplete, onProgress) {
+    return this.dispatch("PUT", data, onComplete, onProgress);
+  },
+
+  /**
+   * Perform an HTTP POST.
+   *
+   * @param data
+   *        Data to be used as the request body. If this isn't a string
+   *        it will be JSONified automatically.
+   * @param onComplete
+   *        Short-circuit way to set the 'onComplete' method. Optional.
+   * @param onProgress
+   *        Short-circuit way to set the 'onProgress' method. Optional.
+   *
+   * @return the request object.
+   */
+  post: function post(data, onComplete, onProgress) {
+    return this.dispatch("POST", data, onComplete, onProgress);
+  },
+
+  /**
+   * Perform an HTTP DELETE.
+   *
+   * @param onComplete
+   *        Short-circuit way to set the 'onComplete' method. Optional.
+   * @param onProgress
+   *        Short-circuit way to set the 'onProgress' method. Optional.
+   *
+   * @return the request object.
+   */
+  delete: function delete_(onComplete, onProgress) {
+    return this.dispatch("DELETE", null, onComplete, onProgress);
+  },
+
+  /**
+   * Abort an active request.
+   */
+  abort: function abort() {
+    if (this.status != this.SENT && this.status != this.IN_PROGRESS) {
+      throw "Can only abort a request that has been sent.";
+    }
+
+    this.status = this.ABORTED;
+    this.channel.cancel(Cr.NS_BINDING_ABORTED);
+
+    if (this.timeoutTimer) {
+      // Clear the abort timer now that the channel is done.
+      this.timeoutTimer.clear();
+    }
+  },
+
+  /*** Implementation stuff ***/
+
+  dispatch: function dispatch(method, data, onComplete, onProgress) {
+    if (this.status != this.NOT_SENT) {
+      throw "Request has already been sent!";
+    }
+
+    this.method = method;
+    if (onComplete) {
+      this.onComplete = onComplete;
+    }
+    if (onProgress) {
+      this.onProgress = onProgress;
+    }
+
+    // Create and initialize HTTP channel.
+    let channel = NetUtil.newChannel({uri: this.uri, loadUsingSystemPrincipal: true})
+                         .QueryInterface(Ci.nsIRequest)
+                         .QueryInterface(Ci.nsIHttpChannel);
+    this.channel = channel;
+    channel.loadFlags |= this.loadFlags;
+    channel.notificationCallbacks = this;
+
+    this._log.debug(`${method} request to ${this.uri.spec}`);
+    // Set request headers.
+    let headers = this._headers;
+    for (let key in headers) {
+      if (key == 'authorization') {
+        this._log.trace("HTTP Header " + key + ": ***** (suppressed)");
+      } else {
+        this._log.trace("HTTP Header " + key + ": " + headers[key]);
+      }
+      channel.setRequestHeader(key, headers[key], false);
+    }
+
+    // Set HTTP request body.
+    if (method == "PUT" || method == "POST" || method == "PATCH") {
+      // Convert non-string bodies into JSON with utf-8 encoding. If a string
+      // is passed we assume they've already encoded it.
+      let contentType = headers["content-type"];
+      if (typeof data != "string") {
+        data = JSON.stringify(data);
+        if (!contentType) {
+          contentType = "application/json";
+        }
+        if (!contentType.includes("charset")) {
+          data = CommonUtils.encodeUTF8(data);
+          contentType += "; charset=utf-8";
+        } else {
+          // If someone handed us an object but also a custom content-type
+          // it's probably confused. We could go to even further lengths to
+          // respect it, but this shouldn't happen in practice.
+          Cu.reportError("rest.js found an object to JSON.stringify but also a " +
+                         "content-type header with a charset specification. " +
+                         "This probably isn't going to do what you expect");
+        }
+      }
+      if (!contentType) {
+        contentType = "text/plain";
+      }
+
+      this._log.debug(method + " Length: " + data.length);
+      if (this._log.level <= Log.Level.Trace) {
+        this._log.trace(method + " Body: " + data);
+      }
+
+      let stream = Cc["@mozilla.org/io/string-input-stream;1"]
+                     .createInstance(Ci.nsIStringInputStream);
+      stream.setData(data, data.length);
+
+      channel.QueryInterface(Ci.nsIUploadChannel);
+      channel.setUploadStream(stream, contentType, data.length);
+    }
+    // We must set this after setting the upload stream, otherwise it
+    // will always be 'PUT'. Yeah, I know.
+    channel.requestMethod = method;
+
+    // Before opening the channel, set the charset that serves as a hint
+    // as to what the response might be encoded as.
+    channel.contentCharset = this.charset;
+
+    // Blast off!
+    try {
+      channel.asyncOpen2(this);
+    } catch (ex) {
+      // asyncOpen can throw in a bunch of cases -- e.g., a forbidden port.
+      this._log.warn("Caught an error in asyncOpen", ex);
+      CommonUtils.nextTick(onComplete.bind(this, ex));
+    }
+    this.status = this.SENT;
+    this.delayTimeout();
+    return this;
+  },
+
+  /**
+   * Create or push back the abort timer that kills this request.
+   */
+  delayTimeout: function delayTimeout() {
+    if (this.timeout) {
+      CommonUtils.namedTimer(this.abortTimeout, this.timeout * 1000, this,
+                             "timeoutTimer");
+    }
+  },
+
+  /**
+   * Abort the request based on a timeout.
+   */
+  abortTimeout: function abortTimeout() {
+    this.abort();
+    let error = Components.Exception("Aborting due to channel inactivity.",
+                                     Cr.NS_ERROR_NET_TIMEOUT);
+    if (!this.onComplete) {
+      this._log.error("Unexpected error: onComplete not defined in " +
+                      "abortTimeout.");
+      return;
+    }
+    this.onComplete(error);
+  },
+
+  /*** nsIStreamListener ***/
+
+  onStartRequest: function onStartRequest(channel) {
+    if (this.status == this.ABORTED) {
+      this._log.trace("Not proceeding with onStartRequest, request was aborted.");
+      return;
+    }
+
+    try {
+      channel.QueryInterface(Ci.nsIHttpChannel);
+    } catch (ex) {
+      this._log.error("Unexpected error: channel is not a nsIHttpChannel!");
+      this.status = this.ABORTED;
+      channel.cancel(Cr.NS_BINDING_ABORTED);
+      return;
+    }
+
+    this.status = this.IN_PROGRESS;
+
+    this._log.trace("onStartRequest: " + channel.requestMethod + " " +
+                    channel.URI.spec);
+
+    // Create a response object and fill it with some data.
+    let response = this.response = new RESTResponse();
+    response.request = this;
+    response.body = "";
+
+    this.delayTimeout();
+  },
+
+  onStopRequest: function onStopRequest(channel, context, statusCode) {
+    if (this.timeoutTimer) {
+      // Clear the abort timer now that the channel is done.
+      this.timeoutTimer.clear();
+    }
+
+    // We don't want to do anything for a request that's already been aborted.
+    if (this.status == this.ABORTED) {
+      this._log.trace("Not proceeding with onStopRequest, request was aborted.");
+      return;
+    }
+
+    try {
+      channel.QueryInterface(Ci.nsIHttpChannel);
+    } catch (ex) {
+      this._log.error("Unexpected error: channel not nsIHttpChannel!");
+      this.status = this.ABORTED;
+      return;
+    }
+    this.status = this.COMPLETED;
+
+    let statusSuccess = Components.isSuccessCode(statusCode);
+    let uri = channel && channel.URI && channel.URI.spec || "<unknown>";
+    this._log.trace("Channel for " + channel.requestMethod + " " + uri +
+                    " returned status code " + statusCode);
+
+    if (!this.onComplete) {
+      this._log.error("Unexpected error: onComplete not defined in " +
+                      "abortRequest.");
+      this.onProgress = null;
+      return;
+    }
+
+    // Throw the failure code and stop execution.  Use Components.Exception()
+    // instead of Error() so the exception is QI-able and can be passed across
+    // XPCOM borders while preserving the status code.
+    if (!statusSuccess) {
+      let message = Components.Exception("", statusCode).name;
+      let error = Components.Exception(message, statusCode);
+      this._log.debug(this.method + " " + uri + " failed: " + statusCode + " - " + message);
+      this.onComplete(error);
+      this.onComplete = this.onProgress = null;
+      return;
+    }
+
+    this._log.debug(this.method + " " + uri + " " + this.response.status);
+
+    // Additionally give the full response body when Trace logging.
+    if (this._log.level <= Log.Level.Trace) {
+      this._log.trace(this.method + " body: " + this.response.body);
+    }
+
+    delete this._inputStream;
+
+    this.onComplete(null);
+    this.onComplete = this.onProgress = null;
+  },
+
+  onDataAvailable: function onDataAvailable(channel, cb, stream, off, count) {
+    // We get an nsIRequest, which doesn't have contentCharset.
+    try {
+      channel.QueryInterface(Ci.nsIHttpChannel);
+    } catch (ex) {
+      this._log.error("Unexpected error: channel not nsIHttpChannel!");
+      this.abort();
+
+      if (this.onComplete) {
+        this.onComplete(ex);
+      }
+
+      this.onComplete = this.onProgress = null;
+      return;
+    }
+
+    if (channel.contentCharset) {
+      this.response.charset = channel.contentCharset;
+
+      if (!this._converterStream) {
+        this._converterStream = Cc["@mozilla.org/intl/converter-input-stream;1"]
+                                   .createInstance(Ci.nsIConverterInputStream);
+      }
+
+      this._converterStream.init(stream, channel.contentCharset, 0,
+                                 this._converterStream.DEFAULT_REPLACEMENT_CHARACTER);
+
+      try {
+        let str = {};
+        let num = this._converterStream.readString(count, str);
+        if (num != 0) {
+          this.response.body += str.value;
+        }
+      } catch (ex) {
+        this._log.warn("Exception thrown reading " + count + " bytes from " +
+                       "the channel", ex);
+        throw ex;
+      }
+    } else {
+      this.response.charset = null;
+
+      if (!this._inputStream) {
+        this._inputStream = Cc["@mozilla.org/scriptableinputstream;1"]
+                              .createInstance(Ci.nsIScriptableInputStream);
+      }
+
+      this._inputStream.init(stream);
+
+      this.response.body += this._inputStream.read(count);
+    }
+
+    try {
+      this.onProgress();
+    } catch (ex) {
+      this._log.warn("Got exception calling onProgress handler, aborting " +
+                     this.method + " " + channel.URI.spec, ex);
+      this.abort();
+
+      if (!this.onComplete) {
+        this._log.error("Unexpected error: onComplete not defined in " +
+                        "onDataAvailable.");
+        this.onProgress = null;
+        return;
+      }
+
+      this.onComplete(ex);
+      this.onComplete = this.onProgress = null;
+      return;
+    }
+
+    this.delayTimeout();
+  },
+
+  /*** nsIInterfaceRequestor ***/
+
+  getInterface: function(aIID) {
+    return this.QueryInterface(aIID);
+  },
+
+  /*** nsIBadCertListener2 ***/
+
+  notifyCertProblem: function notifyCertProblem(socketInfo, sslStatus, targetHost) {
+    this._log.warn("Invalid HTTPS certificate encountered!");
+    // Suppress invalid HTTPS certificate warnings in the UI.
+    // (The request will still fail.)
+    return true;
+  },
+
+  /**
+   * Returns true if headers from the old channel should be
+   * copied to the new channel. Invoked when a channel redirect
+   * is in progress.
+   */
+  shouldCopyOnRedirect: function shouldCopyOnRedirect(oldChannel, newChannel, flags) {
+    let isInternal = !!(flags & Ci.nsIChannelEventSink.REDIRECT_INTERNAL);
+    let isSameURI  = newChannel.URI.equals(oldChannel.URI);
+    this._log.debug("Channel redirect: " + oldChannel.URI.spec + ", " +
+                    newChannel.URI.spec + ", internal = " + isInternal);
+    return isInternal && isSameURI;
+  },
+
+  /*** nsIChannelEventSink ***/
+  asyncOnChannelRedirect:
+    function asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) {
+
+    let oldSpec = (oldChannel && oldChannel.URI) ? oldChannel.URI.spec : "<undefined>";
+    let newSpec = (newChannel && newChannel.URI) ? newChannel.URI.spec : "<undefined>";
+    this._log.debug("Channel redirect: " + oldSpec + ", " + newSpec + ", " + flags);
+
+    try {
+      newChannel.QueryInterface(Ci.nsIHttpChannel);
+    } catch (ex) {
+      this._log.error("Unexpected error: channel not nsIHttpChannel!");
+      callback.onRedirectVerifyCallback(Cr.NS_ERROR_NO_INTERFACE);
+      return;
+    }
+
+    // For internal redirects, copy the headers that our caller set.
+    try {
+      if (this.shouldCopyOnRedirect(oldChannel, newChannel, flags)) {
+        this._log.trace("Copying headers for safe internal redirect.");
+        for (let key in this._headers) {
+          newChannel.setRequestHeader(key, this._headers[key], false);
+        }
+      }
+    } catch (ex) {
+      this._log.error("Error copying headers", ex);
+    }
+
+    this.channel = newChannel;
+
+    // We let all redirects proceed.
+    callback.onRedirectVerifyCallback(Cr.NS_OK);
+  }
+};
+
+/**
+ * Response object for a RESTRequest. This will be created automatically by
+ * the RESTRequest.
+ */
+this.RESTResponse = function RESTResponse() {
+  this._log = Log.repository.getLogger(this._logName);
+  this._log.level =
+    Log.Level[Prefs.get("log.logger.rest.response")];
+}
+RESTResponse.prototype = {
+
+  _logName: "Services.Common.RESTResponse",
+
+  /**
+   * Corresponding REST request
+   */
+  request: null,
+
+  /**
+   * HTTP status code
+   */
+  get status() {
+    let status;
+    try {
+      status = this.request.channel.responseStatus;
+    } catch (ex) {
+      this._log.debug("Caught exception fetching HTTP status code", ex);
+      return null;
+    }
+    Object.defineProperty(this, "status", {value: status});
+    return status;
+  },
+
+  /**
+   * HTTP status text
+   */
+  get statusText() {
+    let statusText;
+    try {
+      statusText = this.request.channel.responseStatusText;
+    } catch (ex) {
+      this._log.debug("Caught exception fetching HTTP status text", ex);
+      return null;
+    }
+    Object.defineProperty(this, "statusText", {value: statusText});
+    return statusText;
+  },
+
+  /**
+   * Boolean flag that indicates whether the HTTP status code is 2xx or not.
+   */
+  get success() {
+    let success;
+    try {
+      success = this.request.channel.requestSucceeded;
+    } catch (ex) {
+      this._log.debug("Caught exception fetching HTTP success flag", ex);
+      return null;
+    }
+    Object.defineProperty(this, "success", {value: success});
+    return success;
+  },
+
+  /**
+   * Object containing HTTP headers (keyed as lower case)
+   */
+  get headers() {
+    let headers = {};
+    try {
+      this._log.trace("Processing response headers.");
+      let channel = this.request.channel.QueryInterface(Ci.nsIHttpChannel);
+      channel.visitResponseHeaders(function (header, value) {
+        headers[header.toLowerCase()] = value;
+      });
+    } catch (ex) {
+      this._log.debug("Caught exception processing response headers", ex);
+      return null;
+    }
+
+    Object.defineProperty(this, "headers", {value: headers});
+    return headers;
+  },
+
+  /**
+   * HTTP body (string)
+   */
+  body: null
+
+};
+
+/**
+ * Single use MAC authenticated HTTP requests to RESTish resources.
+ *
+ * @param uri
+ *        URI going to the RESTRequest constructor.
+ * @param authToken
+ *        (Object) An auth token of the form {id: (string), key: (string)}
+ *        from which the MAC Authentication header for this request will be
+ *        derived. A token as obtained from
+ *        TokenServerClient.getTokenFromBrowserIDAssertion is accepted.
+ * @param extra
+ *        (Object) Optional extra parameters. Valid keys are: nonce_bytes, ts,
+ *        nonce, and ext. See CrytoUtils.computeHTTPMACSHA1 for information on
+ *        the purpose of these values.
+ */
+this.TokenAuthenticatedRESTRequest =
+ function TokenAuthenticatedRESTRequest(uri, authToken, extra) {
+  RESTRequest.call(this, uri);
+  this.authToken = authToken;
+  this.extra = extra || {};
+}
+TokenAuthenticatedRESTRequest.prototype = {
+  __proto__: RESTRequest.prototype,
+
+  dispatch: function dispatch(method, data, onComplete, onProgress) {
+    let sig = CryptoUtils.computeHTTPMACSHA1(
+      this.authToken.id, this.authToken.key, method, this.uri, this.extra
+    );
+
+    this.setHeader("Authorization", sig.getHeader());
+
+    return RESTRequest.prototype.dispatch.call(
+      this, method, data, onComplete, onProgress
+    );
+  },
+};

services/common/services-common.js

@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file contains default preference values for components in
+// services-common.
+
+pref("services.common.log.logger.rest.request", "Debug");
+pref("services.common.log.logger.rest.response", "Debug");
+
+pref("services.common.log.logger.tokenserverclient", "Debug");

services/common/servicesComponents.manifest

@@ -0,0 +1,2 @@
+# Register resource aliases
+resource services-common resource://gre/modules/services-common/

services/common/stringbundle.js

@@ -0,0 +1,203 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["StringBundle"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+/**
+ * A string bundle.
+ *
+ * This object presents two APIs: a deprecated one that is equivalent to the API
+ * for the stringbundle XBL binding, to make it easy to switch from that binding
+ * to this module, and a new one that is simpler and easier to use.
+ *
+ * The benefit of this module over the XBL binding is that it can also be used
+ * in JavaScript modules and components, not only in chrome JS.
+ *
+ * To use this module, import it, create a new instance of StringBundle,
+ * and then use the instance's |get| and |getAll| methods to retrieve strings
+ * (you can get both plain and formatted strings with |get|):
+ *
+ *   let strings =
+ *     new StringBundle("chrome://example/locale/strings.properties");
+ *   let foo = strings.get("foo");
+ *   let barFormatted = strings.get("bar", [arg1, arg2]);
+ *   for (let string of strings.getAll())
+ *     dump (string.key + " = " + string.value + "\n");
+ *
+ * @param url {String}
+ *        the URL of the string bundle
+ */
+this.StringBundle = function StringBundle(url) {
+  this.url = url;
+}
+
+StringBundle.prototype = {
+  /**
+   * the locale associated with the application
+   * @type nsILocale
+   * @private
+   */
+  get _appLocale() {
+    try {
+      return Cc["@mozilla.org/intl/nslocaleservice;1"].
+             getService(Ci.nsILocaleService).
+             getApplicationLocale();
+    }
+    catch(ex) {
+      return null;
+    }
+  },
+
+  /**
+   * the wrapped nsIStringBundle
+   * @type nsIStringBundle
+   * @private
+   */
+  get _stringBundle() {
+    let stringBundle = Cc["@mozilla.org/intl/stringbundle;1"].
+                       getService(Ci.nsIStringBundleService).
+                       createBundle(this.url, this._appLocale);
+    this.__defineGetter__("_stringBundle", () => stringBundle);
+    return this._stringBundle;
+  },
+
+
+  // the new API
+
+  /**
+   * the URL of the string bundle
+   * @type String
+   */
+  _url: null,
+  get url() {
+    return this._url;
+  },
+  set url(newVal) {
+    this._url = newVal;
+    delete this._stringBundle;
+  },
+
+  /**
+   * Get a string from the bundle.
+   *
+   * @param key {String}
+   *        the identifier of the string to get
+   * @param args {array} [optional]
+   *        an array of arguments that replace occurrences of %S in the string
+   *
+   * @returns {String} the value of the string
+   */
+  get: function(key, args) {
+    if (args)
+      return this.stringBundle.formatStringFromName(key, args, args.length);
+    else
+      return this.stringBundle.GetStringFromName(key);
+  },
+
+  /**
+   * Get all the strings in the bundle.
+   *
+   * @returns {Array}
+   *          an array of objects with key and value properties
+   */
+  getAll: function() {
+    let strings = [];
+
+    // FIXME: for performance, return an enumerable array that wraps the string
+    // bundle's nsISimpleEnumerator (does JavaScript already support this?).
+
+    let enumerator = this.stringBundle.getSimpleEnumeration();
+
+    while (enumerator.hasMoreElements()) {
+      // We could simply return the nsIPropertyElement objects, but I think
+      // it's better to return standard JS objects that behave as consumers
+      // expect JS objects to behave (f.e. you can modify them dynamically).
+      let string = enumerator.getNext().QueryInterface(Ci.nsIPropertyElement);
+      strings.push({ key: string.key, value: string.value });
+    }
+
+    return strings;
+  },
+
+
+  // the deprecated XBL binding-compatible API
+
+  /**
+   * the URL of the string bundle
+   * @deprecated because its name doesn't make sense outside of an XBL binding
+   * @type String
+   */
+  get src() {
+    return this.url;
+  },
+  set src(newVal) {
+    this.url = newVal;
+  },
+
+  /**
+   * the locale associated with the application
+   * @deprecated because it has never been used outside the XBL binding itself,
+   * and consumers should obtain it directly from the locale service anyway.
+   * @type nsILocale
+   */
+  get appLocale() {
+    return this._appLocale;
+  },
+
+  /**
+   * the wrapped nsIStringBundle
+   * @deprecated because this module should provide all necessary functionality
+   * @type nsIStringBundle
+   *
+   * If you do ever need to use this, let the authors of this module know why
+   * so they can surface functionality for your use case in the module itself
+   * and you don't have to access this underlying XPCOM component.
+   */
+  get stringBundle() {
+    return this._stringBundle;
+  },
+
+  /**
+   * Get a string from the bundle.
+   * @deprecated use |get| instead
+   *
+   * @param key {String}
+   *        the identifier of the string to get
+   *
+   * @returns {String}
+   *          the value of the string
+   */
+  getString: function(key) {
+    return this.get(key);
+  },
+
+  /**
+   * Get a formatted string from the bundle.
+   * @deprecated use |get| instead
+   *
+   * @param key {string}
+   *        the identifier of the string to get
+   * @param args {array}
+   *        an array of arguments that replace occurrences of %S in the string
+   *
+   * @returns {String}
+   *          the formatted value of the string
+   */
+  getFormattedString: function(key, args) {
+    return this.get(key, args);
+  },
+
+  /**
+   * Get an enumeration of the strings in the bundle.
+   * @deprecated use |getAll| instead
+   *
+   * @returns {nsISimpleEnumerator}
+   *          a enumeration of the strings in the bundle
+   */
+  get strings() {
+    return this.stringBundle.getSimpleEnumeration();
+  }
+}

services/common/tests/mach_commands.py

@@ -0,0 +1,111 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+import mozpack.path as mozpath
+
+from mozbuild.base import (
+    MachCommandBase,
+)
+
+from mach.decorators import (
+    CommandArgument,
+    CommandProvider,
+    Command,
+)
+
+from mach.registrar import (
+    Registrar
+)
+
+from shutil import rmtree
+from subprocess import Popen
+from sys import argv
+from sys import exit
+from tempfile import mkdtemp
+
+
+
+DEFAULT_PORT = 8080
+DEFAULT_HOSTNAME = 'localhost'
+
+SRCDIR = mozpath.abspath(mozpath.dirname(__file__))
+
+STORAGE_SERVER_SCRIPT = mozpath.join(SRCDIR, 'run_storage_server.js')
+
+def SyncStorageCommand(func):
+    """Decorator that adds shared command arguments to services commands."""
+
+    port = CommandArgument('--port', metavar='PORT', type=int,
+                           default=DEFAULT_PORT, help='Port to run server on.')
+    func = port(func)
+
+    address = CommandArgument('--address', metavar='ADDRESS',
+                              default=DEFAULT_HOSTNAME,
+                              help='Hostname to bind server to.')
+    func = address(func)
+
+    return func
+
+Registrar.register_category(name='services',
+                            title='Services utilities',
+                            description='Commands for services development.')
+
+@CommandProvider
+class SyncTestCommands(MachCommandBase):
+    def __init__(self, context):
+        MachCommandBase.__init__(self, context)
+
+    def run_server(self, js_file, hostname, port):
+        topsrcdir = self.topsrcdir
+        topobjdir = self.topobjdir
+
+        unit_test_dir = mozpath.join(SRCDIR, 'unit')
+
+        head_paths = [
+            'head_global.js',
+            'head_helpers.js',
+            'head_http.js',
+            ]
+
+        head_paths = ['"%s"' % mozpath.join(unit_test_dir, path) for path in head_paths]
+
+        args = [
+            '%s/run-mozilla.sh' % self.bindir,
+            '%s/xpcshell' % self.bindir,
+            '-g', self.bindir,
+            '-a', self.bindir,
+            '-r', '%s/components/httpd.manifest' % self.bindir,
+            '-m',
+            '-s',
+            '-e', 'const _TESTING_MODULES_DIR = "%s/_tests/modules";' % topobjdir,
+            '-f', '%s/testing/xpcshell/head.js' % topsrcdir,
+            '-e', 'const _SERVER_ADDR = "%s";' % hostname,
+            '-e', 'const SERVER_PORT = "%s";' % port,
+            '-e', 'const INCLUDE_FILES = [%s];' % ', '.join(head_paths),
+            '-e', '_register_protocol_handlers();',
+            '-e', 'for (let name of INCLUDE_FILES) load(name);',
+            '-e', '_fakeIdleService.activate();',
+            '-f', js_file
+            ]
+
+        profile_dir = mkdtemp()
+        print 'Created profile directory: %s' % profile_dir
+
+        try:
+            env = {'XPCSHELL_TEST_PROFILE_DIR': profile_dir}
+            proc = Popen(args, env=env)
+
+            return proc.wait()
+
+        finally:
+            print 'Removing profile directory %s' % profile_dir
+            rmtree(profile_dir)
+
+    @Command('storage-server', category='services',
+             description='Run a storage server.')
+    @SyncStorageCommand
+    def run_storage_server(self, port=DEFAULT_PORT, address=DEFAULT_HOSTNAME):
+        exit(self.run_server(STORAGE_SERVER_SCRIPT, address, port))

services/common/tests/moz.build

@@ -0,0 +1,11 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += ['unit/xpcshell.ini']
+
+TEST_DIRS += [
+    'unit'
+]

services/common/tests/run_storage_server.js

@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file runs a Storage Service server.
+ *
+ * It is meant to be executed with an xpcshell.
+ *
+ * The Makefile in this directory contains a target to run it:
+ *
+ *   $ make storage-server
+ */
+
+Cu.import("resource://testing-common/services/common/storageserver.js");
+
+initTestLogging();
+
+var server = new StorageServer();
+server.allowAllUsers = true;
+server.startSynchronous(SERVER_PORT);
+_("Storage server started on port " + SERVER_PORT);
+
+// Launch the thread manager.
+_do_main();

services/common/tests/unit/head_global.js

@@ -0,0 +1,29 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu, manager: Cm} = Components;
+
+var gSyncProfile = do_get_profile();
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+Cu.import("resource://testing-common/AppInfo.jsm", this);
+updateAppInfo({
+  name: "XPCShell",
+  ID: "xpcshell@tests.mozilla.org",
+  version: "1",
+  platformVersion: "",
+});
+
+function addResourceAlias() {
+  Cu.import("resource://gre/modules/Services.jsm");
+  const handler = Services.io.getProtocolHandler("resource")
+                  .QueryInterface(Ci.nsIResProtocolHandler);
+
+  let modules = ["common", "crypto"];
+  for (let module of modules) {
+    let uri = Services.io.newURI("resource://gre/modules/services-" + module + "/",
+                                 null, null);
+    handler.setSubstitution("services-" + module, uri);
+  }
+}
+addResourceAlias();

services/common/tests/unit/head_helpers.js

@@ -0,0 +1,172 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://testing-common/httpd.js");
+Cu.import("resource://testing-common/services/common/logging.js");
+Cu.import("resource://testing-common/MockRegistrar.jsm");
+
+var btoa = Cu.import("resource://gre/modules/Log.jsm").btoa;
+var atob = Cu.import("resource://gre/modules/Log.jsm").atob;
+
+function do_check_empty(obj) {
+  do_check_attribute_count(obj, 0);
+}
+
+function do_check_attribute_count(obj, c) {
+  do_check_eq(c, Object.keys(obj).length);
+}
+
+function do_check_throws(aFunc, aResult, aStack) {
+  if (!aStack) {
+    try {
+      // We might not have a 'Components' object.
+      aStack = Components.stack.caller;
+    } catch (e) {}
+  }
+
+  try {
+    aFunc();
+  } catch (e) {
+    do_check_eq(e.result, aResult, aStack);
+    return;
+  }
+  do_throw("Expected result " + aResult + ", none thrown.", aStack);
+}
+
+
+/**
+ * Test whether specified function throws exception with expected
+ * result.
+ *
+ * @param func
+ *        Function to be tested.
+ * @param message
+ *        Message of expected exception. <code>null</code> for no throws.
+ */
+function do_check_throws_message(aFunc, aResult) {
+  try {
+    aFunc();
+  } catch (e) {
+    do_check_eq(e.message, aResult);
+    return;
+  }
+  do_throw("Expected an error, none thrown.");
+}
+
+/**
+ * Print some debug message to the console. All arguments will be printed,
+ * separated by spaces.
+ *
+ * @param [arg0, arg1, arg2, ...]
+ *        Any number of arguments to print out
+ * @usage _("Hello World") -> prints "Hello World"
+ * @usage _(1, 2, 3) -> prints "1 2 3"
+ */
+var _ = function(some, debug, text, to) {
+  print(Array.slice(arguments).join(" "));
+};
+
+function httpd_setup (handlers, port=-1) {
+  let server = new HttpServer();
+  for (let path in handlers) {
+    server.registerPathHandler(path, handlers[path]);
+  }
+  try {
+    server.start(port);
+  } catch (ex) {
+    _("==========================================");
+    _("Got exception starting HTTP server on port " + port);
+    _("Error: " + Log.exceptionStr(ex));
+    _("Is there a process already listening on port " + port + "?");
+    _("==========================================");
+    do_throw(ex);
+  }
+
+  // Set the base URI for convenience.
+  let i = server.identity;
+  server.baseURI = i.primaryScheme + "://" + i.primaryHost + ":" + i.primaryPort;
+
+  return server;
+}
+
+function httpd_handler(statusCode, status, body) {
+  return function handler(request, response) {
+    _("Processing request");
+    // Allow test functions to inspect the request.
+    request.body = readBytesFromInputStream(request.bodyInputStream);
+    handler.request = request;
+
+    response.setStatusLine(request.httpVersion, statusCode, status);
+    if (body) {
+      response.bodyOutputStream.write(body, body.length);
+    }
+  };
+}
+
+/*
+ * Read bytes string from an nsIInputStream.  If 'count' is omitted,
+ * all available input is read.
+ */
+function readBytesFromInputStream(inputStream, count) {
+  return CommonUtils.readBytesFromInputStream(inputStream, count);
+}
+
+/*
+ * Ensure exceptions from inside callbacks leads to test failures.
+ */
+function ensureThrows(func) {
+  return function() {
+    try {
+      func.apply(this, arguments);
+    } catch (ex) {
+      do_throw(ex);
+    }
+  };
+}
+
+/**
+ * Proxy auth helpers.
+ */
+
+/**
+ * Fake a PAC to prompt a channel replacement.
+ */
+var PACSystemSettings = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISystemProxySettings]),
+
+  // Replace this URI for each test to avoid caching. We want to ensure that
+  // each test gets a completely fresh setup.
+  mainThreadOnly: true,
+  PACURI: null,
+  getProxyForURI: function getProxyForURI(aURI) {
+    throw Cr.NS_ERROR_NOT_IMPLEMENTED;
+  }
+};
+
+var fakePACCID;
+function installFakePAC() {
+  _("Installing fake PAC.");
+  fakePACCID = MockRegistrar.register("@mozilla.org/system-proxy-settings;1",
+                                      PACSystemSettings);
+}
+
+function uninstallFakePAC() {
+  _("Uninstalling fake PAC.");
+  MockRegistrar.unregister(fakePACCID);
+}
+
+// Many tests do service.startOver() and don't expect the provider type to
+// change (whereas by default, a startOver will do exactly that so FxA is
+// subsequently used). The tests that know how to deal with
+// the Firefox Accounts identity hack things to ensure that still works.
+function ensureStartOverKeepsIdentity() {
+  Cu.import("resource://gre/modules/Services.jsm");
+  Services.prefs.setBoolPref("services.sync-testing.startOverKeepIdentity", true);
+  do_register_cleanup(function() {
+    Services.prefs.clearUserPref("services.sync-testing.startOverKeepIdentity");
+  });
+}
+ensureStartOverKeepsIdentity();

services/common/tests/unit/head_http.js

@@ -0,0 +1,29 @@
+ /* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function basic_auth_header(user, password) {
+  return "Basic " + btoa(user + ":" + CommonUtils.encodeUTF8(password));
+}
+
+function basic_auth_matches(req, user, password) {
+  if (!req.hasHeader("Authorization")) {
+    return false;
+  }
+
+  let expected = basic_auth_header(user, CommonUtils.encodeUTF8(password));
+  return req.getHeader("Authorization") == expected;
+}
+
+function httpd_basic_auth_handler(body, metadata, response) {
+  if (basic_auth_matches(metadata, "guest", "guest")) {
+    response.setStatusLine(metadata.httpVersion, 200, "OK, authorized");
+    response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false);
+  } else {
+    body = "This path exists and is protected - failed";
+    response.setStatusLine(metadata.httpVersion, 401, "Unauthorized");
+    response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false);
+  }
+  response.bodyOutputStream.write(body, body.length);
+}

services/common/tests/unit/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+TEST_DIRS += [
+    'test_blocklist_signatures'
+]

services/common/tests/unit/test_async_chain.js

@@ -0,0 +1,30 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+
+function run_test() {
+  _("Chain a few async methods, making sure the 'this' object is correct.");
+
+  let methods = {
+    save: function(x, callback) {
+      this.x = x;
+      callback(x);
+    },
+    addX: function(x, callback) {
+      callback(x + this.x);
+    },
+    double: function(x, callback) {
+      callback(x * 2);
+    },
+    neg: function(x, callback) {
+      callback(-x);
+    }
+  };
+  methods.chain = Async.chain;
+
+  // ((1 + 1 + 1) * (-1) + 1) * 2 + 1 = -3
+  methods.chain(methods.save, methods.addX, methods.addX, methods.neg,
+                methods.addX, methods.double, methods.addX, methods.save)(1);
+  do_check_eq(methods.x, -3);
+}

services/common/tests/unit/test_async_querySpinningly.js

@@ -0,0 +1,103 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/utils.js");
+
+_("Make sure querySpinningly will synchronously fetch rows for a query asyncly");
+
+const SQLITE_CONSTRAINT_VIOLATION = 19;  // http://www.sqlite.org/c3ref/c_abort.html
+
+var Svc = {};
+XPCOMUtils.defineLazyServiceGetter(Svc, "Form",
+                                   "@mozilla.org/satchel/form-history;1",
+                                   "nsIFormHistory2");
+
+function querySpinningly(query, names) {
+  let q = Svc.Form.DBConnection.createStatement(query);
+  let r = Async.querySpinningly(q, names);
+  q.finalize();
+  return r;
+}
+
+function run_test() {
+  initTestLogging("Trace");
+
+  _("Make sure the call is async and allows other events to process");
+  let isAsync = false;
+  CommonUtils.nextTick(function() { isAsync = true; });
+  do_check_false(isAsync);
+
+  _("Empty out the formhistory table");
+  let r0 = querySpinningly("DELETE FROM moz_formhistory");
+  do_check_eq(r0, null);
+
+  _("Make sure there's nothing there");
+  let r1 = querySpinningly("SELECT 1 FROM moz_formhistory");
+  do_check_eq(r1, null);
+
+  _("Insert a row");
+  let r2 = querySpinningly("INSERT INTO moz_formhistory (fieldname, value) VALUES ('foo', 'bar')");
+  do_check_eq(r2, null);
+
+  _("Request a known value for the one row");
+  let r3 = querySpinningly("SELECT 42 num FROM moz_formhistory", ["num"]);
+  do_check_eq(r3.length, 1);
+  do_check_eq(r3[0].num, 42);
+
+  _("Get multiple columns");
+  let r4 = querySpinningly("SELECT fieldname, value FROM moz_formhistory", ["fieldname", "value"]);
+  do_check_eq(r4.length, 1);
+  do_check_eq(r4[0].fieldname, "foo");
+  do_check_eq(r4[0].value, "bar");
+
+  _("Get multiple columns with a different order");
+  let r5 = querySpinningly("SELECT fieldname, value FROM moz_formhistory", ["value", "fieldname"]);
+  do_check_eq(r5.length, 1);
+  do_check_eq(r5[0].fieldname, "foo");
+  do_check_eq(r5[0].value, "bar");
+
+  _("Add multiple entries (sqlite doesn't support multiple VALUES)");
+  let r6 = querySpinningly("INSERT INTO moz_formhistory (fieldname, value) SELECT 'foo', 'baz' UNION SELECT 'more', 'values'");
+  do_check_eq(r6, null);
+
+  _("Get multiple rows");
+  let r7 = querySpinningly("SELECT fieldname, value FROM moz_formhistory WHERE fieldname = 'foo'", ["fieldname", "value"]);
+  do_check_eq(r7.length, 2);
+  do_check_eq(r7[0].fieldname, "foo");
+  do_check_eq(r7[1].fieldname, "foo");
+
+  _("Make sure updates work");
+  let r8 = querySpinningly("UPDATE moz_formhistory SET value = 'updated' WHERE fieldname = 'more'");
+  do_check_eq(r8, null);
+
+  _("Get the updated");
+  let r9 = querySpinningly("SELECT value, fieldname FROM moz_formhistory WHERE fieldname = 'more'", ["fieldname", "value"]);
+  do_check_eq(r9.length, 1);
+  do_check_eq(r9[0].fieldname, "more");
+  do_check_eq(r9[0].value, "updated");
+
+  _("Grabbing fewer fields than queried is fine");
+  let r10 = querySpinningly("SELECT value, fieldname FROM moz_formhistory", ["fieldname"]);
+  do_check_eq(r10.length, 3);
+
+  _("Generate an execution error");
+  let query = "INSERT INTO moz_formhistory (fieldname, value) VALUES ('one', NULL)";
+  let stmt = Svc.Form.DBConnection.createStatement(query);
+  let r11, except; ;
+  try {
+    r11 = Async.querySpinningly(stmt);
+  } catch(e) {
+    except = e;
+  }
+  stmt.finalize()
+  do_check_true(!!except);
+  do_check_eq(except.result, SQLITE_CONSTRAINT_VIOLATION);
+
+  _("Cleaning up");
+  querySpinningly("DELETE FROM moz_formhistory");
+
+  _("Make sure the timeout got to run before this function ends");
+  do_check_true(isAsync);
+}

services/common/tests/unit/test_blocklist_certificates.js

@@ -0,0 +1,224 @@
+const { Constructor: CC } = Components;
+
+Cu.import("resource://testing-common/httpd.js");
+
+const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+let server;
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let kintoClient;
+
+function do_get_kinto_collection(collectionName) {
+  if (!kintoClient) {
+    let config = {
+      // Set the remote to be some server that will cause test failure when
+      // hit since we should never hit the server directly, only via maybeSync()
+      remote: "https://firefox.settings.services.mozilla.com/v1/",
+      // Set up the adapter and bucket as normal
+      adapter: FirefoxAdapter,
+      bucket: "blocklists"
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection(collectionName);
+}
+
+// Some simple tests to demonstrate that the logic inside maybeSync works
+// correctly and that simple kinto operations are working as expected. There
+// are more tests for core Kinto.js (and its storage adapter) in the
+// xpcshell tests under /services/common
+add_task(function* test_something(){
+  const configPath = "/v1/";
+  const recordsPath = "/v1/buckets/blocklists/collections/certificates/records";
+
+  Services.prefs.setCharPref("services.settings.server",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // register a handler
+  function handleResponse (request, response) {
+    try {
+      const sample = getSampleResponse(request, server.identity.primaryPort);
+      if (!sample) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sample.status.status,
+                             sample.status.statusText);
+      // send the headers
+      for (let headerLine of sample.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sample.responseBody);
+    } catch (e) {
+      do_print(e);
+    }
+  }
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(recordsPath, handleResponse);
+
+  // Test an empty db populates
+  let result = yield OneCRLBlocklistClient.maybeSync(2000, Date.now());
+
+  // Open the collection, verify it's been populated:
+  // Our test data has a single record; it should be in the local collection
+  let collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  let list = yield collection.list();
+  do_check_eq(list.data.length, 1);
+  yield collection.db.close();
+
+  // Test the db is updated when we call again with a later lastModified value
+  result = yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
+
+  // Open the collection, verify it's been updated:
+  // Our test data now has two records; both should be in the local collection
+  collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  list = yield collection.list();
+  do_check_eq(list.data.length, 3);
+  yield collection.db.close();
+
+  // Try to maybeSync with the current lastModified value - no connection
+  // should be attempted.
+  // Clear the kinto base pref so any connections will cause a test failure
+  Services.prefs.clearUserPref("services.settings.server");
+  yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
+
+  // Try again with a lastModified value at some point in the past
+  yield OneCRLBlocklistClient.maybeSync(3000, Date.now());
+
+  // Check the OneCRL check time pref is modified, even if the collection
+  // hasn't changed
+  Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
+  yield OneCRLBlocklistClient.maybeSync(3000, Date.now());
+  let newValue = Services.prefs.getIntPref("services.blocklist.onecrl.checked");
+  do_check_neq(newValue, 0);
+
+  // Check that a sync completes even when there's bad data in the
+  // collection. This will throw on fail, so just calling maybeSync is an
+  // acceptible test.
+  Services.prefs.setCharPref("services.settings.server",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+  yield OneCRLBlocklistClient.maybeSync(5000, Date.now());
+});
+
+function run_test() {
+  // Ensure that signature verification is disabled to prevent interference
+  // with basic certificate sync tests
+  Services.prefs.setBoolPref("services.blocklist.signing.enforced", false);
+
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(() => { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName": "MEQxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xHjAcBgNVBAMTFXRoYXd0ZSBFViBTU0wgQ0EgLSBHMw==",
+        "serialNumber":"CrTHPEE6AZSfI3jysin2bA==",
+        "id":"78cf8900-fdea-4ce5-f8fb-b78710617718",
+        "last_modified":3000
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName":"MFkxCzAJBgNVBAYTAk5MMR4wHAYDVQQKExVTdGFhdCBkZXIgTmVkZXJsYW5kZW4xKjAoBgNVBAMTIVN0YWF0IGRlciBOZWRlcmxhbmRlbiBPdmVyaGVpZCBDQQ",
+        "serialNumber":"ATFpsA==",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02c",
+        "last_modified":4000
+      },{
+        "subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
+        "pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02d",
+        "last_modified":4000
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"5000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName":"not a base64 encoded issuer",
+        "serialNumber":"not a base64 encoded serial",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02e",
+        "last_modified":5000
+      },{
+        "subject":"not a base64 encoded subject",
+        "pubKeyHash":"not a base64 encoded pubKeyHash",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02f",
+        "last_modified":5000
+      },{
+        "subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
+        "pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02g",
+        "last_modified":5000
+      }]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/test_blocklist_clients.js

@@ -0,0 +1,412 @@
+const { Constructor: CC } = Components;
+
+const KEY_PROFILEDIR = "ProfD";
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://testing-common/httpd.js");
+Cu.import("resource://gre/modules/Timer.jsm");
+const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm");
+const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
+
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const BlocklistClients = Cu.import("resource://services-common/blocklist-clients.js");
+
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+const gBlocklistClients = [
+  {client: BlocklistClients.AddonBlocklistClient, filename: BlocklistClients.FILENAME_ADDONS_JSON, testData: ["i808","i720", "i539"]},
+  {client: BlocklistClients.PluginBlocklistClient, filename: BlocklistClients.FILENAME_PLUGINS_JSON, testData: ["p1044","p32","p28"]},
+  {client: BlocklistClients.GfxBlocklistClient, filename: BlocklistClients.FILENAME_GFX_JSON, testData: ["g204","g200","g36"]},
+];
+
+
+let server;
+let kintoClient;
+
+function kintoCollection(collectionName) {
+  if (!kintoClient) {
+    const Kinto = loadKinto();
+    const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+    const config = {
+      // Set the remote to be some server that will cause test failure when
+      // hit since we should never hit the server directly, only via maybeSync()
+      remote: "https://firefox.settings.services.mozilla.com/v1/",
+      adapter: FirefoxAdapter,
+      bucket: "blocklists"
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection(collectionName);
+}
+
+function* readJSON(filepath) {
+  const binaryData = yield OS.File.read(filepath);
+  const textData = (new TextDecoder()).decode(binaryData);
+  return Promise.resolve(JSON.parse(textData));
+}
+
+function* clear_state() {
+  for (let {client} of gBlocklistClients) {
+    // Remove last server times.
+    Services.prefs.clearUserPref(client.lastCheckTimePref);
+
+    // Clear local DB.
+    const collection = kintoCollection(client.collectionName);
+    try {
+      yield collection.db.open();
+      yield collection.clear();
+    } finally {
+      yield collection.db.close();
+    }
+  }
+
+  // Remove profile data.
+  for (let {filename} of gBlocklistClients) {
+    const blocklist = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    if (blocklist.exists()) {
+      blocklist.remove(true);
+    }
+  }
+}
+
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  // Point the blocklist clients to use this local HTTP server.
+  Services.prefs.setCharPref("services.settings.server",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // Setup server fake responses.
+  function handleResponse(request, response) {
+    try {
+      const sample = getSampleResponse(request, server.identity.primaryPort);
+      if (!sample) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sample.status.status,
+                             sample.status.statusText);
+      // send the headers
+      for (let headerLine of sample.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sample.responseBody);
+      response.finish();
+    } catch (e) {
+      do_print(e);
+    }
+  }
+  const configPath = "/v1/";
+  const addonsRecordsPath  = "/v1/buckets/blocklists/collections/addons/records";
+  const gfxRecordsPath     = "/v1/buckets/blocklists/collections/gfx/records";
+  const pluginsRecordsPath = "/v1/buckets/blocklists/collections/plugins/records";
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(addonsRecordsPath, handleResponse);
+  server.registerPathHandler(gfxRecordsPath, handleResponse);
+  server.registerPathHandler(pluginsRecordsPath, handleResponse);
+
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(() => { });
+  });
+}
+
+add_task(function* test_records_obtained_from_server_are_stored_in_db(){
+  for (let {client} of gBlocklistClients) {
+    // Test an empty db populates
+    let result = yield client.maybeSync(2000, Date.now());
+
+    // Open the collection, verify it's been populated:
+    // Our test data has a single record; it should be in the local collection
+    let collection = kintoCollection(client.collectionName);
+    yield collection.db.open();
+    let list = yield collection.list();
+    equal(list.data.length, 1);
+    yield collection.db.close();
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_list_is_written_to_file_in_profile(){
+  for (let {client, filename, testData} of gBlocklistClients) {
+    const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    strictEqual(profFile.exists(), false);
+
+    let result = yield client.maybeSync(2000, Date.now());
+
+    strictEqual(profFile.exists(), true);
+    const content = yield readJSON(profFile.path);
+    equal(content.data[0].blockID, testData[testData.length - 1]);
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_current_server_time_is_saved_in_pref(){
+  for (let {client} of gBlocklistClients) {
+    const before = Services.prefs.getIntPref(client.lastCheckTimePref);
+    const serverTime = Date.now();
+    yield client.maybeSync(2000, serverTime);
+    const after = Services.prefs.getIntPref(client.lastCheckTimePref);
+    equal(after, Math.round(serverTime / 1000));
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_update_json_file_when_addons_has_changes(){
+  for (let {client, filename, testData} of gBlocklistClients) {
+    yield client.maybeSync(2000, Date.now() - 1000);
+    const before = Services.prefs.getIntPref(client.lastCheckTimePref);
+    const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
+    const serverTime = Date.now();
+
+    yield client.maybeSync(3001, serverTime);
+
+    // File was updated.
+    notEqual(fileLastModified, profFile.lastModifiedTime);
+    const content = yield readJSON(profFile.path);
+    deepEqual(content.data.map((r) => r.blockID), testData);
+    // Server time was updated.
+    const after = Services.prefs.getIntPref(client.lastCheckTimePref);
+    equal(after, Math.round(serverTime / 1000));
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_sends_reload_message_when_blocklist_has_changes(){
+  for (let {client, filename} of gBlocklistClients) {
+    let received = yield new Promise((resolve, reject) => {
+      Services.ppmm.addMessageListener("Blocklist:reload-from-disk", {
+        receiveMessage(aMsg) { resolve(aMsg) }
+      });
+
+      client.maybeSync(2000, Date.now() - 1000);
+    });
+
+    equal(received.data.filename, filename);
+  }
+});
+add_task(clear_state);
+
+add_task(function* test_do_nothing_when_blocklist_is_up_to_date(){
+  for (let {client, filename} of gBlocklistClients) {
+    yield client.maybeSync(2000, Date.now() - 1000);
+    const before = Services.prefs.getIntPref(client.lastCheckTimePref);
+    const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
+    const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
+    const serverTime = Date.now();
+
+    yield client.maybeSync(3000, serverTime);
+
+    // File was not updated.
+    equal(fileLastModified, profFile.lastModifiedTime);
+    // Server time was updated.
+    const after = Services.prefs.getIntPref(client.lastCheckTimePref);
+    equal(after, Math.round(serverTime / 1000));
+  }
+});
+add_task(clear_state);
+
+
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "prefs": [],
+        "blockID": "i539",
+        "last_modified": 3000,
+        "versionRange": [{
+          "targetApplication": [],
+          "maxVersion": "*",
+          "minVersion": "0",
+          "severity": "1"
+        }],
+        "guid": "ScorpionSaver@jetpack",
+        "id": "9d500963-d80e-3a91-6e74-66f3811b99cc"
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/plugins/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "matchFilename": "NPFFAddOn.dll",
+        "blockID": "p28",
+        "id": "7b1e0b3c-e390-a817-11b6-a6887f65f56e",
+        "last_modified": 3000,
+        "versionRange": []
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/gfx/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "driverVersionComparator": "LESS_THAN_OR_EQUAL",
+        "driverVersion": "8.17.12.5896",
+        "vendor": "0x10de",
+        "blockID": "g36",
+        "feature": "DIRECT3D_9_LAYERS",
+        "devices": ["0x0a6c"],
+        "featureStatus": "BLOCKED_DRIVER_VERSION",
+        "last_modified": 3000,
+        "os": "WINNT 6.1",
+        "id": "3f947f16-37c2-4e96-d356-78b26363729b"
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "prefs": [],
+        "blockID": "i808",
+        "last_modified": 4000,
+        "versionRange": [{
+          "targetApplication": [],
+          "maxVersion": "*",
+          "minVersion": "0",
+          "severity": "3"
+        }],
+        "guid": "{c96d1ae6-c4cf-4984-b110-f5f561b33b5a}",
+        "id": "9ccfac91-e463-c30c-f0bd-14143794a8dd"
+      }, {
+        "prefs": ["browser.startup.homepage"],
+        "blockID": "i720",
+        "last_modified": 3500,
+        "versionRange": [{
+          "targetApplication": [],
+          "maxVersion": "*",
+          "minVersion": "0",
+          "severity": "1"
+        }],
+        "guid": "FXqG@xeeR.net",
+        "id": "cf9b3129-a97e-dbd7-9525-a8575ac03c25"
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/plugins/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "infoURL": "https://get.adobe.com/flashplayer/",
+        "blockID": "p1044",
+        "matchFilename": "libflashplayer\\.so",
+        "last_modified": 4000,
+        "versionRange": [{
+          "targetApplication": [],
+          "minVersion": "11.2.202.509",
+          "maxVersion": "11.2.202.539",
+          "severity": "0",
+          "vulnerabilityStatus": "1"
+        }],
+        "os": "Linux",
+        "id": "aabad965-e556-ffe7-4191-074f5dee3df3"
+      }, {
+        "matchFilename": "npViewpoint.dll",
+        "blockID": "p32",
+        "id": "1f48af42-c508-b8ef-b8d5-609d48e4f6c9",
+        "last_modified": 3500,
+        "versionRange": [{
+          "targetApplication": [{
+            "minVersion": "3.0",
+            "guid": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
+            "maxVersion": "*"
+          }]
+        }]
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/gfx/records?_sort=-last_modified&_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "vendor": "0x8086",
+        "blockID": "g204",
+        "feature": "WEBGL_MSAA",
+        "devices": [],
+        "id": "c96bca82-e6bd-044d-14c4-9c1d67e9283a",
+        "last_modified": 4000,
+        "os": "Darwin 10",
+        "featureStatus": "BLOCKED_DEVICE"
+      }, {
+        "vendor": "0x10de",
+        "blockID": "g200",
+        "feature": "WEBGL_MSAA",
+        "devices": [],
+        "id": "c3a15ba9-e0e2-421f-e399-c995e5b8d14e",
+        "last_modified": 3500,
+        "os": "Darwin 11",
+        "featureStatus": "BLOCKED_DEVICE"
+      }]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/test_blocklist_signatures.js

@@ -0,0 +1,510 @@
+"use strict";
+
+Cu.import("resource://services-common/blocklist-updater.js");
+Cu.import("resource://testing-common/httpd.js");
+
+const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
+const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
+
+let server;
+
+const PREF_BLOCKLIST_BUCKET            = "services.blocklist.bucket";
+const PREF_BLOCKLIST_ENFORCE_SIGNING   = "services.blocklist.signing.enforced";
+const PREF_BLOCKLIST_ONECRL_COLLECTION = "services.blocklist.onecrl.collection";
+const PREF_SETTINGS_SERVER             = "services.settings.server";
+const PREF_SIGNATURE_ROOT              = "security.content.signature.root_hash";
+
+
+const CERT_DIR = "test_blocklist_signatures/";
+const CHAIN_FILES =
+    ["collection_signing_ee.pem",
+     "collection_signing_int.pem",
+     "collection_signing_root.pem"];
+
+function getFileData(file) {
+  const stream = Cc["@mozilla.org/network/file-input-stream;1"]
+                   .createInstance(Ci.nsIFileInputStream);
+  stream.init(file, -1, 0, 0);
+  const data = NetUtil.readInputStreamToString(stream, stream.available());
+  stream.close();
+  return data;
+}
+
+function setRoot() {
+  const filename = CERT_DIR + CHAIN_FILES[0];
+
+  const certFile = do_get_file(filename, false);
+  const b64cert = getFileData(certFile)
+                    .replace(/-----BEGIN CERTIFICATE-----/, "")
+                    .replace(/-----END CERTIFICATE-----/, "")
+                    .replace(/[\r\n]/g, "");
+  const certdb = Cc["@mozilla.org/security/x509certdb;1"]
+                   .getService(Ci.nsIX509CertDB);
+  const cert = certdb.constructX509FromBase64(b64cert);
+  Services.prefs.setCharPref(PREF_SIGNATURE_ROOT, cert.sha256Fingerprint);
+}
+
+function getCertChain() {
+  const chain = [];
+  for (let file of CHAIN_FILES) {
+    chain.push(getFileData(do_get_file(CERT_DIR + file)));
+  }
+  return chain.join("\n");
+}
+
+function* checkRecordCount(count) {
+  // open the collection manually
+  const base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+  const bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
+  const collectionName =
+      Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION);
+
+  const Kinto = loadKinto();
+
+  const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+
+  const config = {
+    remote: base,
+    bucket: bucket,
+    adapter: FirefoxAdapter,
+  };
+
+  const db = new Kinto(config);
+  const collection = db.collection(collectionName);
+
+  yield collection.db.open();
+
+  // Check we have the expected number of records
+  let records = yield collection.list();
+  do_check_eq(count, records.data.length);
+
+  // Close the collection so the test can exit cleanly
+  yield collection.db.close();
+}
+
+// Check to ensure maybeSync is called with correct values when a changes
+// document contains information on when a collection was last modified
+add_task(function* test_check_signatures(){
+  const port = server.identity.primaryPort;
+
+  // a response to give the client when the cert chain is expected
+  function makeMetaResponseBody(lastModified, signature) {
+    return {
+      data: {
+        id: "certificates",
+        last_modified: lastModified,
+        signature: {
+          x5u: `http://localhost:${port}/test_blocklist_signatures/test_cert_chain.pem`,
+          public_key: "fake",
+          "content-signature": `x5u=http://localhost:${port}/test_blocklist_signatures/test_cert_chain.pem;p384ecdsa=${signature}`,
+          signature_encoding: "rs_base64url",
+          signature: signature,
+          hash_algorithm: "sha384",
+          ref: "1yryrnmzou5rf31ou80znpnq8n"
+        }
+      }
+    };
+  }
+
+  function makeMetaResponse(eTag, body, comment) {
+    return {
+      comment: comment,
+      sampleHeaders: [
+        "Content-Type: application/json; charset=UTF-8",
+        `ETag: \"${eTag}\"`
+      ],
+      status: {status: 200, statusText: "OK"},
+      responseBody: JSON.stringify(body)
+    };
+  }
+
+  function registerHandlers(responses){
+    function handleResponse (serverTimeMillis, request, response) {
+      const key = `${request.method}:${request.path}?${request.queryString}`;
+      const available = responses[key];
+      const sampled = available.length > 1 ? available.shift() : available[0];
+
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                            sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+
+      // set the server date
+      response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
+
+      response.write(sampled.responseBody);
+    }
+
+    for (let key of Object.keys(responses)) {
+      const keyParts = key.split(":");
+      const method = keyParts[0];
+      const valueParts = keyParts[1].split("?");
+      const path = valueParts[0];
+
+      server.registerPathHandler(path, handleResponse.bind(null, 2000));
+    }
+  }
+
+  // First, perform a signature verification with known data and signature
+  // to ensure things are working correctly
+  let verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
+                   .createInstance(Ci.nsIContentSignatureVerifier);
+
+  const emptyData = '[]';
+  const emptySignature = "p384ecdsa=zbugm2FDitsHwk5-IWsas1PpWwY29f0Fg5ZHeqD8fzep7AVl2vfcaHA7LdmCZ28qZLOioGKvco3qT117Q4-HlqFTJM7COHzxGyU2MMJ0ZTnhJrPOC1fP3cVQjU1PTWi9";
+  const name = "onecrl.content-signature.mozilla.org";
+  ok(verifier.verifyContentSignature(emptyData, emptySignature,
+                                     getCertChain(), name));
+
+  verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
+               .createInstance(Ci.nsIContentSignatureVerifier);
+
+  const collectionData = '[{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:43:37Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"97fbf7c4-3ef2-f54f-0029-1ba6540c63ea","issuerName":"MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==","last_modified":2000,"serialNumber":"BAAAAAABA/A35EU="},{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:48:11Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc","issuerName":"MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB","last_modified":3000,"serialNumber":"BAAAAAABI54PryQ="}]';
+  const collectionSignature = "p384ecdsa=f4pA2tYM5jQgWY6YUmhUwQiBLj6QO5sHLD_5MqLePz95qv-7cNCuQoZnPQwxoptDtW8hcWH3kLb0quR7SB-r82gkpR9POVofsnWJRA-ETb0BcIz6VvI3pDT49ZLlNg3p";
+
+  ok(verifier.verifyContentSignature(collectionData, collectionSignature, getCertChain(), name));
+
+  // set up prefs so the kinto updater talks to the test server
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // Set up some data we need for our test
+  let startTime = Date.now();
+
+  // These are records we'll use in the test collections
+  const RECORD1 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:43:37Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: "."
+    },
+    enabled: true,
+    id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
+    issuerName: "MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==",
+    last_modified: 2000,
+    serialNumber: "BAAAAAABA/A35EU="
+  };
+
+  const RECORD2 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:48:11Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: "."
+    },
+    enabled: true,
+    id: "e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc",
+    issuerName: "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
+    last_modified: 3000,
+    serialNumber: "BAAAAAABI54PryQ="
+  };
+
+  const RECORD3 = {
+    details: {
+      bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
+      created: "2016-01-18T14:48:11Z",
+      name: "GlobalSign certs",
+      who: ".",
+      why: "."
+    },
+    enabled: true,
+    id: "c7c49b69-a4ab-418e-92a9-e1961459aa7f",
+    issuerName: "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
+    last_modified: 4000,
+    serialNumber: "BAAAAAABI54PryQ="
+  };
+
+  const RECORD1_DELETION = {
+    deleted: true,
+    enabled: true,
+    id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
+    last_modified: 3500,
+  };
+
+  // Check that a signature on an empty collection is OK
+  // We need to set up paths on the HTTP server to return specific data from
+  // specific paths for each test. Here we prepare data for each response.
+
+  // A cert chain response (this the cert chain that contains the signing
+  // cert, the root and any intermediates in between). This is used in each
+  // sync.
+  const RESPONSE_CERT_CHAIN = {
+    comment: "RESPONSE_CERT_CHAIN",
+    sampleHeaders: [
+      "Content-Type: text/plain; charset=UTF-8"
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: getCertChain()
+  };
+
+  // A server settings response. This is used in each sync.
+  const RESPONSE_SERVER_SETTINGS = {
+    comment: "RESPONSE_SERVER_SETTINGS",
+    sampleHeaders: [
+      "Access-Control-Allow-Origin: *",
+      "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+      "Content-Type: application/json; charset=UTF-8",
+      "Server: waitress"
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+  };
+
+  // This is the initial, empty state of the collection. This is only used
+  // for the first sync.
+  const RESPONSE_EMPTY_INITIAL = {
+    comment: "RESPONSE_EMPTY_INITIAL",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"1000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": []})
+  };
+
+  const RESPONSE_BODY_META_EMPTY_SIG = makeMetaResponseBody(1000,
+    "vxuAg5rDCB-1pul4a91vqSBQRXJG_j7WOYUTswxRSMltdYmbhLRH8R8brQ9YKuNDF56F-w6pn4HWxb076qgKPwgcEBtUeZAO_RtaHXRkRUUgVzAr86yQL4-aJTbv3D6u");
+
+  // The collection metadata containing the signature for the empty
+  // collection.
+  const RESPONSE_META_EMPTY_SIG =
+    makeMetaResponse(1000, RESPONSE_BODY_META_EMPTY_SIG,
+                     "RESPONSE_META_EMPTY_SIG");
+
+  // Here, we map request method and path to the available responses
+  const emptyCollectionResponses = {
+    "GET:/test_blocklist_signatures/test_cert_chain.pem?":[RESPONSE_CERT_CHAIN],
+    "GET:/v1/?": [RESPONSE_SERVER_SETTINGS],
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified":
+      [RESPONSE_EMPTY_INITIAL],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_EMPTY_SIG]
+  };
+
+  // .. and use this map to register handlers for each path
+  registerHandlers(emptyCollectionResponses);
+
+  // With all of this set up, we attempt a sync. This will resolve if all is
+  // well and throw if something goes wrong.
+  yield OneCRLBlocklistClient.maybeSync(1000, startTime);
+
+  // Check that some additions (2 records) to the collection have a valid
+  // signature.
+
+  // This response adds two entries (RECORD1 and RECORD2) to the collection
+  const RESPONSE_TWO_ADDED = {
+    comment: "RESPONSE_TWO_ADDED",
+    sampleHeaders: [
+        "Content-Type: application/json; charset=UTF-8",
+        "ETag: \"3000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD2, RECORD1]})
+  };
+
+  const RESPONSE_BODY_META_TWO_ITEMS_SIG = makeMetaResponseBody(3000,
+    "dwhJeypadNIyzGj3QdI0KMRTPnHhFPF_j73mNrsPAHKMW46S2Ftf4BzsPMvPMB8h0TjDus13wo_R4l432DHe7tYyMIWXY0PBeMcoe5BREhFIxMxTsh9eGVXBD1e3UwRy");
+
+  // A signature response for the collection containg RECORD1 and RECORD2
+  const RESPONSE_META_TWO_ITEMS_SIG =
+    makeMetaResponse(3000, RESPONSE_BODY_META_TWO_ITEMS_SIG,
+                     "RESPONSE_META_TWO_ITEMS_SIG");
+
+  const twoItemsResponses = {
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=1000":
+      [RESPONSE_TWO_ADDED],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_TWO_ITEMS_SIG]
+  };
+  registerHandlers(twoItemsResponses);
+  yield OneCRLBlocklistClient.maybeSync(3000, startTime);
+
+  // Check the collection with one addition and one removal has a valid
+  // signature
+
+  // Remove RECORD1, add RECORD3
+  const RESPONSE_ONE_ADDED_ONE_REMOVED = {
+    comment: "RESPONSE_ONE_ADDED_ONE_REMOVED ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD3, RECORD1_DELETION]})
+  };
+
+  const RESPONSE_BODY_META_THREE_ITEMS_SIG = makeMetaResponseBody(4000,
+    "MIEmNghKnkz12UodAAIc3q_Y4a3IJJ7GhHF4JYNYmm8avAGyPM9fYU7NzVo94pzjotG7vmtiYuHyIX2rTHTbT587w0LdRWxipgFd_PC1mHiwUyjFYNqBBG-kifYk7kEw");
+
+  // signature response for the collection containing RECORD2 and RECORD3
+  const RESPONSE_META_THREE_ITEMS_SIG =
+    makeMetaResponse(4000, RESPONSE_BODY_META_THREE_ITEMS_SIG,
+                     "RESPONSE_META_THREE_ITEMS_SIG");
+
+  const oneAddedOneRemovedResponses = {
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=3000":
+      [RESPONSE_ONE_ADDED_ONE_REMOVED],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_THREE_ITEMS_SIG]
+  };
+  registerHandlers(oneAddedOneRemovedResponses);
+  yield OneCRLBlocklistClient.maybeSync(4000, startTime);
+
+  // Check the signature is still valid with no operation (no changes)
+
+  // Leave the collection unchanged
+  const RESPONSE_EMPTY_NO_UPDATE = {
+    comment: "RESPONSE_EMPTY_NO_UPDATE ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": []})
+  };
+
+  const noOpResponses = {
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_THREE_ITEMS_SIG]
+  };
+  registerHandlers(noOpResponses);
+  yield OneCRLBlocklistClient.maybeSync(4100, startTime);
+
+  // Check the collection is reset when the signature is invalid
+
+  // Prepare a (deliberately) bad signature to check the collection state is
+  // reset if something is inconsistent
+  const RESPONSE_COMPLETE_INITIAL = {
+    comment: "RESPONSE_COMPLETE_INITIAL ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD2, RECORD3]})
+  };
+
+  const RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID = {
+    comment: "RESPONSE_COMPLETE_INITIAL ",
+    sampleHeaders: [
+      "Content-Type: application/json; charset=UTF-8",
+      "ETag: \"4000\""
+    ],
+    status: {status: 200, statusText: "OK"},
+    responseBody: JSON.stringify({"data": [RECORD3, RECORD2]})
+  };
+
+  const RESPONSE_BODY_META_BAD_SIG = makeMetaResponseBody(4000,
+      "aW52YWxpZCBzaWduYXR1cmUK");
+
+  const RESPONSE_META_BAD_SIG =
+      makeMetaResponse(4000, RESPONSE_BODY_META_BAD_SIG, "RESPONSE_META_BAD_SIG");
+
+  const badSigGoodSigResponses = {
+    // In this test, we deliberately serve a bad signature initially. The
+    // subsequent signature returned is a valid one for the three item
+    // collection.
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_BAD_SIG, RESPONSE_META_THREE_ITEMS_SIG],
+    // The first collection state is the three item collection (since
+    // there's a sync with no updates) - but, since the signature is wrong,
+    // another request will be made...
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    // The next request is for the full collection. This will be checked
+    // against the valid signature - so the sync should succeed.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified":
+      [RESPONSE_COMPLETE_INITIAL],
+    // The next request is for the full collection sorted by id. This will be
+    // checked against the valid signature - so the sync should succeed.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
+      [RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID]
+  };
+
+  registerHandlers(badSigGoodSigResponses);
+  yield OneCRLBlocklistClient.maybeSync(5000, startTime);
+
+  const badSigGoodOldResponses = {
+    // In this test, we deliberately serve a bad signature initially. The
+    // subsequent sitnature returned is a valid one for the three item
+    // collection.
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_BAD_SIG, RESPONSE_META_EMPTY_SIG],
+    // The first collection state is the current state (since there's no update
+    // - but, since the signature is wrong, another request will be made)
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    // The next request is for the full collection sorted by id. This will be
+    // checked against the valid signature and last_modified times will be
+    // compared. Sync should fail, even though the signature is good,
+    // because the local collection is newer.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
+      [RESPONSE_EMPTY_INITIAL],
+  };
+
+  // ensure our collection hasn't been replaced with an older, empty one
+  yield checkRecordCount(2);
+
+  registerHandlers(badSigGoodOldResponses);
+  yield OneCRLBlocklistClient.maybeSync(5000, startTime);
+
+  const allBadSigResponses = {
+    // In this test, we deliberately serve only a bad signature.
+    "GET:/v1/buckets/blocklists/collections/certificates?":
+      [RESPONSE_META_BAD_SIG],
+    // The first collection state is the three item collection (since
+    // there's a sync with no updates) - but, since the signature is wrong,
+    // another request will be made...
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
+      [RESPONSE_EMPTY_NO_UPDATE],
+    // The next request is for the full collection sorted by id. This will be
+    // checked against the valid signature - so the sync should succeed.
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
+      [RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID]
+  };
+
+  registerHandlers(allBadSigResponses);
+  try {
+    yield OneCRLBlocklistClient.maybeSync(6000, startTime);
+    do_throw("Sync should fail (the signature is intentionally bad)");
+  } catch (e) {
+    yield checkRecordCount(2);
+  }
+});
+
+function run_test() {
+  // ensure signatures are enforced
+  Services.prefs.setBoolPref(PREF_BLOCKLIST_ENFORCE_SIGNING, true);
+
+  // get a signature verifier to ensure nsNSSComponent is initialized
+  Cc["@mozilla.org/security/contentsignatureverifier;1"]
+    .createInstance(Ci.nsIContentSignatureVerifier);
+
+  // set the content signing root to our test root
+  setRoot();
+
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+

services/common/tests/unit/test_blocklist_signatures/collection_signing_ee.pem.certspec

@@ -0,0 +1,5 @@
+issuer:collection-signer-int-CA
+subject:collection-signer-ee-int-CA
+subjectKey:secp384r1
+extension:extKeyUsage:codeSigning
+extension:subjectAlternativeName:onecrl.content-signature.mozilla.org

services/common/tests/unit/test_blocklist_signatures/collection_signing_int.pem.certspec

@@ -0,0 +1,4 @@
+issuer:collection-signer-ca
+subject:collection-signer-int-CA
+extension:basicConstraints:cA,
+extension:extKeyUsage:codeSigning

services/common/tests/unit/test_blocklist_signatures/collection_signing_root.pem.certspec

@@ -0,0 +1,4 @@
+issuer:collection-signer-ca
+subject:collection-signer-ca
+extension:basicConstraints:cA,
+extension:extKeyUsage:codeSigning

services/common/tests/unit/test_blocklist_signatures/moz.build

@@ -0,0 +1,14 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+test_certificates = (
+    'collection_signing_root.pem',
+    'collection_signing_int.pem',
+    'collection_signing_ee.pem',
+)
+
+for test_certificate in test_certificates:
+    GeneratedTestCertificate(test_certificate)

services/common/tests/unit/test_blocklist_updater.js

@@ -0,0 +1,173 @@
+Cu.import("resource://testing-common/httpd.js");
+
+var server;
+
+const PREF_SETTINGS_SERVER = "services.settings.server";
+const PREF_LAST_UPDATE = "services.blocklist.last_update_seconds";
+const PREF_LAST_ETAG = "services.blocklist.last_etag";
+const PREF_CLOCK_SKEW_SECONDS = "services.blocklist.clock_skew_seconds";
+
+// Check to ensure maybeSync is called with correct values when a changes
+// document contains information on when a collection was last modified
+add_task(function* test_check_maybeSync(){
+  const changesPath = "/v1/buckets/monitor/collections/changes/records";
+
+  // register a handler
+  function handleResponse (serverTimeMillis, request, response) {
+    try {
+      const sampled = getSampleResponse(request, server.identity.primaryPort);
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                             sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+
+      // set the server date
+      response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
+
+      response.write(sampled.responseBody);
+    } catch (e) {
+      dump(`${e}\n`);
+    }
+  }
+
+  server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
+
+  // set up prefs so the kinto updater talks to the test server
+  Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
+    `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // set some initial values so we can check these are updated appropriately
+  Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
+  Services.prefs.setIntPref(PREF_CLOCK_SKEW_SECONDS, 0);
+  Services.prefs.clearUserPref(PREF_LAST_ETAG);
+
+
+  let startTime = Date.now();
+
+  let updater = Cu.import("resource://services-common/blocklist-updater.js");
+
+  let syncPromise = new Promise(function(resolve, reject) {
+    // add a test kinto client that will respond to lastModified information
+    // for a collection called 'test-collection'
+    updater.addTestBlocklistClient("test-collection", {
+      maybeSync(lastModified, serverTime) {
+        do_check_eq(lastModified, 1000);
+        do_check_eq(serverTime, 2000);
+        resolve();
+      }
+    });
+    updater.checkVersions();
+  });
+
+  // ensure we get the maybeSync call
+  yield syncPromise;
+
+  // check the last_update is updated
+  do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
+
+  // How does the clock difference look?
+  let endTime = Date.now();
+  let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to 2 (seconds past epoch)
+  do_check_true(clockDifference <= endTime / 1000
+              && clockDifference >= Math.floor(startTime / 1000) - 2);
+  // Last timestamp was saved. An ETag header value is a quoted string.
+  let lastEtag = Services.prefs.getCharPref(PREF_LAST_ETAG);
+  do_check_eq(lastEtag, "\"1100\"");
+
+  // Simulate a poll with up-to-date collection.
+  Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
+  // If server has no change, a 304 is received, maybeSync() is not called.
+  updater.addTestBlocklistClient("test-collection", {
+    maybeSync: () => {throw new Error("Should not be called");}
+  });
+  yield updater.checkVersions();
+  // Last update is overwritten
+  do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
+
+
+  // Simulate a server error.
+  function simulateErrorResponse (request, response) {
+    response.setHeader("Date", (new Date(3000)).toUTCString());
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.write(JSON.stringify({
+      code: 503,
+      errno: 999,
+      error: "Service Unavailable",
+    }));
+    response.setStatusLine(null, 503, "Service Unavailable");
+  }
+  server.registerPathHandler(changesPath, simulateErrorResponse);
+  // checkVersions() fails with adequate error.
+  let error;
+  try {
+    yield updater.checkVersions();
+  } catch (e) {
+    error = e;
+  }
+  do_check_eq(error.message, "Polling for changes failed.");
+  // When an error occurs, last update was not overwritten (see Date header above).
+  do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
+
+  // check negative clock skew times
+
+  // set to a time in the future
+  server.registerPathHandler(changesPath, handleResponse.bind(null, Date.now() + 10000));
+
+  yield updater.checkVersions();
+
+  clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
+  // we previously set the serverTime to Date.now() + 10000 ms past epoch
+  do_check_true(clockDifference <= 0 && clockDifference >= -10);
+});
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "GET:/v1/buckets/monitor/collections/changes/records?": {
+      "sampleHeaders": [
+        "Content-Type: application/json; charset=UTF-8",
+        "ETag: \"1100\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data": [{
+        "host": "localhost",
+        "last_modified": 1100,
+        "bucket": "blocklists:aurora",
+        "id": "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
+        "collection": "test-collection"
+      }, {
+        "host": "localhost",
+        "last_modified": 1000,
+        "bucket": "blocklists",
+        "id": "254cbb9e-6888-4d9f-8e60-58b74faa8778",
+        "collection": "test-collection"
+      }]})
+    }
+  };
+
+  if (req.hasHeader("if-none-match") && req.getHeader("if-none-match", "") == "\"1100\"")
+    return {sampleHeaders: [], status: {status: 304, statusText: "Not Modified"}, responseBody: ""};
+
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+}

services/common/tests/unit/test_hawkclient.js

@@ -0,0 +1,520 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://services-common/hawkclient.js");
+
+const SECOND_MS = 1000;
+const MINUTE_MS = SECOND_MS * 60;
+const HOUR_MS = MINUTE_MS * 60;
+
+const TEST_CREDS = {
+  id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+  key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+  algorithm: "sha256"
+};
+
+initTestLogging("Trace");
+
+add_task(function test_now() {
+  let client = new HawkClient("https://example.com");
+
+  do_check_true(client.now() - Date.now() < SECOND_MS);
+});
+
+add_task(function test_updateClockOffset() {
+  let client = new HawkClient("https://example.com");
+
+  let now = new Date();
+  let serverDate = now.toUTCString();
+
+  // Client's clock is off
+  client.now = () => { return now.valueOf() + HOUR_MS; }
+
+  client._updateClockOffset(serverDate);
+
+  // Check that they're close; there will likely be a one-second rounding
+  // error, so checking strict equality will likely fail.
+  //
+  // localtimeOffsetMsec is how many milliseconds to add to the local clock so
+  // that it agrees with the server.  We are one hour ahead of the server, so
+  // our offset should be -1 hour.
+  do_check_true(Math.abs(client.localtimeOffsetMsec + HOUR_MS) <= SECOND_MS);
+});
+
+add_task(function* test_authenticated_get_request() {
+  let message = "{\"msg\": \"Great Success!\"}";
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      do_check_true(request.hasHeader("Authorization"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", method, TEST_CREDS);
+  let result = JSON.parse(response.body);
+
+  do_check_eq("Great Success!", result.msg);
+
+  yield deferredStop(server);
+});
+
+function* check_authenticated_request(method) {
+  let server = httpd_setup({"/foo": (request, response) => {
+      do_check_true(request.hasHeader("Authorization"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+      response.bodyOutputStream.writeFrom(request.bodyInputStream, request.bodyInputStream.available());
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", method, TEST_CREDS, {foo: "bar"});
+  let result = JSON.parse(response.body);
+
+  do_check_eq("bar", result.foo);
+
+  yield deferredStop(server);
+}
+
+add_task(function test_authenticated_post_request() {
+  check_authenticated_request("POST");
+});
+
+add_task(function test_authenticated_put_request() {
+  check_authenticated_request("PUT");
+});
+
+add_task(function test_authenticated_patch_request() {
+  check_authenticated_request("PATCH");
+});
+
+add_task(function* test_extra_headers() {
+  let server = httpd_setup({"/foo": (request, response) => {
+      do_check_true(request.hasHeader("Authorization"));
+      do_check_true(request.hasHeader("myHeader"));
+      do_check_eq(request.getHeader("myHeader"), "fake");
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+      response.bodyOutputStream.writeFrom(request.bodyInputStream, request.bodyInputStream.available());
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", "POST", TEST_CREDS, {foo: "bar"},
+                                      {"myHeader": "fake"});
+  let result = JSON.parse(response.body);
+
+  do_check_eq("bar", result.foo);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_credentials_optional() {
+  let method = "GET";
+  let server = httpd_setup({
+    "/foo": (request, response) => {
+      do_check_false(request.hasHeader("Authorization"));
+
+      let message = JSON.stringify({msg: "you're in the friend zone"});
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  let result = yield client.request("/foo", method); // credentials undefined
+  do_check_eq(JSON.parse(result.body).msg, "you're in the friend zone");
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_server_error() {
+  let message = "Ohai!";
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 418, "I am a Teapot");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  try {
+    yield client.request("/foo", method, TEST_CREDS);
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_eq(418, err.code);
+    do_check_eq("I am a Teapot", err.message);
+  }
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_server_error_json() {
+  let message = JSON.stringify({error: "Cannot get ye flask."});
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 400, "What wouldst thou deau?");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  try {
+    yield client.request("/foo", method, TEST_CREDS);
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_eq("Cannot get ye flask.", err.error);
+  }
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_offset_after_request() {
+  let message = "Ohai!";
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  let now = Date.now();
+  client.now = () => { return now + HOUR_MS; };
+
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  let response = yield client.request("/foo", method, TEST_CREDS);
+  // Should be about an hour off
+  do_check_true(Math.abs(client.localtimeOffsetMsec + HOUR_MS) < SECOND_MS);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_offset_in_hawk_header() {
+  let message = "Ohai!";
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/first": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    },
+
+    "/second": function(request, response) {
+      // We see a better date now in the ts component of the header
+      let delta = getTimestampDelta(request.getHeader("Authorization"));
+      let message = "Delta: " + delta;
+
+      // We're now within HAWK's one-minute window.
+      // I hope this isn't a recipe for intermittent oranges ...
+      if (delta < MINUTE_MS) {
+        response.setStatusLine(request.httpVersion, 200, "OK");
+      } else {
+        response.setStatusLine(request.httpVersion, 400, "Delta: " + delta);
+      }
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() + 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+  yield client.request("/first", method, TEST_CREDS);
+
+  // After the first server response, our offset is updated to -12 hours.
+  // We should be safely in the window, now.
+  do_check_true(Math.abs(client.localtimeOffsetMsec + 12 * HOUR_MS) < MINUTE_MS);
+  yield client.request("/second", method, TEST_CREDS);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_2xx_success() {
+  // Just to ensure that we're not biased toward 200 OK for success
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({"/foo": (request, response) => {
+      response.setStatusLine(request.httpVersion, 202, "Accepted");
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+
+  let response = yield client.request("/foo", method, credentials);
+
+  // Shouldn't be any content in a 202
+  do_check_eq(response.body, "");
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_retry_request_on_fail() {
+  let attempts = 0;
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/maybe": function(request, response) {
+      // This path should be hit exactly twice; once with a bad timestamp, and
+      // again when the client retries the request with a corrected timestamp.
+      attempts += 1;
+      do_check_true(attempts <= 2);
+
+      let delta = getTimestampDelta(request.getHeader("Authorization"));
+
+      // First time through, we should have a bad timestamp
+      if (attempts === 1) {
+        do_check_true(delta > MINUTE_MS);
+        let message = "never!!!";
+        response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+        response.bodyOutputStream.write(message, message.length);
+        return;
+      }
+
+      // Second time through, timestamp should be corrected by client
+      do_check_true(delta < MINUTE_MS);
+      let message = "i love you!!!";
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+      return;
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() + 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  // Request will have bad timestamp; client will retry once
+  let response = yield client.request("/maybe", method, credentials);
+  do_check_eq(response.body, "i love you!!!");
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_multiple_401_retry_once() {
+  // Like test_retry_request_on_fail, but always return a 401
+  // and ensure that the client only retries once.
+  let attempts = 0;
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/maybe": function(request, response) {
+      // This path should be hit exactly twice; once with a bad timestamp, and
+      // again when the client retries the request with a corrected timestamp.
+      attempts += 1;
+
+      do_check_true(attempts <= 2);
+
+      let message = "never!!!";
+      response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() - 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  // Request will have bad timestamp; client will retry once
+  try {
+    yield client.request("/maybe", method, credentials);
+    do_throw("Expected an error");
+  } catch (err) {
+    do_check_eq(err.code, 401);
+  }
+  do_check_eq(attempts, 2);
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_500_no_retry() {
+  // If we get a 500 error, the client should not retry (as it would with a
+  // 401)
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/no-shutup": function() {
+      let message = "Cannot get ye flask.";
+      response.setStatusLine(request.httpVersion, 500, "Internal server error");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  // Throw off the clock so the HawkClient would want to retry the request if
+  // it could
+  client.now = () => {
+    return Date.now() - 12 * HOUR_MS;
+  };
+
+  // Request will 500; no retries
+  try {
+    yield client.request("/no-shutup", method, credentials);
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_eq(err.code, 500);
+  }
+
+  yield deferredStop(server);
+});
+
+add_task(function* test_401_then_500() {
+  // Like test_multiple_401_retry_once, but return a 500 to the
+  // second request, ensuring that the promise is properly rejected
+  // in client.request.
+  let attempts = 0;
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+  let method = "GET";
+
+  let server = httpd_setup({
+    "/maybe": function(request, response) {
+      // This path should be hit exactly twice; once with a bad timestamp, and
+      // again when the client retries the request with a corrected timestamp.
+      attempts += 1;
+      do_check_true(attempts <= 2);
+
+      let delta = getTimestampDelta(request.getHeader("Authorization"));
+
+      // First time through, we should have a bad timestamp
+      // Client will retry
+      if (attempts === 1) {
+        do_check_true(delta > MINUTE_MS);
+        let message = "never!!!";
+        response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+        response.bodyOutputStream.write(message, message.length);
+        return;
+      }
+
+      // Second time through, timestamp should be corrected by client
+      // And fail on the client
+      do_check_true(delta < MINUTE_MS);
+      let message = "Cannot get ye flask.";
+      response.setStatusLine(request.httpVersion, 500, "Internal server error");
+      response.bodyOutputStream.write(message, message.length);
+      return;
+    }
+  });
+
+  let client = new HawkClient(server.baseURI);
+  function getOffset() {
+    return client.localtimeOffsetMsec;
+  }
+
+  client.now = () => {
+    return Date.now() - 12 * HOUR_MS;
+  };
+
+  // We begin with no offset
+  do_check_eq(client.localtimeOffsetMsec, 0);
+
+  // Request will have bad timestamp; client will retry once
+  try {
+    yield client.request("/maybe", method, credentials);
+  } catch(err) {
+    do_check_eq(err.code, 500);
+  }
+  do_check_eq(attempts, 2);
+
+  yield deferredStop(server);
+});
+
+add_task(function* throw_if_not_json_body() {
+  let client = new HawkClient("https://example.com");
+  try {
+    yield client.request("/bogus", "GET", {}, "I am not json");
+    do_throw("Expected an error");
+  } catch(err) {
+    do_check_true(!!err.message);
+  }
+});
+
+// End of tests.
+// Utility functions follow
+
+function getTimestampDelta(authHeader, now=Date.now()) {
+  let tsMS = new Date(
+      parseInt(/ts="(\d+)"/.exec(authHeader)[1], 10) * SECOND_MS);
+  return Math.abs(tsMS - now);
+}
+
+function deferredStop(server) {
+  let deferred = Promise.defer();
+  server.stop(deferred.resolve);
+  return deferred.promise;
+}
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+

services/common/tests/unit/test_hawkrequest.js

@@ -0,0 +1,235 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-common/hawkrequest.js");
+
+// https://github.com/mozilla/fxa-auth-server/wiki/onepw-protocol#wiki-use-session-certificatesign-etc
+var SESSION_KEYS = {
+  sessionToken: h("a0a1a2a3a4a5a6a7 a8a9aaabacadaeaf"+
+                  "b0b1b2b3b4b5b6b7 b8b9babbbcbdbebf"),
+
+  tokenID:      h("c0a29dcf46174973 da1378696e4c82ae"+
+                  "10f723cf4f4d9f75 e39f4ae3851595ab"),
+
+  reqHMACkey:   h("9d8f22998ee7f579 8b887042466b72d5"+
+                  "3e56ab0c094388bf 65831f702d2febc0"),
+};
+
+function do_register_cleanup() {
+  Services.prefs.resetUserPrefs();
+
+  // remove the pref change listener
+  let hawk = new HAWKAuthenticatedRESTRequest("https://example.com");
+  hawk._intl.uninit();
+}
+
+function run_test() {
+  Log.repository.getLogger("Services.Common.RESTRequest").level =
+    Log.Level.Trace;
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+
+add_test(function test_intl_accept_language() {
+  let testCount = 0;
+  let languages = [
+    "zu-NP;vo",     // Nepalese dialect of Zulu, defaulting to Volapük
+    "fa-CG;ik",     // Congolese dialect of Farsei, defaulting to Inupiaq
+  ];
+
+  function setLanguagePref(lang) {
+    let acceptLanguage = Cc["@mozilla.org/supports-string;1"]
+                           .createInstance(Ci.nsISupportsString);
+    acceptLanguage.data = lang;
+    Services.prefs.setComplexValue(
+      "intl.accept_languages", Ci.nsISupportsString, acceptLanguage);
+  }
+
+  let hawk = new HAWKAuthenticatedRESTRequest("https://example.com");
+
+  Services.prefs.addObserver("intl.accept_languages", checkLanguagePref, false);
+  setLanguagePref(languages[testCount]);
+
+  function checkLanguagePref() {
+    var _done = false;
+    CommonUtils.nextTick(function() {
+      // Ensure we're only called for the number of entries in languages[].
+      do_check_true(testCount < languages.length);
+
+      do_check_eq(hawk._intl.accept_languages, languages[testCount]);
+
+      testCount++;
+      if (testCount < languages.length) {
+        // Set next language in prefs; Pref service will call checkNextLanguage.
+        setLanguagePref(languages[testCount]);
+        return;
+      }
+
+      // We've checked all the entries in languages[]. Cleanup and move on.
+      do_print("Checked " + testCount + " languages. Removing checkLanguagePref as pref observer.");
+      Services.prefs.removeObserver("intl.accept_languages", checkLanguagePref);
+      run_next_test();
+      return;
+    });
+  }
+});
+
+add_test(function test_hawk_authenticated_request() {
+  let onProgressCalled = false;
+  let postData = {your: "data"};
+
+  // An arbitrary date - Feb 2, 1971.  It ends in a bunch of zeroes to make our
+  // computation with the hawk timestamp easier, since hawk throws away the
+  // millisecond values.
+  let then = 34329600000;
+
+  let clockSkew = 120000;
+  let timeOffset = -1 * clockSkew;
+  let localTime = then + clockSkew;
+
+  // Set the accept-languages pref to the Nepalese dialect of Zulu.
+  let acceptLanguage = Cc['@mozilla.org/supports-string;1'].createInstance(Ci.nsISupportsString);
+  acceptLanguage.data = 'zu-NP'; // omit trailing ';', which our HTTP libs snip
+  Services.prefs.setComplexValue('intl.accept_languages', Ci.nsISupportsString, acceptLanguage);
+
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256"
+  };
+
+  let server = httpd_setup({
+    "/elysium": function(request, response) {
+      do_check_true(request.hasHeader("Authorization"));
+
+      // check that the header timestamp is our arbitrary system date, not
+      // today's date.  Note that hawk header timestamps are in seconds, not
+      // milliseconds.
+      let authorization = request.getHeader("Authorization");
+      let tsMS = parseInt(/ts="(\d+)"/.exec(authorization)[1], 10) * 1000;
+      do_check_eq(tsMS, then);
+
+      // This testing can be a little wonky. In an environment where
+      //   pref("intl.accept_languages") === 'en-US, en'
+      // the header is sent as:
+      //   'en-US,en;q=0.5'
+      // hence our fake value for acceptLanguage.
+      let lang = request.getHeader("Accept-Language");
+      do_check_eq(lang, acceptLanguage);
+
+      let message = "yay";
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+
+  function onProgress() {
+    onProgressCalled = true;
+  }
+
+  function onComplete(error) {
+    do_check_eq(200, this.response.status);
+    do_check_eq(this.response.body, "yay");
+    do_check_true(onProgressCalled);
+
+    Services.prefs.resetUserPrefs();
+    let pref = Services.prefs.getComplexValue(
+      "intl.accept_languages", Ci.nsIPrefLocalizedString);
+    do_check_neq(acceptLanguage.data, pref.data);
+
+    server.stop(run_next_test);
+  }
+
+  let url = server.baseURI + "/elysium";
+  let extra = {
+    now: localTime,
+    localtimeOffsetMsec: timeOffset
+  };
+
+  let request = new HAWKAuthenticatedRESTRequest(url, credentials, extra);
+
+  // Allow hawk._intl to respond to the language pref change
+  CommonUtils.nextTick(function() {
+    request.post(postData, onComplete, onProgress);
+  });
+});
+
+add_test(function test_hawk_language_pref_changed() {
+  let languages = [
+    "zu-NP",        // Nepalese dialect of Zulu
+    "fa-CG",        // Congolese dialect of Farsi
+  ];
+
+  let credentials = {
+    id: "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x",
+    key: "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=",
+    algorithm: "sha256",
+  };
+
+  function setLanguage(lang) {
+    let acceptLanguage = Cc["@mozilla.org/supports-string;1"].createInstance(Ci.nsISupportsString);
+    acceptLanguage.data = lang;
+    Services.prefs.setComplexValue("intl.accept_languages", Ci.nsISupportsString, acceptLanguage);
+  }
+
+  let server = httpd_setup({
+    "/foo": function(request, response) {
+      do_check_eq(languages[1], request.getHeader("Accept-Language"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+    },
+  });
+
+  let url = server.baseURI + "/foo";
+  let postData = {};
+  let request;
+
+  setLanguage(languages[0]);
+
+  // A new request should create the stateful object for tracking the current
+  // language.
+  request = new HAWKAuthenticatedRESTRequest(url, credentials);
+  CommonUtils.nextTick(testFirstLanguage);
+
+  function testFirstLanguage() {
+    do_check_eq(languages[0], request._intl.accept_languages);
+
+    // Change the language pref ...
+    setLanguage(languages[1]); 
+    CommonUtils.nextTick(testRequest);
+  }
+
+  function testRequest() {
+    // Change of language pref should be picked up, which we can see on the
+    // server by inspecting the request headers.
+    request = new HAWKAuthenticatedRESTRequest(url, credentials);
+    request.post({}, function(error) {
+      do_check_null(error);
+      do_check_eq(200, this.response.status);
+
+      Services.prefs.resetUserPrefs();
+
+      server.stop(run_next_test);
+    });
+  }
+});
+
+add_task(function test_deriveHawkCredentials() {
+  let credentials = deriveHawkCredentials(
+    SESSION_KEYS.sessionToken, "sessionToken");
+
+  do_check_eq(credentials.algorithm, "sha256");
+  do_check_eq(credentials.id, SESSION_KEYS.tokenID);
+  do_check_eq(CommonUtils.bytesAsHex(credentials.key), SESSION_KEYS.reqHMACkey);
+});
+
+// turn formatted test vectors into normal hex strings
+function h(hexStr) {
+  return hexStr.replace(/\s+/g, "");
+}

services/common/tests/unit/test_kinto.js

@@ -0,0 +1,412 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/kinto-offline-client.js");
+Cu.import("resource://testing-common/httpd.js");
+
+const BinaryInputStream = Components.Constructor("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+var server;
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let kintoClient;
+
+function do_get_kinto_collection() {
+  if (!kintoClient) {
+    let config = {
+      remote:`http://localhost:${server.identity.primaryPort}/v1/`,
+      headers: {Authorization: "Basic " + btoa("user:pass")},
+      adapter: FirefoxAdapter
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection("test_collection");
+}
+
+function* clear_collection() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    yield collection.clear();
+  } finally {
+    yield collection.db.close();
+  }
+}
+
+// test some operations on a local collection
+add_task(function* test_kinto_add_get() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+
+    let newRecord = { foo: "bar" };
+    // check a record is created
+    let createResult = yield collection.create(newRecord);
+    do_check_eq(createResult.data.foo, newRecord.foo);
+    // check getting the record gets the same info
+    let getResult = yield collection.get(createResult.data.id);
+    deepEqual(createResult.data, getResult.data);
+    // check what happens if we create the same item again (it should throw
+    // since you can't create with id)
+    try {
+      yield collection.create(createResult.data);
+      do_throw("Creation of a record with an id should fail");
+    } catch (err) { }
+    // try a few creates without waiting for the first few to resolve
+    let promises = [];
+    promises.push(collection.create(newRecord));
+    promises.push(collection.create(newRecord));
+    promises.push(collection.create(newRecord));
+    yield collection.create(newRecord);
+    yield Promise.all(promises);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+// test some operations on multiple connections
+add_task(function* test_kinto_add_get() {
+  const collection1 = do_get_kinto_collection();
+  const collection2 = kintoClient.collection("test_collection_2");
+
+  try {
+    yield collection1.db.open();
+    yield collection2.db.open();
+
+    let newRecord = { foo: "bar" };
+
+    // perform several write operations alternately without waiting for promises
+    // to resolve
+    let promises = [];
+    for (let i = 0; i < 10; i++) {
+      promises.push(collection1.create(newRecord));
+      promises.push(collection2.create(newRecord));
+    }
+
+    // ensure subsequent operations still work
+    yield Promise.all([collection1.create(newRecord),
+                       collection2.create(newRecord)]);
+    yield Promise.all(promises);
+  } finally {
+    yield collection1.db.close();
+    yield collection2.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_kinto_update() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const newRecord = { foo: "bar" };
+    // check a record is created
+    let createResult = yield collection.create(newRecord);
+    do_check_eq(createResult.data.foo, newRecord.foo);
+    do_check_eq(createResult.data._status, "created");
+    // check we can update this OK
+    let copiedRecord = Object.assign(createResult.data, {});
+    deepEqual(createResult.data, copiedRecord);
+    copiedRecord.foo = "wibble";
+    let updateResult = yield collection.update(copiedRecord);
+    // check the field was updated
+    do_check_eq(updateResult.data.foo, copiedRecord.foo);
+    // check the status is still "created", since we haven't synced
+    // the record
+    do_check_eq(updateResult.data._status, "created");
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_kinto_clear() {
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+
+    // create an expected number of records
+    const expected = 10;
+    const newRecord = { foo: "bar" };
+    for (let i = 0; i < expected; i++) {
+      yield collection.create(newRecord);
+    }
+    // check the collection contains the correct number
+    let list = yield collection.list();
+    do_check_eq(list.data.length, expected);
+    // clear the collection and check again - should be 0
+    yield collection.clear();
+    list = yield collection.list();
+    do_check_eq(list.data.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_kinto_delete(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const newRecord = { foo: "bar" };
+    // check a record is created
+    let createResult = yield collection.create(newRecord);
+    do_check_eq(createResult.data.foo, newRecord.foo);
+    // check getting the record gets the same info
+    let getResult = yield collection.get(createResult.data.id);
+    deepEqual(createResult.data, getResult.data);
+    // delete that record
+    let deleteResult = yield collection.delete(createResult.data.id);
+    // check the ID is set on the result
+    do_check_eq(getResult.data.id, deleteResult.data.id);
+    // and check that get no longer returns the record
+    try {
+      getResult = yield collection.get(createResult.data.id);
+      do_throw("there should not be a result");
+    } catch (e) { }
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(function* test_kinto_list(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const expected = 10;
+    const created = [];
+    for (let i = 0; i < expected; i++) {
+      let newRecord = { foo: "test " + i };
+      let createResult = yield collection.create(newRecord);
+      created.push(createResult.data);
+    }
+    // check the collection contains the correct number
+    let list = yield collection.list();
+    do_check_eq(list.data.length, expected);
+
+    // check that all created records exist in the retrieved list
+    for (let createdRecord of created) {
+      let found = false;
+      for (let retrievedRecord of list.data) {
+        if (createdRecord.id == retrievedRecord.id) {
+          deepEqual(createdRecord, retrievedRecord);
+          found = true;
+        }
+      }
+      do_check_true(found);
+    }
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_ignores_already_imported_records(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const record = {id: "41b71c13-17e9-4ee3-9268-6a41abf9730f", title: "foo", last_modified: 1457896541};
+    yield collection.loadDump([record]);
+    let impactedRecords = yield collection.loadDump([record]);
+    do_check_eq(impactedRecords.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_should_overwrite_old_records(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const record = {id: "41b71c13-17e9-4ee3-9268-6a41abf9730f", title: "foo", last_modified: 1457896541};
+    yield collection.loadDump([record]);
+    const updated = Object.assign({}, record, {last_modified: 1457896543});
+    let impactedRecords = yield collection.loadDump([updated]);
+    do_check_eq(impactedRecords.length, 1);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_should_not_overwrite_unsynced_records(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const recordId = "41b71c13-17e9-4ee3-9268-6a41abf9730f";
+    yield collection.create({id: recordId, title: "foo"}, {useRecordId: true});
+    const record = {id: recordId, title: "bar", last_modified: 1457896541};
+    let impactedRecords = yield collection.loadDump([record]);
+    do_check_eq(impactedRecords.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+add_task(function* test_loadDump_should_not_overwrite_records_without_last_modified(){
+  const collection = do_get_kinto_collection();
+  try {
+    yield collection.db.open();
+    const recordId = "41b71c13-17e9-4ee3-9268-6a41abf9730f";
+    yield collection.create({id: recordId, title: "foo"}, {synced: true});
+    const record = {id: recordId, title: "bar", last_modified: 1457896541};
+    let impactedRecords = yield collection.loadDump([record]);
+    do_check_eq(impactedRecords.length, 0);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+add_task(clear_collection);
+
+// Now do some sanity checks against a server - we're not looking to test
+// core kinto.js functionality here (there is excellent test coverage in
+// kinto.js), more making sure things are basically working as expected.
+add_task(function* test_kinto_sync(){
+  const configPath = "/v1/";
+  const recordsPath = "/v1/buckets/default/collections/test_collection/records";
+  // register a handler
+  function handleResponse (request, response) {
+    try {
+      const sampled = getSampleResponse(request, server.identity.primaryPort);
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                             sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sampled.responseBody);
+    } catch (e) {
+      dump(`${e}\n`);
+    }
+  }
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(recordsPath, handleResponse);
+
+  // create an empty collection, sync to populate
+  const collection = do_get_kinto_collection();
+  try {
+    let result;
+
+    yield collection.db.open();
+    result = yield collection.sync();
+    do_check_true(result.ok);
+
+    // our test data has a single record; it should be in the local collection
+    let list = yield collection.list();
+    do_check_eq(list.data.length, 1);
+
+    // now sync again; we should now have 2 records
+    result = yield collection.sync();
+    do_check_true(result.ok);
+    list = yield collection.list();
+    do_check_eq(list.data.length, 2);
+
+    // sync again; the second records should have been modified
+    const before = list.data[0].title;
+    result = yield collection.sync();
+    do_check_true(result.ok);
+    list = yield collection.list();
+    const after = list.data[0].title;
+    do_check_neq(before, after);
+  } finally {
+    yield collection.db.close();
+  }
+});
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1445606341071\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{"last_modified":1445606341071, "done":false, "id":"68db8313-686e-4fff-835e-07d78ad6f2af", "title":"New test"}]})
+    },
+    "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified&_since=1445606341071": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1445607941223\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{"last_modified":1445607941223, "done":false, "id":"901967b0-f729-4b30-8d8d-499cba7f4b1d", "title":"Another new test"}]})
+    },
+    "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified&_since=1445607941223": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1445607541265\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{"last_modified":1445607541265, "done":false, "id":"901967b0-f729-4b30-8d8d-499cba7f4b1d", "title":"Modified title"}]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}

services/common/tests/unit/test_load_modules.js

@@ -0,0 +1,69 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Components.utils.import("resource://gre/modules/AppConstants.jsm");
+
+const MODULE_BASE = "resource://services-common/";
+const shared_modules = [
+  "async.js",
+  "logmanager.js",
+  "rest.js",
+  "stringbundle.js",
+  "utils.js",
+];
+
+const non_android_modules = [
+  "tokenserverclient.js",
+];
+
+const TEST_BASE = "resource://testing-common/services/common/";
+const shared_test_modules = [
+  "logging.js",
+];
+
+const non_android_test_modules = [
+  "storageserver.js",
+];
+
+function expectImportsToSucceed(mm, base=MODULE_BASE) {
+  for (let m of mm) {
+    let resource = base + m;
+    let succeeded = false;
+    try {
+      Components.utils.import(resource, {});
+      succeeded = true;
+    } catch (e) {}
+
+    if (!succeeded) {
+      throw "Importing " + resource + " should have succeeded!";
+    }
+  }
+}
+
+function expectImportsToFail(mm, base=MODULE_BASE) {
+  for (let m of mm) {
+    let resource = base + m;
+    let succeeded = false;
+    try {
+      Components.utils.import(resource, {});
+      succeeded = true;
+    } catch (e) {}
+
+    if (succeeded) {
+      throw "Importing " + resource + " should have failed!";
+    }
+  }
+}
+
+function run_test() {
+  expectImportsToSucceed(shared_modules);
+  expectImportsToSucceed(shared_test_modules, TEST_BASE);
+
+  if (AppConstants.platform != "android") {
+    expectImportsToSucceed(non_android_modules);
+    expectImportsToSucceed(non_android_test_modules, TEST_BASE);
+  } else {
+    expectImportsToFail(non_android_modules);
+    expectImportsToFail(non_android_test_modules, TEST_BASE);
+  }
+}

services/common/tests/unit/test_logmanager.js

@@ -0,0 +1,229 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// NOTE: The sync test_errorhandler_* tests have quite good coverage for
+// other aspects of this.
+
+Cu.import("resource://services-common/logmanager.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+
+function run_test() {
+  run_next_test();
+}
+
+// Returns an array of [consoleAppender, dumpAppender, [fileAppenders]] for
+// the specified log.  Note that fileAppenders will usually have length=1
+function getAppenders(log) {
+  let capps = log.appenders.filter(app => app instanceof Log.ConsoleAppender);
+  equal(capps.length, 1, "should only have one console appender");
+  let dapps = log.appenders.filter(app => app instanceof Log.DumpAppender);
+  equal(dapps.length, 1, "should only have one dump appender");
+  let fapps = log.appenders.filter(app => app instanceof Log.StorageStreamAppender);
+  return [capps[0], dapps[0], fapps];
+}
+
+// Test that the correct thing happens when no prefs exist for the log manager.
+add_task(function* test_noPrefs() {
+  // tell the log manager to init with a pref branch that doesn't exist.
+  let lm = new LogManager("no-such-branch.", ["TestLog"], "test");
+
+  let log = Log.repository.getLogger("TestLog");
+  let [capp, dapp, fapps] = getAppenders(log);
+  // The console appender gets "Fatal" while the "dump" appender gets "Error" levels
+  equal(capp.level, Log.Level.Fatal);
+  equal(dapp.level, Log.Level.Error);
+  // and the file (stream) appender gets Debug by default
+  equal(fapps.length, 1, "only 1 file appender");
+  equal(fapps[0].level, Log.Level.Debug);
+  lm.finalize();
+});
+
+// Test that changes to the prefs used by the log manager are updated dynamically.
+add_task(function* test_PrefChanges() {
+  Services.prefs.setCharPref("log-manager.test.log.appender.console", "Trace");
+  Services.prefs.setCharPref("log-manager.test.log.appender.dump", "Trace");
+  Services.prefs.setCharPref("log-manager.test.log.appender.file.level", "Trace");
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  let [capp, dapp, [fapp]] = getAppenders(log);
+  equal(capp.level, Log.Level.Trace);
+  equal(dapp.level, Log.Level.Trace);
+  equal(fapp.level, Log.Level.Trace);
+  // adjust the prefs and they should magically be reflected in the appenders.
+  Services.prefs.setCharPref("log-manager.test.log.appender.console", "Debug");
+  Services.prefs.setCharPref("log-manager.test.log.appender.dump", "Debug");
+  Services.prefs.setCharPref("log-manager.test.log.appender.file.level", "Debug");
+  equal(capp.level, Log.Level.Debug);
+  equal(dapp.level, Log.Level.Debug);
+  equal(fapp.level, Log.Level.Debug);
+  // and invalid values should cause them to fallback to their defaults.
+  Services.prefs.setCharPref("log-manager.test.log.appender.console", "xxx");
+  Services.prefs.setCharPref("log-manager.test.log.appender.dump", "xxx");
+  Services.prefs.setCharPref("log-manager.test.log.appender.file.level", "xxx");
+  equal(capp.level, Log.Level.Fatal);
+  equal(dapp.level, Log.Level.Error);
+  equal(fapp.level, Log.Level.Debug);
+  lm.finalize();
+});
+
+// Test that the same log used by multiple log managers does the right thing.
+add_task(function* test_SharedLogs() {
+  // create the prefs for the first instance.
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.console", "Trace");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.dump", "Trace");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.file.level", "Trace");
+  let lm1 = new LogManager("log-manager-1.test.", ["TestLog3"], "test");
+
+  // and the second.
+  Services.prefs.setCharPref("log-manager-2.test.log.appender.console", "Debug");
+  Services.prefs.setCharPref("log-manager-2.test.log.appender.dump", "Debug");
+  Services.prefs.setCharPref("log-manager-2.test.log.appender.file.level", "Debug");
+  let lm2 = new LogManager("log-manager-2.test.", ["TestLog3"], "test");
+
+  let log = Log.repository.getLogger("TestLog3");
+  let [capp, dapp, fapps] = getAppenders(log);
+
+  // console and dump appenders should be "trace" as it is more verbose than
+  // "debug"
+  equal(capp.level, Log.Level.Trace);
+  equal(dapp.level, Log.Level.Trace);
+
+  // Set the prefs on the -1 branch to "Error" - it should then end up with
+  // "Debug" from the -2 branch.
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.console", "Error");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.dump", "Error");
+  Services.prefs.setCharPref("log-manager-1.test.log.appender.file.level", "Error");
+
+  equal(capp.level, Log.Level.Debug);
+  equal(dapp.level, Log.Level.Debug);
+
+  lm1.finalize();
+  lm2.finalize();
+});
+
+// A little helper to test what log files exist.  We expect exactly zero (if
+// prefix is null) or exactly one with the specified prefix.
+function checkLogFile(prefix) {
+  let logsdir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
+  let entries = logsdir.directoryEntries;
+  if (!prefix) {
+    // expecting no files.
+    ok(!entries.hasMoreElements());
+  } else {
+    // expecting 1 file.
+    ok(entries.hasMoreElements());
+    let logfile = entries.getNext().QueryInterface(Ci.nsILocalFile);
+    equal(logfile.leafName.slice(-4), ".txt");
+    ok(logfile.leafName.startsWith(prefix + "-test-"), logfile.leafName);
+    // and remove it ready for the next check.
+    logfile.remove(false);
+  }
+}
+
+// Test that we correctly write error logs by default
+add_task(function* test_logFileErrorDefault() {
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  log.error("an error message");
+  yield lm.resetFileLog(lm.REASON_ERROR);
+  // One error log file exists.
+  checkLogFile("error");
+
+  lm.finalize();
+});
+
+// Test that we correctly write success logs.
+add_task(function* test_logFileSuccess() {
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnError", false);
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", false);
+
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  log.info("an info message");
+  yield lm.resetFileLog();
+  // Zero log files exist.
+  checkLogFile(null);
+
+  // Reset logOnSuccess and do it again - log should appear.
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", true);
+  log.info("an info message");
+  yield lm.resetFileLog();
+
+  checkLogFile("success");
+
+  // Now test with no "reason" specified and no "error" record.
+  log.info("an info message");
+  yield lm.resetFileLog();
+  // should get a "success" entry.
+  checkLogFile("success");
+
+  // With no "reason" and an error record - should get no success log.
+  log.error("an error message");
+  yield lm.resetFileLog();
+  // should get no entry
+  checkLogFile(null);
+
+  // And finally now with no error, to ensure that the fact we had an error
+  // previously doesn't persist after the .resetFileLog call.
+  log.info("an info message");
+  yield lm.resetFileLog();
+  checkLogFile("success");
+
+  lm.finalize();
+});
+
+// Test that we correctly write error logs.
+add_task(function* test_logFileError() {
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnError", false);
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", false);
+
+  let lm = new LogManager("log-manager.test.", ["TestLog2"], "test");
+
+  let log = Log.repository.getLogger("TestLog2");
+  log.info("an info message");
+  let reason = yield lm.resetFileLog();
+  Assert.equal(reason, null, "null returned when no file created.");
+  // Zero log files exist.
+  checkLogFile(null);
+
+  // Reset logOnSuccess - success logs should appear if no error records.
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", true);
+  log.info("an info message");
+  reason = yield lm.resetFileLog();
+  Assert.equal(reason, lm.SUCCESS_LOG_WRITTEN);
+  checkLogFile("success");
+
+  // Set logOnError and unset logOnSuccess - error logs should appear.
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnSuccess", false);
+  Services.prefs.setBoolPref("log-manager.test.log.appender.file.logOnError", true);
+  log.error("an error message");
+  reason = yield lm.resetFileLog();
+  Assert.equal(reason, lm.ERROR_LOG_WRITTEN);
+  checkLogFile("error");
+
+  // Now test with no "error" record.
+  log.info("an info message");
+  reason = yield lm.resetFileLog();
+  // should get no file
+  Assert.equal(reason, null);
+  checkLogFile(null);
+
+  // With an error record we should get an error log.
+  log.error("an error message");
+  reason = yield lm.resetFileLog();
+  // should get en error log
+  Assert.equal(reason, lm.ERROR_LOG_WRITTEN);
+  checkLogFile("error");
+
+  // And finally now with success, to ensure that the fact we had an error
+  // previously doesn't persist after the .resetFileLog call.
+  log.info("an info message");
+  yield lm.resetFileLog();
+  checkLogFile(null);
+
+  lm.finalize();
+});

services/common/tests/unit/test_observers.js

@@ -0,0 +1,84 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Components.utils.import("resource://services-common/observers.js");
+
+var gSubject = {};
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_function_observer() {
+  let foo = false;
+
+  let onFoo = function(subject, data) {
+    foo = !foo;
+    do_check_eq(subject, gSubject);
+    do_check_eq(data, "some data");
+  };
+
+  Observers.add("foo", onFoo);
+  Observers.notify("foo", gSubject, "some data");
+
+  // The observer was notified after being added.
+  do_check_true(foo);
+
+  Observers.remove("foo", onFoo);
+  Observers.notify("foo");
+
+  // The observer was not notified after being removed.
+  do_check_true(foo);
+
+  run_next_test();
+});
+
+add_test(function test_method_observer() {
+  let obj = {
+    foo: false,
+    onFoo: function(subject, data) {
+      this.foo = !this.foo;
+      do_check_eq(subject, gSubject);
+      do_check_eq(data, "some data");
+    }
+  };
+
+  // The observer is notified after being added.
+  Observers.add("foo", obj.onFoo, obj);
+  Observers.notify("foo", gSubject, "some data");
+  do_check_true(obj.foo);
+
+  // The observer is not notified after being removed.
+  Observers.remove("foo", obj.onFoo, obj);
+  Observers.notify("foo");
+  do_check_true(obj.foo);
+
+  run_next_test();
+});
+
+add_test(function test_object_observer() {
+  let obj = {
+    foo: false,
+    observe: function(subject, topic, data) {
+      this.foo = !this.foo;
+
+      do_check_eq(subject, gSubject);
+      do_check_eq(topic, "foo");
+      do_check_eq(data, "some data");
+    }
+  };
+
+  Observers.add("foo", obj);
+  Observers.notify("foo", gSubject, "some data");
+
+  // The observer is notified after being added.
+  do_check_true(obj.foo);
+
+  Observers.remove("foo", obj);
+  Observers.notify("foo");
+
+  // The observer is not notified after being removed.
+  do_check_true(obj.foo);
+
+  run_next_test();
+});

services/common/tests/unit/test_restrequest.js

@@ -0,0 +1,873 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/NetUtil.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  Log.repository.getLogger("Services.Common.RESTRequest").level =
+    Log.Level.Trace;
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+/**
+ * Initializing a RESTRequest with an invalid URI throws
+ * NS_ERROR_MALFORMED_URI.
+ */
+add_test(function test_invalid_uri() {
+  do_check_throws(function() {
+    new RESTRequest("an invalid URI");
+  }, Cr.NS_ERROR_MALFORMED_URI);
+  run_next_test();
+});
+
+/**
+ * Verify initial values for attributes.
+ */
+add_test(function test_attributes() {
+  let uri = "http://foo.com/bar/baz";
+  let request = new RESTRequest(uri);
+
+  do_check_true(request.uri instanceof Ci.nsIURI);
+  do_check_eq(request.uri.spec, uri);
+  do_check_eq(request.response, null);
+  do_check_eq(request.status, request.NOT_SENT);
+  let expectedLoadFlags = Ci.nsIRequest.LOAD_BYPASS_CACHE |
+                          Ci.nsIRequest.INHIBIT_CACHING |
+                          Ci.nsIRequest.LOAD_ANONYMOUS;
+  do_check_eq(request.loadFlags, expectedLoadFlags);
+
+  run_next_test();
+});
+
+/**
+ * Verify that a proxy auth redirect doesn't break us. This has to be the first
+ * request made in the file!
+ */
+add_test(function test_proxy_auth_redirect() {
+  let pacFetched = false;
+  function pacHandler(metadata, response) {
+    pacFetched = true;
+    let body = 'function FindProxyForURL(url, host) { return "DIRECT"; }';
+    response.setStatusLine(metadata.httpVersion, 200, "OK");
+    response.setHeader("Content-Type", "application/x-ns-proxy-autoconfig", false);
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let fetched = false;
+  function original(metadata, response) {
+    fetched = true;
+    let body = "TADA!";
+    response.setStatusLine(metadata.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let server = httpd_setup({
+    "/original": original,
+    "/pac3":     pacHandler
+  });
+  PACSystemSettings.PACURI = server.baseURI + "/pac3";
+  installFakePAC();
+
+  let res = new RESTRequest(server.baseURI + "/original");
+  res.get(function (error) {
+    do_check_true(pacFetched);
+    do_check_true(fetched);
+    do_check_true(!error);
+    do_check_true(this.response.success);
+    do_check_eq("TADA!", this.response.body);
+    uninstallFakePAC();
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Ensure that failures that cause asyncOpen to throw
+ * result in callbacks being invoked.
+ * Bug 826086.
+ */
+add_test(function test_forbidden_port() {
+  let request = new RESTRequest("http://localhost:6000/");
+  request.get(function(error) {
+    if (!error) {
+      do_throw("Should have got an error.");
+    }
+    do_check_eq(error.result, Components.results.NS_ERROR_PORT_ACCESS_NOT_ALLOWED);
+    run_next_test();
+  });
+});
+
+/**
+ * Demonstrate API short-hand: create a request and dispatch it immediately.
+ */
+add_test(function test_simple_get() {
+  let handler = httpd_handler(200, "OK", "Huzzah!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource").get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Huzzah!");
+
+    server.stop(run_next_test);
+  });
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, "GET");
+});
+
+/**
+ * Test HTTP GET with all bells and whistles.
+ */
+add_test(function test_get() {
+  let handler = httpd_handler(200, "OK", "Huzzah!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  do_check_eq(request.status, request.NOT_SENT);
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("This function should have been overwritten!");
+  };
+
+  let onProgress_called = false;
+  function onProgress() {
+    onProgress_called = true;
+    do_check_eq(this.status, request.IN_PROGRESS);
+    do_check_true(this.response.body.length > 0);
+
+    do_check_true(!!(this.channel.loadFlags & Ci.nsIRequest.LOAD_BYPASS_CACHE));
+    do_check_true(!!(this.channel.loadFlags & Ci.nsIRequest.INHIBIT_CACHING));
+  };
+
+  function onComplete(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Huzzah!");
+    do_check_eq(handler.request.method, "GET");
+
+    do_check_true(onProgress_called);
+    CommonUtils.nextTick(function () {
+      do_check_eq(request.onComplete, null);
+      do_check_eq(request.onProgress, null);
+      server.stop(run_next_test);
+    });
+  };
+
+  do_check_eq(request.get(onComplete, onProgress), request);
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, "GET");
+  do_check_throws(function () {
+    request.get();
+  });
+});
+
+/**
+ * Test HTTP GET with UTF-8 content, and custom Content-Type.
+ */
+add_test(function test_get_utf8() {
+  let response = "Hello World or Καλημέρα κόσμε or こんにちは 世界";
+
+  let contentType = "text/plain";
+  let charset = true;
+  let charsetSuffix = "; charset=UTF-8";
+
+  let server = httpd_setup({"/resource": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", contentType + (charset ? charsetSuffix : ""));
+
+    let converter = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                    .createInstance(Ci.nsIConverterOutputStream);
+    converter.init(res.bodyOutputStream, "UTF-8", 0, 0x0000);
+    converter.writeString(response);
+    converter.close();
+  }});
+
+  // Check if charset in Content-Type is propertly interpreted.
+  let request1 = new RESTRequest(server.baseURI + "/resource");
+  request1.get(function(error) {
+    do_check_null(error);
+
+    do_check_eq(request1.response.status, 200);
+    do_check_eq(request1.response.body, response);
+    do_check_eq(request1.response.headers["content-type"],
+                contentType + charsetSuffix);
+
+    // Check that we default to UTF-8 if Content-Type doesn't have a charset.
+    charset = false;
+    let request2 = new RESTRequest(server.baseURI + "/resource");
+    request2.get(function(error) {
+      do_check_null(error);
+
+      do_check_eq(request2.response.status, 200);
+      do_check_eq(request2.response.body, response);
+      do_check_eq(request2.response.headers["content-type"], contentType);
+      do_check_eq(request2.response.charset, "utf-8");
+
+      server.stop(run_next_test);
+    });
+  });
+});
+
+/**
+ * Test HTTP POST data is encoded as UTF-8 by default.
+ */
+add_test(function test_post_utf8() {
+  // We setup a handler that responds with exactly what it received.
+  // Given we've already tested above that responses are correctly utf-8
+  // decoded we can surmise that the correct response coming back means the
+  // input must also have been encoded.
+  let server = httpd_setup({"/echo": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", req.getHeader("content-type"));
+    // Get the body as bytes and write them back without touching them
+    let sis = Cc["@mozilla.org/scriptableinputstream;1"]
+              .createInstance(Ci.nsIScriptableInputStream);
+    sis.init(req.bodyInputStream);
+    let body = sis.read(sis.available());
+    sis.close()
+    res.write(body);
+  }});
+
+  let data = {copyright: "\xa9"}; // \xa9 is the copyright symbol
+  let request1 = new RESTRequest(server.baseURI + "/echo");
+  request1.post(data, function(error) {
+    do_check_null(error);
+
+    do_check_eq(request1.response.status, 200);
+    deepEqual(JSON.parse(request1.response.body), data);
+    do_check_eq(request1.response.headers["content-type"],
+                "application/json; charset=utf-8")
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test more variations of charset handling.
+ */
+add_test(function test_charsets() {
+  let response = "Hello World, I can't speak Russian";
+
+  let contentType = "text/plain";
+  let charset = true;
+  let charsetSuffix = "; charset=us-ascii";
+
+  let server = httpd_setup({"/resource": function(req, res) {
+    res.setStatusLine(req.httpVersion, 200, "OK");
+    res.setHeader("Content-Type", contentType + (charset ? charsetSuffix : ""));
+
+    let converter = Cc["@mozilla.org/intl/converter-output-stream;1"]
+                    .createInstance(Ci.nsIConverterOutputStream);
+    converter.init(res.bodyOutputStream, "us-ascii", 0, 0x0000);
+    converter.writeString(response);
+    converter.close();
+  }});
+
+  // Check that provided charset overrides hint.
+  let request1 = new RESTRequest(server.baseURI + "/resource");
+  request1.charset = "not-a-charset";
+  request1.get(function(error) {
+    do_check_null(error);
+
+    do_check_eq(request1.response.status, 200);
+    do_check_eq(request1.response.body, response);
+    do_check_eq(request1.response.headers["content-type"],
+                contentType + charsetSuffix);
+    do_check_eq(request1.response.charset, "us-ascii");
+
+    // Check that hint is used if Content-Type doesn't have a charset.
+    charset = false;
+    let request2 = new RESTRequest(server.baseURI + "/resource");
+    request2.charset = "us-ascii";
+    request2.get(function(error) {
+      do_check_null(error);
+
+      do_check_eq(request2.response.status, 200);
+      do_check_eq(request2.response.body, response);
+      do_check_eq(request2.response.headers["content-type"], contentType);
+      do_check_eq(request2.response.charset, "us-ascii");
+
+      server.stop(run_next_test);
+    });
+  });
+});
+
+/**
+ * Used for testing PATCH/PUT/POST methods.
+ */
+function check_posting_data(method) {
+  let funcName = method.toLowerCase();
+  let handler = httpd_handler(200, "OK", "Got it!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  do_check_eq(request.status, request.NOT_SENT);
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("This function should have been overwritten!");
+  };
+
+  let onProgress_called = false;
+  function onProgress() {
+    onProgress_called = true;
+    do_check_eq(this.status, request.IN_PROGRESS);
+    do_check_true(this.response.body.length > 0);
+  };
+
+  function onComplete(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Got it!");
+
+    do_check_eq(handler.request.method, method);
+    do_check_eq(handler.request.body, "Hullo?");
+    do_check_eq(handler.request.getHeader("Content-Type"), "text/plain");
+
+    do_check_true(onProgress_called);
+    CommonUtils.nextTick(function () {
+      do_check_eq(request.onComplete, null);
+      do_check_eq(request.onProgress, null);
+      server.stop(run_next_test);
+    });
+  };
+
+  do_check_eq(request[funcName]("Hullo?", onComplete, onProgress), request);
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, method);
+  do_check_throws(function () {
+    request[funcName]("Hai!");
+  });
+}
+
+/**
+ * Test HTTP PATCH with a simple string argument and default Content-Type.
+ */
+add_test(function test_patch() {
+  check_posting_data("PATCH");
+});
+
+/**
+ * Test HTTP PUT with a simple string argument and default Content-Type.
+ */
+add_test(function test_put() {
+  check_posting_data("PUT");
+});
+
+/**
+ * Test HTTP POST with a simple string argument and default Content-Type.
+ */
+add_test(function test_post() {
+  check_posting_data("POST");
+});
+
+/**
+ * Test HTTP DELETE.
+ */
+add_test(function test_delete() {
+  let handler = httpd_handler(200, "OK", "Got it!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  do_check_eq(request.status, request.NOT_SENT);
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("This function should have been overwritten!");
+  };
+
+  let onProgress_called = false;
+  function onProgress() {
+    onProgress_called = true;
+    do_check_eq(this.status, request.IN_PROGRESS);
+    do_check_true(this.response.body.length > 0);
+  };
+
+  function onComplete(error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "Got it!");
+    do_check_eq(handler.request.method, "DELETE");
+
+    do_check_true(onProgress_called);
+    CommonUtils.nextTick(function () {
+      do_check_eq(request.onComplete, null);
+      do_check_eq(request.onProgress, null);
+      server.stop(run_next_test);
+    });
+  };
+
+  do_check_eq(request.delete(onComplete, onProgress), request);
+  do_check_eq(request.status, request.SENT);
+  do_check_eq(request.method, "DELETE");
+  do_check_throws(function () {
+    request.delete();
+  });
+});
+
+/**
+ * Test an HTTP response with a non-200 status code.
+ */
+add_test(function test_get_404() {
+  let handler = httpd_handler(404, "Not Found", "Cannae find it!");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_false(this.response.success);
+    do_check_eq(this.response.status, 404);
+    do_check_eq(this.response.body, "Cannae find it!");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The 'data' argument to PUT, if not a string already, is automatically
+ * stringified as JSON.
+ */
+add_test(function test_put_json() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let sample_data = {
+    some: "sample_data",
+    injson: "format",
+    number: 42
+  };
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.put(sample_data, function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "PUT");
+    do_check_eq(handler.request.body, JSON.stringify(sample_data));
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/json; charset=utf-8");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The 'data' argument to POST, if not a string already, is automatically
+ * stringified as JSON.
+ */
+add_test(function test_post_json() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let sample_data = {
+    some: "sample_data",
+    injson: "format",
+    number: 42
+  };
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.post(sample_data, function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "POST");
+    do_check_eq(handler.request.body, JSON.stringify(sample_data));
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/json; charset=utf-8");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The content-type will be text/plain without a charset if the 'data' argument
+ * to POST is already a string.
+ */
+add_test(function test_post_json() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let sample_data = "hello";
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.post(sample_data, function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "POST");
+    do_check_eq(handler.request.body, sample_data);
+    do_check_eq(handler.request.getHeader("Content-Type"), "text/plain");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * HTTP PUT with a custom Content-Type header.
+ */
+add_test(function test_put_override_content_type() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.setHeader("Content-Type", "application/lolcat");
+  request.put("O HAI!!1!", function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "PUT");
+    do_check_eq(handler.request.body, "O HAI!!1!");
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/lolcat");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * HTTP POST with a custom Content-Type header.
+ */
+add_test(function test_post_override_content_type() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.setHeader("Content-Type", "application/lolcat");
+  request.post("O HAI!!1!", function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.status, this.COMPLETED);
+    do_check_true(this.response.success);
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.method, "POST");
+    do_check_eq(handler.request.body, "O HAI!!1!");
+    do_check_eq(handler.request.getHeader("Content-Type"), "application/lolcat");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * No special headers are sent by default on a GET request.
+ */
+add_test(function test_get_no_headers() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let ignore_headers = ["host", "user-agent", "accept", "accept-language",
+                        "accept-encoding", "accept-charset", "keep-alive",
+                        "connection", "pragma", "cache-control",
+                        "content-length"];
+
+  new RESTRequest(server.baseURI + "/resource").get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    let server_headers = handler.request.headers;
+    while (server_headers.hasMoreElements()) {
+      let header = server_headers.getNext().toString();
+      if (ignore_headers.indexOf(header) == -1) {
+        do_throw("Got unexpected header!");
+      }
+    }
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test changing the URI after having created the request.
+ */
+add_test(function test_changing_uri() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest("http://localhost:1234/the-wrong-resource");
+  request.uri = CommonUtils.makeURI(server.baseURI + "/resource");
+  request.get(function (error) {
+    do_check_eq(error, null);
+    do_check_eq(this.response.status, 200);
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test setting HTTP request headers.
+ */
+add_test(function test_request_setHeader() {
+  let handler = httpd_handler(200, "OK");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+
+  request.setHeader("X-What-Is-Weave", "awesome");
+  request.setHeader("X-WHAT-is-Weave", "more awesomer");
+  request.setHeader("Another-Header", "Hello World");
+
+  request.get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(handler.request.getHeader("X-What-Is-Weave"), "more awesomer");
+    do_check_eq(handler.request.getHeader("another-header"), "Hello World");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * Test receiving HTTP response headers.
+ */
+add_test(function test_response_headers() {
+  function handler(request, response) {
+    response.setHeader("X-What-Is-Weave", "awesome");
+    response.setHeader("Another-Header", "Hello World");
+    response.setStatusLine(request.httpVersion, 200, "OK");
+  }
+  let server = httpd_setup({"/resource": handler});
+  let request = new RESTRequest(server.baseURI + "/resource");
+
+  request.get(function (error) {
+    do_check_eq(error, null);
+
+    do_check_eq(this.response.status, 200);
+    do_check_eq(this.response.body, "");
+
+    do_check_eq(this.response.headers["x-what-is-weave"], "awesome");
+    do_check_eq(this.response.headers["another-header"], "Hello World");
+
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * The onComplete() handler gets called in case of any network errors
+ * (e.g. NS_ERROR_CONNECTION_REFUSED).
+ */
+add_test(function test_connection_refused() {
+  let request = new RESTRequest("http://localhost:1234/resource");
+  request.onProgress = function onProgress() {
+    do_throw("Shouldn't have called request.onProgress()!");
+  };
+  request.get(function (error) {
+    do_check_eq(error.result, Cr.NS_ERROR_CONNECTION_REFUSED);
+    do_check_eq(error.message, "NS_ERROR_CONNECTION_REFUSED");
+    do_check_eq(this.status, this.COMPLETED);
+    run_next_test();
+  });
+  do_check_eq(request.status, request.SENT);
+});
+
+/**
+ * Abort a request that just sent off.
+ */
+add_test(function test_abort() {
+  function handler() {
+    do_throw("Shouldn't have gotten here!");
+  }
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+
+  // Aborting a request that hasn't been sent yet is pointless and will throw.
+  do_check_throws(function () {
+    request.abort();
+  });
+
+  request.onProgress = request.onComplete = function () {
+    do_throw("Shouldn't have gotten here!");
+  };
+  request.get();
+  request.abort();
+
+  // Aborting an already aborted request is pointless and will throw.
+  do_check_throws(function () {
+    request.abort();
+  });
+
+  do_check_eq(request.status, request.ABORTED);
+  CommonUtils.nextTick(function () {
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * A non-zero 'timeout' property specifies the amount of seconds to wait after
+ * channel activity until the request is automatically canceled.
+ */
+add_test(function test_timeout() {
+  let server = new HttpServer();
+  let server_connection;
+  server._handler.handleResponse = function(connection) {
+    // This is a handler that doesn't do anything, just keeps the connection
+    // open, thereby mimicking a timing out connection. We keep a reference to
+    // the open connection for later so it can be properly disposed of. That's
+    // why you really only want to make one HTTP request to this server ever.
+    server_connection = connection;
+  };
+  server.start();
+  let identity = server.identity;
+  let uri = identity.primaryScheme + "://" + identity.primaryHost + ":" +
+            identity.primaryPort;
+
+  let request = new RESTRequest(uri + "/resource");
+  request.timeout = 0.1; // 100 milliseconds
+  request.get(function (error) {
+    do_check_eq(error.result, Cr.NS_ERROR_NET_TIMEOUT);
+    do_check_eq(this.status, this.ABORTED);
+
+    // server_connection is undefined on the Android emulator for reasons
+    // unknown. Yet, we still get here. If this test is refactored, we should
+    // investigate the reason why the above callback is behaving differently.
+    if (server_connection) {
+      _("Closing connection.");
+      server_connection.close();
+    }
+
+    _("Shutting down server.");
+    server.stop(run_next_test);
+  });
+});
+
+/**
+ * An exception thrown in 'onProgress' propagates to the 'onComplete' handler.
+ */
+add_test(function test_exception_in_onProgress() {
+  let handler = httpd_handler(200, "OK", "Foobar");
+  let server = httpd_setup({"/resource": handler});
+
+  let request = new RESTRequest(server.baseURI + "/resource");
+  request.onProgress = function onProgress() {
+    it.does.not.exist();
+  };
+  request.get(function onComplete(error) {
+    do_check_eq(error, "ReferenceError: it is not defined");
+    do_check_eq(this.status, this.ABORTED);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_new_channel() {
+  _("Ensure a redirect to a new channel is handled properly.");
+
+  function checkUA(metadata) {
+    let ua = metadata.getHeader("User-Agent");
+    _("User-Agent is " + ua);
+    do_check_eq("foo bar", ua);
+  }
+
+  let redirectRequested = false;
+  let redirectURL;
+  function redirectHandler(metadata, response) {
+    checkUA(metadata);
+    redirectRequested = true;
+
+    let body = "Redirecting";
+    response.setStatusLine(metadata.httpVersion, 307, "TEMPORARY REDIRECT");
+    response.setHeader("Location", redirectURL);
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let resourceRequested = false;
+  function resourceHandler(metadata, response) {
+    checkUA(metadata);
+    resourceRequested = true;
+
+    let body = "Test";
+    response.setHeader("Content-Type", "text/plain");
+    response.bodyOutputStream.write(body, body.length);
+  }
+
+  let server1 = httpd_setup({"/redirect": redirectHandler});
+  let server2 = httpd_setup({"/resource": resourceHandler});
+  redirectURL = server2.baseURI + "/resource";
+
+  function advance() {
+    server1.stop(function () {
+      server2.stop(run_next_test);
+    });
+  }
+
+  let request = new RESTRequest(server1.baseURI + "/redirect");
+  request.setHeader("User-Agent", "foo bar");
+
+  // Swizzle in our own fakery, because this redirect is neither
+  // internal nor URI-preserving. RESTRequest's policy is to only
+  // copy headers under certain circumstances.
+  let protoMethod = request.shouldCopyOnRedirect;
+  request.shouldCopyOnRedirect = function wrapped(o, n, f) {
+    // Check the default policy.
+    do_check_false(protoMethod.call(this, o, n, f));
+    return true;
+  };
+
+  request.get(function onComplete(error) {
+    let response = this.response;
+
+    do_check_eq(200, response.status);
+    do_check_eq("Test", response.body);
+    do_check_true(redirectRequested);
+    do_check_true(resourceRequested);
+
+    advance();
+  });
+});
+
+add_test(function test_not_sending_cookie() {
+  function handler(metadata, response) {
+    let body = "COOKIE!";
+    response.setStatusLine(metadata.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(body, body.length);
+    do_check_false(metadata.hasHeader("Cookie"));
+  }
+  let server = httpd_setup({"/test": handler});
+
+  let cookieSer = Cc["@mozilla.org/cookieService;1"]
+                    .getService(Ci.nsICookieService);
+  let uri = CommonUtils.makeURI(server.baseURI);
+  cookieSer.setCookieString(uri, null, "test=test; path=/;", null);
+
+  let res = new RESTRequest(server.baseURI + "/test");
+  res.get(function (error) {
+    do_check_null(error);
+    do_check_true(this.response.success);
+    do_check_eq("COOKIE!", this.response.body);
+    server.stop(run_next_test);
+  });
+});
+

services/common/tests/unit/test_storage_adapter.js

@@ -0,0 +1,269 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/kinto-offline-client.js");
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let gFirefoxAdapter = null;
+
+function do_get_kinto_adapter() {
+  if (gFirefoxAdapter == null) {
+    gFirefoxAdapter = new FirefoxAdapter("test");
+  }
+  return gFirefoxAdapter;
+}
+
+function do_get_kinto_db() {
+  let profile = do_get_profile();
+  let kintoDB = profile.clone();
+  kintoDB.append(kintoFilename);
+  return kintoDB;
+}
+
+function cleanup_kinto() {
+  add_test(function cleanup_kinto_files(){
+    let kintoDB = do_get_kinto_db();
+    // clean up the db
+    kintoDB.remove(false);
+    // force re-creation of the adapter
+    gFirefoxAdapter = null;
+    run_next_test();
+  });
+}
+
+function test_collection_operations() {
+  add_task(function* test_kinto_clear() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.clear();
+    yield adapter.close();
+  });
+
+  // test creating new records... and getting them again
+  add_task(function* test_kinto_create_new_get_existing() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let record = {id:"test-id", foo:"bar"};
+    yield adapter.execute((transaction) => transaction.create(record));
+    let newRecord = yield adapter.get("test-id");
+    // ensure the record is the same as when it was added
+    deepEqual(record, newRecord);
+    yield adapter.close();
+  });
+
+  // test removing records
+  add_task(function* test_kinto_can_remove_some_records() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    // create a second record
+    let record = {id:"test-id-2", foo:"baz"};
+    yield adapter.execute((transaction) => transaction.create(record));
+    let newRecord = yield adapter.get("test-id-2");
+    deepEqual(record, newRecord);
+    // delete the record
+    yield adapter.execute((transaction) => transaction.delete(record.id));
+    newRecord = yield adapter.get(record.id);
+    // ... and ensure it's no longer there
+    do_check_eq(newRecord, undefined);
+    // ensure the other record still exists
+    newRecord = yield adapter.get("test-id");
+    do_check_neq(newRecord, undefined);
+    yield adapter.close();
+  });
+
+  // test getting records that don't exist
+  add_task(function* test_kinto_get_non_existant() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    // Kinto expects adapters to either:
+    let newRecord = yield adapter.get("missing-test-id");
+    // resolve with an undefined record
+    do_check_eq(newRecord, undefined);
+    yield adapter.close();
+  });
+
+  // test updating records... and getting them again
+  add_task(function* test_kinto_update_get_existing() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let originalRecord = {id:"test-id", foo:"bar"};
+    let updatedRecord = {id:"test-id", foo:"baz"};
+    yield adapter.clear();
+    yield adapter.execute((transaction) => transaction.create(originalRecord));
+    yield adapter.execute((transaction) => transaction.update(updatedRecord));
+    // ensure the record exists
+    let newRecord = yield adapter.get("test-id");
+    // ensure the record is the same as when it was added
+    deepEqual(updatedRecord, newRecord);
+    yield adapter.close();
+  });
+
+  // test listing records
+  add_task(function* test_kinto_list() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let originalRecord = {id:"test-id-1", foo:"bar"};
+    let records = yield adapter.list();
+    do_check_eq(records.length, 1);
+    yield adapter.execute((transaction) => transaction.create(originalRecord));
+    records = yield adapter.list();
+    do_check_eq(records.length, 2);
+    yield adapter.close();
+  });
+
+  // test aborting transaction
+  add_task(function* test_kinto_aborting_transaction() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.clear();
+    let record = {id: 1, foo: "bar"};
+    let error = null;
+    try {
+      yield adapter.execute((transaction) => {
+        transaction.create(record);
+        throw new Error("unexpected");
+      });
+    } catch (e) {
+      error = e;
+    }
+    do_check_neq(error, null);
+    records = yield adapter.list();
+    do_check_eq(records.length, 0);
+    yield adapter.close();
+  });
+
+  // test save and get last modified
+  add_task(function* test_kinto_last_modified() {
+    const initialValue = 0;
+    const intendedValue = 12345678;
+
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, initialValue);
+    let result = yield adapter.saveLastModified(intendedValue);
+    do_check_eq(result, intendedValue);
+    lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, intendedValue);
+
+    // test saveLastModified parses values correctly
+    result = yield adapter.saveLastModified(" " + intendedValue + " blah");
+    // should resolve with the parsed int
+    do_check_eq(result, intendedValue);
+    // and should have saved correctly
+    lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, intendedValue);
+    yield adapter.close();
+  });
+
+  // test loadDump(records)
+  add_task(function* test_kinto_import_records() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    let record1 = {id: 1, foo: "bar"};
+    let record2 = {id: 2, foo: "baz"};
+    let impactedRecords = yield adapter.loadDump([
+      record1, record2
+    ]);
+    do_check_eq(impactedRecords.length, 2);
+    let newRecord1 = yield adapter.get("1");
+    // ensure the record is the same as when it was added
+    deepEqual(record1, newRecord1);
+    let newRecord2 = yield adapter.get("2");
+    // ensure the record is the same as when it was added
+    deepEqual(record2, newRecord2);
+    yield adapter.close();
+  });
+
+  add_task(function* test_kinto_import_records_should_override_existing() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.clear();
+    records = yield adapter.list();
+    do_check_eq(records.length, 0);
+    let impactedRecords = yield adapter.loadDump([
+      {id: 1, foo: "bar"},
+      {id: 2, foo: "baz"},
+    ]);
+    do_check_eq(impactedRecords.length, 2);
+    yield adapter.loadDump([
+      {id: 1, foo: "baz"},
+      {id: 3, foo: "bab"},
+    ]);
+    records = yield adapter.list();
+    do_check_eq(records.length, 3);
+    let newRecord1 = yield adapter.get("1");
+    deepEqual(newRecord1.foo, "baz");
+    yield adapter.close();
+  });
+
+  add_task(function* test_import_updates_lastModified() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.loadDump([
+      {id: 1, foo: "bar", last_modified: 1457896541},
+      {id: 2, foo: "baz", last_modified: 1458796542},
+    ]);
+    let lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, 1458796542);
+    yield adapter.close();
+  });
+
+  add_task(function* test_import_preserves_older_lastModified() {
+    let adapter = do_get_kinto_adapter();
+    yield adapter.open();
+    yield adapter.saveLastModified(1458796543);
+
+    yield adapter.loadDump([
+      {id: 1, foo: "bar", last_modified: 1457896541},
+      {id: 2, foo: "baz", last_modified: 1458796542},
+    ]);
+    let lastModified = yield adapter.getLastModified();
+    do_check_eq(lastModified, 1458796543);
+    yield adapter.close();
+  });
+}
+
+// test kinto db setup and operations in various scenarios
+// test from scratch - no current existing database
+add_test(function test_db_creation() {
+  add_test(function test_create_from_scratch() {
+    // ensure the file does not exist in the profile
+    let kintoDB = do_get_kinto_db();
+    do_check_false(kintoDB.exists());
+    run_next_test();
+  });
+
+  test_collection_operations();
+
+  cleanup_kinto();
+  run_next_test();
+});
+
+// this is the closest we can get to a schema version upgrade at v1 - test an
+// existing database
+add_test(function test_creation_from_empty_db() {
+  add_test(function test_create_from_empty_db() {
+    // place an empty kinto db file in the profile
+    let profile = do_get_profile();
+    let kintoDB = do_get_kinto_db();
+
+    let emptyDB = do_get_file("test_storage_adapter/empty.sqlite");
+    emptyDB.copyTo(profile,kintoFilename);
+
+    run_next_test();
+  });
+
+  test_collection_operations();
+
+  cleanup_kinto();
+  run_next_test();
+});
+
+function run_test() {
+  run_next_test();
+}

services/common/tests/unit/test_storage_server.js

@@ -0,0 +1,692 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://testing-common/services/common/storageserver.js");
+
+const DEFAULT_USER = "123";
+const DEFAULT_PASSWORD = "password";
+
+/**
+ * Helper function to prepare a RESTRequest against the server.
+ */
+function localRequest(server, path, user=DEFAULT_USER, password=DEFAULT_PASSWORD) {
+  _("localRequest: " + path);
+  let identity = server.server.identity;
+  let url = identity.primaryScheme + "://" + identity.primaryHost + ":" +
+            identity.primaryPort + path;
+  _("url: " + url);
+  let req = new RESTRequest(url);
+
+  let header = basic_auth_header(user, password);
+  req.setHeader("Authorization", header);
+  req.setHeader("Accept", "application/json");
+
+  return req;
+}
+
+/**
+ * Helper function to validate an HTTP response from the server.
+ */
+function validateResponse(response) {
+  do_check_true("x-timestamp" in response.headers);
+
+  if ("content-length" in response.headers) {
+    let cl = parseInt(response.headers["content-length"]);
+
+    if (cl != 0) {
+      do_check_true("content-type" in response.headers);
+      do_check_eq("application/json", response.headers["content-type"]);
+    }
+  }
+
+  if (response.status == 204 || response.status == 304) {
+    do_check_false("content-type" in response.headers);
+
+    if ("content-length" in response.headers) {
+      do_check_eq(response.headers["content-length"], "0");
+    }
+  }
+
+  if (response.status == 405) {
+    do_check_true("allow" in response.headers);
+  }
+}
+
+/**
+ * Helper function to synchronously wait for a response and validate it.
+ */
+function waitAndValidateResponse(cb, request) {
+  let error = cb.wait();
+
+  if (!error) {
+    validateResponse(request.response);
+  }
+
+  return error;
+}
+
+/**
+ * Helper function to synchronously perform a GET request.
+ *
+ * @return Error instance or null if no error.
+ */
+function doGetRequest(request) {
+  let cb = Async.makeSpinningCallback();
+  request.get(cb);
+
+  return waitAndValidateResponse(cb, request);
+}
+
+/**
+ * Helper function to synchronously perform a PUT request.
+ *
+ * @return Error instance or null if no error.
+ */
+function doPutRequest(request, data) {
+  let cb = Async.makeSpinningCallback();
+  request.put(data, cb);
+
+  return waitAndValidateResponse(cb, request);
+}
+
+/**
+ * Helper function to synchronously perform a DELETE request.
+ *
+ * @return Error or null if no error was encountered.
+ */
+function doDeleteRequest(request) {
+  let cb = Async.makeSpinningCallback();
+  request.delete(cb);
+
+  return waitAndValidateResponse(cb, request);
+}
+
+function run_test() {
+  Log.repository.getLogger("Services.Common.Test.StorageServer").level =
+    Log.Level.Trace;
+  initTestLogging();
+
+  run_next_test();
+}
+
+add_test(function test_creation() {
+  _("Ensure a simple server can be created.");
+
+  // Explicit callback for this one.
+  let server = new StorageServer({
+    __proto__: StorageServerCallback,
+  });
+  do_check_true(!!server);
+
+  server.start(-1, function () {
+    _("Started on " + server.port);
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_synchronous_start() {
+  _("Ensure starting using startSynchronous works.");
+
+  let server = new StorageServer();
+  server.startSynchronous();
+  server.stop(run_next_test);
+});
+
+add_test(function test_url_parsing() {
+  _("Ensure server parses URLs properly.");
+
+  let server = new StorageServer();
+
+  // Check that we can parse a BSO URI.
+  let parts = server.pathRE.exec("/2.0/12345/storage/crypto/keys");
+  let [all, version, user, first, rest] = parts;
+  do_check_eq(all, "/2.0/12345/storage/crypto/keys");
+  do_check_eq(version, "2.0");
+  do_check_eq(user, "12345");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, "crypto/keys");
+  do_check_eq(null, server.pathRE.exec("/nothing/else"));
+
+  // Check that we can parse a collection URI.
+  parts = server.pathRE.exec("/2.0/123/storage/crypto");
+  [all, version, user, first, rest] = parts;
+  do_check_eq(all, "/2.0/123/storage/crypto");
+  do_check_eq(version, "2.0");
+  do_check_eq(user, "123");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, "crypto");
+
+  // We don't allow trailing slash on storage URI.
+  parts = server.pathRE.exec("/2.0/1234/storage/");
+  do_check_eq(parts, undefined);
+
+  // storage alone is a valid request.
+  parts = server.pathRE.exec("/2.0/123456/storage");
+  [all, version, user, first, rest] = parts;
+  do_check_eq(all, "/2.0/123456/storage");
+  do_check_eq(version, "2.0");
+  do_check_eq(user, "123456");
+  do_check_eq(first, "storage");
+  do_check_eq(rest, undefined);
+
+  parts = server.storageRE.exec("storage");
+  let storage, collection, id;
+  [all, storage, collection, id] = parts;
+  do_check_eq(all, "storage");
+  do_check_eq(collection, undefined);
+
+  run_next_test();
+});
+
+add_test(function test_basic_http() {
+  let server = new StorageServer();
+  server.registerUser("345", "password");
+  do_check_true(server.userExists("345"));
+  server.startSynchronous();
+
+  _("Started on " + server.port);
+  do_check_eq(server.requestCount, 0);
+  let req = localRequest(server, "/2.0/storage/crypto/keys");
+  _("req is " + req);
+  req.get(function (err) {
+    do_check_eq(null, err);
+    do_check_eq(server.requestCount, 1);
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_info_collections() {
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let path = "/2.0/123/info/collections";
+
+  _("info/collections on empty server should be empty object.");
+  let request = localRequest(server, path, "123", "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 200);
+  do_check_eq(request.response.body, "{}");
+
+  _("Creating an empty collection should result in collection appearing.");
+  let coll = server.createCollection("123", "col1");
+  request = localRequest(server, path, "123", "password");
+  error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 200);
+  let info = JSON.parse(request.response.body);
+  do_check_attribute_count(info, 1);
+  do_check_true("col1" in info);
+  do_check_eq(info.col1, coll.timestamp);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_get_existing() {
+  _("Ensure that BSO retrieval works.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {"bso": {"foo": "bar"}}
+  });
+  server.startSynchronous();
+
+  let coll = server.user("123").collection("test");
+
+  let request = localRequest(server, "/2.0/123/storage/test/bso", "123",
+                             "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 200);
+  do_check_eq(request.response.headers["content-type"], "application/json");
+  let bso = JSON.parse(request.response.body);
+  do_check_attribute_count(bso, 3);
+  do_check_eq(bso.id, "bso");
+  do_check_eq(bso.modified, coll.bso("bso").modified);
+  let payload = JSON.parse(bso.payload);
+  do_check_attribute_count(payload, 1);
+  do_check_eq(payload.foo, "bar");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_percent_decoding() {
+  _("Ensure query string arguments with percent encoded are handled.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let coll = server.user("123").createCollection("test");
+  coll.insert("001", {foo: "bar"});
+  coll.insert("002", {bar: "foo"});
+
+  let request = localRequest(server, "/2.0/123/storage/test?ids=001%2C002",
+                             "123", "password");
+  let error = doGetRequest(request);
+  do_check_null(error);
+  do_check_eq(request.response.status, 200);
+  let items = JSON.parse(request.response.body).items;
+  do_check_attribute_count(items, 2);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_404() {
+  _("Ensure the server responds with a 404 if a BSO does not exist.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {}
+  });
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage/test/foo");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+
+  do_check_eq(request.response.status, 404);
+  do_check_false("content-type" in request.response.headers);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_if_modified_since_304() {
+  _("Ensure the server responds properly to X-If-Modified-Since for BSOs.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {bso: {foo: "bar"}}
+  });
+  server.startSynchronous();
+
+  let coll = server.user("123").collection("test");
+  do_check_neq(coll, null);
+
+  // Rewind clock just in case.
+  coll.timestamp -= 10000;
+  coll.bso("bso").modified -= 10000;
+
+  let request = localRequest(server, "/2.0/123/storage/test/bso",
+                             "123", "password");
+  request.setHeader("X-If-Modified-Since", "" + server.serverTime());
+  let error = doGetRequest(request);
+  do_check_eq(null, error);
+
+  do_check_eq(request.response.status, 304);
+  do_check_false("content-type" in request.response.headers);
+
+  request = localRequest(server, "/2.0/123/storage/test/bso",
+                             "123", "password");
+  request.setHeader("X-If-Modified-Since", "" + (server.serverTime() - 20000));
+  error = doGetRequest(request);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 200);
+  do_check_eq(request.response.headers["content-type"], "application/json");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_if_unmodified_since() {
+  _("Ensure X-If-Unmodified-Since works properly on BSOs.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {bso: {foo: "bar"}}
+  });
+  server.startSynchronous();
+
+  let coll = server.user("123").collection("test");
+  do_check_neq(coll, null);
+
+  let time = coll.bso("bso").modified;
+
+  _("Ensure we get a 412 for specified times older than server time.");
+  let request = localRequest(server, "/2.0/123/storage/test/bso",
+                             "123", "password");
+  request.setHeader("X-If-Unmodified-Since", time - 5000);
+  request.setHeader("Content-Type", "application/json");
+  let payload = JSON.stringify({"payload": "foobar"});
+  let error = doPutRequest(request, payload);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 412);
+
+  _("Ensure we get a 204 if update goes through.");
+  request = localRequest(server, "/2.0/123/storage/test/bso",
+                         "123", "password");
+  request.setHeader("Content-Type", "application/json");
+  request.setHeader("X-If-Unmodified-Since", time + 1);
+  error = doPutRequest(request, payload);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 204);
+  do_check_true(coll.timestamp > time);
+
+  // Not sure why a client would send X-If-Unmodified-Since if a BSO doesn't
+  // exist. But, why not test it?
+  _("Ensure we get a 201 if creation goes through.");
+  request = localRequest(server, "/2.0/123/storage/test/none",
+                         "123", "password");
+  request.setHeader("Content-Type", "application/json");
+  request.setHeader("X-If-Unmodified-Since", time);
+  error = doPutRequest(request, payload);
+  do_check_eq(null, error);
+  do_check_eq(request.response.status, 201);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_delete_not_exist() {
+  _("Ensure server behaves properly when deleting a BSO that does not exist.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.user("123").createCollection("empty");
+  server.startSynchronous();
+
+  server.callback.onItemDeleted = function onItemDeleted(username, collection,
+                                                         id) {
+    do_throw("onItemDeleted should not have been called.");
+  };
+
+  let request = localRequest(server, "/2.0/123/storage/empty/nada",
+                             "123", "password");
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 404);
+  do_check_false("content-type" in request.response.headers);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_delete_exists() {
+  _("Ensure proper semantics when deleting a BSO that exists.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let coll = server.user("123").createCollection("test");
+  let bso = coll.insert("myid", {foo: "bar"});
+  let timestamp = coll.timestamp;
+
+  server.callback.onItemDeleted = function onDeleted(username, collection, id) {
+    delete server.callback.onItemDeleted;
+    do_check_eq(username, "123");
+    do_check_eq(collection, "test");
+    do_check_eq(id, "myid");
+  };
+
+  let request = localRequest(server, "/2.0/123/storage/test/myid",
+                             "123", "password");
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 204);
+  do_check_eq(coll.bsos().length, 0);
+  do_check_true(coll.timestamp > timestamp);
+
+  _("On next request the BSO should not exist.");
+  request = localRequest(server, "/2.0/123/storage/test/myid",
+                         "123", "password");
+  error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 404);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_delete_unmodified() {
+  _("Ensure X-If-Unmodified-Since works when deleting BSOs.");
+
+  let server = new StorageServer();
+  server.startSynchronous();
+  server.registerUser("123", "password");
+  let coll = server.user("123").createCollection("test");
+  let bso = coll.insert("myid", {foo: "bar"});
+
+  let modified = bso.modified;
+
+  _("Issuing a DELETE with an older time should fail.");
+  let path = "/2.0/123/storage/test/myid";
+  let request = localRequest(server, path, "123", "password");
+  request.setHeader("X-If-Unmodified-Since", modified - 1000);
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 412);
+  do_check_false("content-type" in request.response.headers);
+  do_check_neq(coll.bso("myid"), null);
+
+  _("Issuing a DELETE with a newer time should work.");
+  request = localRequest(server, path, "123", "password");
+  request.setHeader("X-If-Unmodified-Since", modified + 1000);
+  error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 204);
+  do_check_true(coll.bso("myid").deleted);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_collection_get_unmodified_since() {
+  _("Ensure conditional unmodified get on collection works when it should.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+  let collection = server.user("123").createCollection("testcoll");
+  collection.insert("bso0", {foo: "bar"});
+
+  let serverModified = collection.timestamp;
+
+  let request1 = localRequest(server, "/2.0/123/storage/testcoll",
+                              "123", "password");
+  request1.setHeader("X-If-Unmodified-Since", serverModified);
+  let error = doGetRequest(request1);
+  do_check_null(error);
+  do_check_eq(request1.response.status, 200);
+
+  let request2 = localRequest(server, "/2.0/123/storage/testcoll",
+                              "123", "password");
+  request2.setHeader("X-If-Unmodified-Since", serverModified - 1);
+  error = doGetRequest(request2);
+  do_check_null(error);
+  do_check_eq(request2.response.status, 412);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_bso_get_unmodified_since() {
+  _("Ensure conditional unmodified get on BSO works appropriately.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+  let collection = server.user("123").createCollection("testcoll");
+  let bso = collection.insert("bso0", {foo: "bar"});
+
+  let serverModified = bso.modified;
+
+  let request1 = localRequest(server, "/2.0/123/storage/testcoll/bso0",
+                              "123", "password");
+  request1.setHeader("X-If-Unmodified-Since", serverModified);
+  let error = doGetRequest(request1);
+  do_check_null(error);
+  do_check_eq(request1.response.status, 200);
+
+  let request2 = localRequest(server, "/2.0/123/storage/testcoll/bso0",
+                              "123", "password");
+  request2.setHeader("X-If-Unmodified-Since", serverModified - 1);
+  error = doGetRequest(request2);
+  do_check_null(error);
+  do_check_eq(request2.response.status, 412);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_missing_collection_404() {
+  _("Ensure a missing collection returns a 404.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage/none", "123", "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 404);
+  do_check_false("content-type" in request.response.headers);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_get_storage_405() {
+  _("Ensure that a GET on /storage results in a 405.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage", "123", "password");
+  let error = doGetRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 405);
+  do_check_eq(request.response.headers["allow"], "DELETE");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_delete_storage() {
+  _("Ensure that deleting all of storage works.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    foo: {a: {foo: "bar"}, b: {bar: "foo"}},
+    baz: {c: {bob: "law"}, blah: {law: "blog"}}
+  });
+
+  server.startSynchronous();
+
+  let request = localRequest(server, "/2.0/123/storage", "123", "password");
+  let error = doDeleteRequest(request);
+  do_check_eq(error, null);
+  do_check_eq(request.response.status, 204);
+  do_check_attribute_count(server.users["123"].collections, 0);
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_x_num_records() {
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+
+  server.createContents("123", {
+    crypto: {foos: {foo: "bar"},
+             bars: {foo: "baz"}}
+  });
+  server.startSynchronous();
+  let bso = localRequest(server, "/2.0/123/storage/crypto/foos");
+  bso.get(function (err) {
+    // BSO fetches don't have one.
+    do_check_false("x-num-records" in this.response.headers);
+    let col = localRequest(server, "/2.0/123/storage/crypto");
+    col.get(function (err) {
+      // Collection fetches do.
+      do_check_eq(this.response.headers["x-num-records"], "2");
+      server.stop(run_next_test);
+    });
+  });
+});
+
+add_test(function test_put_delete_put() {
+  _("Bug 790397: Ensure BSO deleted flag is reset on PUT.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.createContents("123", {
+    test: {bso: {foo: "bar"}}
+  });
+  server.startSynchronous();
+
+  _("Ensure we can PUT an existing record.");
+  let request1 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  request1.setHeader("Content-Type", "application/json");
+  let payload1 = JSON.stringify({"payload": "foobar"});
+  let error1 = doPutRequest(request1, payload1);
+  do_check_eq(null, error1);
+  do_check_eq(request1.response.status, 204);
+
+  _("Ensure we can DELETE it.");
+  let request2 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  let error2 = doDeleteRequest(request2);
+  do_check_eq(error2, null);
+  do_check_eq(request2.response.status, 204);
+  do_check_false("content-type" in request2.response.headers);
+
+  _("Ensure we can PUT a previously deleted record.");
+  let request3 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  request3.setHeader("Content-Type", "application/json");
+  let payload3 = JSON.stringify({"payload": "foobar"});
+  let error3 = doPutRequest(request3, payload3);
+  do_check_eq(null, error3);
+  do_check_eq(request3.response.status, 201);
+
+  _("Ensure we can GET the re-uploaded record.");
+  let request4 = localRequest(server, "/2.0/123/storage/test/bso", "123", "password");
+  let error4 = doGetRequest(request4);
+  do_check_eq(error4, null);
+  do_check_eq(request4.response.status, 200);
+  do_check_eq(request4.response.headers["content-type"], "application/json");
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_collection_get_newer() {
+  _("Ensure get with newer argument on collection works.");
+
+  let server = new StorageServer();
+  server.registerUser("123", "password");
+  server.startSynchronous();
+
+  let coll = server.user("123").createCollection("test");
+  let bso1 = coll.insert("001", {foo: "bar"});
+  let bso2 = coll.insert("002", {bar: "foo"});
+
+  // Don't want both records to have the same timestamp.
+  bso2.modified = bso1.modified + 1000;
+
+  function newerRequest(newer) {
+    return localRequest(server, "/2.0/123/storage/test?newer=" + newer,
+                        "123", "password");
+  }
+
+  let request1 = newerRequest(0);
+  let error1 = doGetRequest(request1);
+  do_check_null(error1);
+  do_check_eq(request1.response.status, 200);
+  let items1 = JSON.parse(request1.response.body).items;
+  do_check_attribute_count(items1, 2);
+
+  let request2 = newerRequest(bso1.modified + 1);
+  let error2 = doGetRequest(request2);
+  do_check_null(error2);
+  do_check_eq(request2.response.status, 200);
+  let items2 = JSON.parse(request2.response.body).items;
+  do_check_attribute_count(items2, 1);
+
+  let request3 = newerRequest(bso2.modified + 1);
+  let error3 = doGetRequest(request3);
+  do_check_null(error3);
+  do_check_eq(request3.response.status, 200);
+  let items3 = JSON.parse(request3.response.body).items;
+  do_check_attribute_count(items3, 0);
+
+  server.stop(run_next_test);
+});

services/common/tests/unit/test_tokenauthenticatedrequest.js

@@ -0,0 +1,52 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  initTestLogging("Trace");
+  run_next_test();
+}
+
+add_test(function test_authenticated_request() {
+  _("Ensure that sending a MAC authenticated GET request works as expected.");
+
+  let message = "Great Success!";
+
+  // TODO: We use a preset key here, but use getTokenFromBrowserIDAssertion()
+  // from TokenServerClient to get a real one when possible. (Bug 745800)
+  let id = "eyJleHBpcmVzIjogMTM2NTAxMDg5OC4x";
+  let key = "qTZf4ZFpAMpMoeSsX3zVRjiqmNs=";
+  let method = "GET";
+
+  let nonce = btoa(CryptoUtils.generateRandomBytes(16));
+  let ts = Math.floor(Date.now() / 1000);
+  let extra = {ts: ts, nonce: nonce};
+
+  let auth;
+
+  let server = httpd_setup({"/foo": function(request, response) {
+      do_check_true(request.hasHeader("Authorization"));
+      do_check_eq(auth, request.getHeader("Authorization"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.bodyOutputStream.write(message, message.length);
+    }
+  });
+  let uri = CommonUtils.makeURI(server.baseURI + "/foo");
+  let sig = CryptoUtils.computeHTTPMACSHA1(id, key, method, uri, extra);
+  auth = sig.getHeader();
+
+  let req = new TokenAuthenticatedRESTRequest(uri, {id: id, key: key}, extra);
+  let cb = Async.makeSpinningCallback();
+  req.get(cb);
+  let result = cb.wait();
+
+  do_check_eq(null, result);
+  do_check_eq(message, req.response.body);
+
+  server.stop(run_next_test);
+});

services/common/tests/unit/test_tokenserverclient.js

@@ -0,0 +1,466 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/tokenserverclient.js");
+
+function run_test() {
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+add_test(function test_working_bid_exchange() {
+  _("Ensure that working BrowserID token exchange works as expected.");
+
+  let service = "http://example.com/foo";
+  let duration = 300;
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      do_check_true(request.hasHeader("accept"));
+      do_check_false(request.hasHeader("x-conditions-accepted"));
+      do_check_eq("application/json", request.getHeader("accept"));
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: service,
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let cb = Async.makeSpinningCallback();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", cb);
+  let result = cb.wait();
+  do_check_eq("object", typeof(result));
+  do_check_attribute_count(result, 6);
+  do_check_eq(service, result.endpoint);
+  do_check_eq("id", result.id);
+  do_check_eq("key", result.key);
+  do_check_eq("uid", result.uid);
+  do_check_eq(duration, result.duration);
+  server.stop(run_next_test);
+});
+
+add_test(function test_invalid_arguments() {
+  _("Ensure invalid arguments to APIs are rejected.");
+
+  let args = [
+    [null, "assertion", function() {}],
+    ["http://example.com/", null, function() {}],
+    ["http://example.com/", "assertion", null]
+  ];
+
+  for (let arg of args) {
+    try {
+      let client = new TokenServerClient();
+      client.getTokenFromBrowserIDAssertion(arg[0], arg[1], arg[2]);
+      do_throw("Should never get here.");
+    } catch (ex) {
+      do_check_true(ex instanceof TokenServerClientError);
+    }
+  }
+
+  run_next_test();
+});
+
+add_test(function test_conditions_required_response_handling() {
+  _("Ensure that a conditions required response is handled properly.");
+
+  let description = "Need to accept conditions";
+  let tosURL = "http://example.com/tos";
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      do_check_false(request.hasHeader("x-conditions-accepted"));
+
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        errors: [{description: description, location: "body", name: ""}],
+        urls: {tos: tosURL}
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "conditions-required");
+    // Check a JSON.stringify works on our errors as our logging will try and use it.
+    do_check_true(JSON.stringify(error), "JSON.stringify worked");
+    do_check_null(token);
+
+    do_check_eq(error.urls.tos, tosURL);
+
+    server.stop(run_next_test);
+  }
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_invalid_403_no_content_type() {
+  _("Ensure that a 403 without content-type is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      // No Content-Type header by design.
+
+      let body = JSON.stringify({
+        errors: [{description: "irrelevant", location: "body", name: ""}],
+        urls: {foo: "http://bar"}
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_null(token);
+
+    do_check_null(error.urls);
+
+    server.stop(run_next_test);
+  }
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_invalid_403_bad_json() {
+  _("Ensure that a 403 with JSON that isn't proper is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = JSON.stringify({
+        foo: "bar"
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_null(token);
+    do_check_null(error.urls);
+
+    server.stop(run_next_test);
+  }
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_403_no_urls() {
+  _("Ensure that a 403 without a urls field is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 403, "Forbidden");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = "{}";
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion",
+                                        function onResponse(error, result) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_null(result);
+
+    server.stop(run_next_test);
+
+  });
+});
+
+add_test(function test_send_extra_headers() {
+  _("Ensures that the condition acceptance header is sent when asked.");
+
+  let duration = 300;
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      do_check_true(request.hasHeader("x-foo"));
+      do_check_eq(request.getHeader("x-foo"), "42");
+
+      do_check_true(request.hasHeader("x-bar"));
+      do_check_eq(request.getHeader("x-bar"), "17");
+
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: "http://example.com/",
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+
+  function onResponse(error, token) {
+    do_check_null(error);
+
+    // Other tests validate other things.
+
+    server.stop(run_next_test);
+  }
+
+  let extra = {
+    "X-Foo": 42,
+    "X-Bar": 17
+  };
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse, extra);
+});
+
+add_test(function test_error_404_empty() {
+  _("Ensure that 404 responses without proper response are handled properly.");
+
+  let server = httpd_setup();
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/foo";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "malformed-response");
+
+    do_check_neq(null, error.response);
+    do_check_null(r);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_error_404_proper_response() {
+  _("Ensure that a Cornice error report for 404 is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 404, "Not Found");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = JSON.stringify({
+        status: 404,
+        errors: [{description: "No service", location: "body", name: ""}],
+      });
+
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  function onResponse(error, token) {
+    do_check_true(error instanceof TokenServerClientServerError);
+    do_check_eq(error.cause, "unknown-service");
+    do_check_null(token);
+
+    server.stop(run_next_test);
+  }
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", onResponse);
+});
+
+add_test(function test_bad_json() {
+  _("Ensure that malformed JSON is handled properly.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = '{"id": "id", baz}'
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_eq(error.cause, "malformed-response");
+    do_check_neq(null, error.response);
+    do_check_eq(null, r);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_400_response() {
+  _("Ensure HTTP 400 is converted to malformed-request.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 400, "Bad Request");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = "{}"; // Actual content may not be used.
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_neq(null, error.response);
+    do_check_eq(error.cause, "malformed-request");
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_401_with_error_cause() {
+  _("Ensure 401 cause is specified in body.status");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 401, "Unauthorized");
+      response.setHeader("Content-Type", "application/json; charset=utf-8");
+
+      let body = JSON.stringify({status: "no-soup-for-you"});
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let client = new TokenServerClient();
+  let url = server.baseURI + "/1.0/foo/1.0";
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_neq(null, error.response);
+    do_check_eq(error.cause, "no-soup-for-you");
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_unhandled_media_type() {
+  _("Ensure that unhandled media types throw an error.");
+
+  let server = httpd_setup({
+    "/1.0/foo/1.0": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "text/plain");
+
+      let body = "hello, world";
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let url = server.baseURI + "/1.0/foo/1.0";
+  let client = new TokenServerClient();
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_neq(null, error);
+    do_check_eq("TokenServerClientServerError", error.name);
+    do_check_neq(null, error.response);
+    do_check_eq(null, r);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_rich_media_types() {
+  _("Ensure that extra tokens in the media type aren't rejected.");
+
+  let duration = 300;
+  let server = httpd_setup({
+    "/foo": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json; foo=bar; bar=foo");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: "foo",
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let url = server.baseURI + "/foo";
+  let client = new TokenServerClient();
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_eq(null, error);
+
+    server.stop(run_next_test);
+  });
+});
+
+add_test(function test_exception_during_callback() {
+  _("Ensure that exceptions thrown during callback handling are handled.");
+
+  let duration = 300;
+  let server = httpd_setup({
+    "/foo": function(request, response) {
+      response.setStatusLine(request.httpVersion, 200, "OK");
+      response.setHeader("Content-Type", "application/json");
+
+      let body = JSON.stringify({
+        id:           "id",
+        key:          "key",
+        api_endpoint: "foo",
+        uid:          "uid",
+        duration:     duration,
+      });
+      response.bodyOutputStream.write(body, body.length);
+    }
+  });
+
+  let url = server.baseURI + "/foo";
+  let client = new TokenServerClient();
+  let cb = Async.makeSpinningCallback();
+  let callbackCount = 0;
+
+  client.getTokenFromBrowserIDAssertion(url, "assertion", function(error, r) {
+    do_check_eq(null, error);
+
+    cb();
+
+    callbackCount += 1;
+    throw new Error("I am a bad function!");
+  });
+
+  cb.wait();
+  // This relies on some heavy event loop magic. The error in the main
+  // callback should already have been raised at this point.
+  do_check_eq(callbackCount, 1);
+
+  server.stop(run_next_test);
+});

services/common/tests/unit/test_utils_atob.js

@@ -0,0 +1,11 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  let data = ["Zm9vYmE=", "Zm9vYmE==", "Zm9vYmE==="];
+  for (let d in data) {
+    do_check_eq(CommonUtils.safeAtoB(data[d]), "fooba");
+  }
+}

services/common/tests/unit/test_utils_convert_string.js

@@ -0,0 +1,132 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-common/utils.js");
+
+// A wise line of Greek verse, and the utf-8 byte encoding.
+// N.b., Greek begins at utf-8 ce 91
+const TEST_STR = "πόλλ' οἶδ' ἀλώπηξ, ἀλλ' ἐχῖνος ἓν μέγα";
+const TEST_HEX = h("cf 80 cf 8c ce bb ce bb   27 20 ce bf e1 bc b6 ce"+
+                   "b4 27 20 e1 bc 80 ce bb   cf 8e cf 80 ce b7 ce be"+
+                   "2c 20 e1 bc 80 ce bb ce   bb 27 20 e1 bc 90 cf 87"+
+                   "e1 bf 96 ce bd ce bf cf   82 20 e1 bc 93 ce bd 20"+
+                   "ce bc ce ad ce b3 ce b1");
+// Integer byte values for the above
+const TEST_BYTES = [207,128,207,140,206,187,206,187,
+                     39, 32,206,191,225,188,182,206,
+                    180, 39, 32,225,188,128,206,187,
+                    207,142,207,128,206,183,206,190,
+                     44, 32,225,188,128,206,187,206,
+                    187, 39, 32,225,188,144,207,135,
+                    225,191,150,206,189,206,191,207,
+                    130, 32,225,188,147,206,189, 32,
+                    206,188,206,173,206,179,206,177];
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_compress_string() {
+  const INPUT = "hello";
+
+  let result = CommonUtils.convertString(INPUT, "uncompressed", "deflate");
+  do_check_eq(result.length, 13);
+
+  let result2 = CommonUtils.convertString(INPUT, "uncompressed", "deflate");
+  do_check_eq(result, result2);
+
+  let result3 = CommonUtils.convertString(result, "deflate", "uncompressed");
+  do_check_eq(result3, INPUT);
+
+  run_next_test();
+});
+
+add_test(function test_compress_utf8() {
+  const INPUT = "Árvíztűrő tükörfúrógép いろはにほへとちりぬるを Pijamalı hasta, yağız şoföre çabucak güvendi.";
+  let inputUTF8 = CommonUtils.encodeUTF8(INPUT);
+
+  let compressed = CommonUtils.convertString(inputUTF8, "uncompressed", "deflate");
+  let uncompressed = CommonUtils.convertString(compressed, "deflate", "uncompressed");
+
+  do_check_eq(uncompressed, inputUTF8);
+
+  let outputUTF8 = CommonUtils.decodeUTF8(uncompressed);
+  do_check_eq(outputUTF8, INPUT);
+
+  run_next_test();
+});
+
+add_test(function test_bad_argument() {
+  let failed = false;
+  try {
+    CommonUtils.convertString(null, "uncompressed", "deflate");
+  } catch (ex) {
+    failed = true;
+    do_check_true(ex.message.startsWith("Input string must be defined"));
+  } finally {
+    do_check_true(failed);
+  }
+
+  run_next_test();
+});
+
+add_task(function test_stringAsHex() {
+  do_check_eq(TEST_HEX, CommonUtils.stringAsHex(TEST_STR));
+});
+
+add_task(function test_hexAsString() {
+  do_check_eq(TEST_STR, CommonUtils.hexAsString(TEST_HEX));
+});
+
+add_task(function test_hexToBytes() {
+  let bytes = CommonUtils.hexToBytes(TEST_HEX);
+  do_check_eq(TEST_BYTES.length, bytes.length);
+  // Ensure that the decimal values of each byte are correct
+  do_check_true(arraysEqual(TEST_BYTES,
+      CommonUtils.stringToByteArray(bytes)));
+});
+
+add_task(function test_bytesToHex() {
+  // Create a list of our character bytes from the reference int values
+  let bytes = CommonUtils.byteArrayToString(TEST_BYTES);
+  do_check_eq(TEST_HEX, CommonUtils.bytesAsHex(bytes));
+});
+
+add_task(function test_stringToBytes() {
+  do_check_true(arraysEqual(TEST_BYTES,
+      CommonUtils.stringToByteArray(CommonUtils.stringToBytes(TEST_STR))));
+});
+
+add_task(function test_stringRoundTrip() {
+  do_check_eq(TEST_STR,
+    CommonUtils.hexAsString(CommonUtils.stringAsHex(TEST_STR)));
+});
+
+add_task(function test_hexRoundTrip() {
+  do_check_eq(TEST_HEX,
+    CommonUtils.stringAsHex(CommonUtils.hexAsString(TEST_HEX)));
+});
+
+add_task(function test_byteArrayRoundTrip() {
+  do_check_true(arraysEqual(TEST_BYTES,
+    CommonUtils.stringToByteArray(CommonUtils.byteArrayToString(TEST_BYTES))));
+});
+
+// turn formatted test vectors into normal hex strings
+function h(hexStr) {
+  return hexStr.replace(/\s+/g, "");
+}
+
+function arraysEqual(a1, a2) {
+  if (a1.length !== a2.length) {
+    return false;
+  }
+  for (let i = 0; i < a1.length; i++) {
+    if (a1[i] !== a2[i]) {
+      return false;
+    }
+  }
+  return true;
+}

services/common/tests/unit/test_utils_dateprefs.js

@@ -0,0 +1,85 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://services-common/utils.js");
+
+
+var prefs = new Preferences("servicescommon.tests.");
+
+function DummyLogger() {
+  this.messages = [];
+}
+DummyLogger.prototype.warn = function warn(message) {
+  this.messages.push(message);
+};
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_set_basic() {
+  let now = new Date();
+
+  CommonUtils.setDatePref(prefs, "test00", now);
+  let value = prefs.get("test00");
+  do_check_eq(value, "" + now.getTime());
+
+  let now2 = CommonUtils.getDatePref(prefs, "test00");
+
+  do_check_eq(now.getTime(), now2.getTime());
+
+  run_next_test();
+});
+
+add_test(function test_set_bounds_checking() {
+  let d = new Date(2342354);
+
+  let failed = false;
+  try {
+    CommonUtils.setDatePref(prefs, "test01", d);
+  } catch (ex) {
+    do_check_true(ex.message.startsWith("Trying to set"));
+    failed = true;
+  }
+
+  do_check_true(failed);
+  run_next_test();
+});
+
+add_test(function test_get_bounds_checking() {
+  prefs.set("test_bounds_checking", "13241431");
+
+  let log = new DummyLogger();
+  let d = CommonUtils.getDatePref(prefs, "test_bounds_checking", 0, log);
+  do_check_eq(d.getTime(), 0);
+  do_check_eq(log.messages.length, 1);
+
+  run_next_test();
+});
+
+add_test(function test_get_bad_default() {
+  let failed = false;
+  try {
+    CommonUtils.getDatePref(prefs, "get_bad_default", new Date());
+  } catch (ex) {
+    do_check_true(ex.message.startsWith("Default value is not a number"));
+    failed = true;
+  }
+
+  do_check_true(failed);
+  run_next_test();
+});
+
+add_test(function test_get_invalid_number() {
+  prefs.set("get_invalid_number", "hello world");
+
+  let log = new DummyLogger();
+  let d = CommonUtils.getDatePref(prefs, "get_invalid_number", 42, log);
+  do_check_eq(d.getTime(), 42);
+  do_check_eq(log.messages.length, 1);
+
+  run_next_test();
+});

services/common/tests/unit/test_utils_deepCopy.js

@@ -0,0 +1,18 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://testing-common/services/common/utils.js");
+
+function run_test() {
+  let thing = {o: {foo: "foo", bar: ["bar"]}, a: ["foo", {bar: "bar"}]};
+  let ret = TestingUtils.deepCopy(thing);
+  do_check_neq(ret, thing)
+  do_check_neq(ret.o, thing.o);
+  do_check_neq(ret.o.bar, thing.o.bar);
+  do_check_neq(ret.a, thing.a);
+  do_check_neq(ret.a[1], thing.a[1]);
+  do_check_eq(ret.o.foo, thing.o.foo);
+  do_check_eq(ret.o.bar[0], thing.o.bar[0]);
+  do_check_eq(ret.a[0], thing.a[0]);
+  do_check_eq(ret.a[1].bar, thing.a[1].bar);
+}

services/common/tests/unit/test_utils_encodeBase32.js

@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  // Testing byte array manipulation.
+  do_check_eq("FOOBAR", CommonUtils.byteArrayToString([70, 79, 79, 66, 65, 82]));
+  do_check_eq("", CommonUtils.byteArrayToString([]));
+
+  _("Testing encoding...");
+  // Test vectors from RFC 4648
+  do_check_eq(CommonUtils.encodeBase32(""), "");
+  do_check_eq(CommonUtils.encodeBase32("f"), "MY======");
+  do_check_eq(CommonUtils.encodeBase32("fo"), "MZXQ====");
+  do_check_eq(CommonUtils.encodeBase32("foo"), "MZXW6===");
+  do_check_eq(CommonUtils.encodeBase32("foob"), "MZXW6YQ=");
+  do_check_eq(CommonUtils.encodeBase32("fooba"), "MZXW6YTB");
+  do_check_eq(CommonUtils.encodeBase32("foobar"), "MZXW6YTBOI======");
+
+  do_check_eq(CommonUtils.encodeBase32("Bacon is a vegetable."),
+              "IJQWG33OEBUXGIDBEB3GKZ3FORQWE3DFFY======");
+
+  _("Checking assumptions...");
+  for (let i = 0; i <= 255; ++i)
+    do_check_eq(undefined | i, i);
+
+  _("Testing decoding...");
+  do_check_eq(CommonUtils.decodeBase32(""), "");
+  do_check_eq(CommonUtils.decodeBase32("MY======"), "f");
+  do_check_eq(CommonUtils.decodeBase32("MZXQ===="), "fo");
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTB"), "fooba");
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTBOI======"), "foobar");
+
+  // Same with incorrect or missing padding.
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTBOI=="), "foobar");
+  do_check_eq(CommonUtils.decodeBase32("MZXW6YTBOI"), "foobar");
+
+  let encoded = CommonUtils.encodeBase32("Bacon is a vegetable.");
+  _("Encoded to " + JSON.stringify(encoded));
+  do_check_eq(CommonUtils.decodeBase32(encoded), "Bacon is a vegetable.");
+
+  // Test failure.
+  let err;
+  try {
+    CommonUtils.decodeBase32("000");
+  } catch (ex) {
+    err = ex;
+  }
+  do_check_eq(err, "Unknown character in base32: 0");
+}

services/common/tests/unit/test_utils_encodeBase64URL.js

@@ -0,0 +1,27 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_simple() {
+  let expected = {
+    hello: "aGVsbG8=",
+    "<>?": "PD4_",
+  };
+
+  for (let [k,v] of Object.entries(expected)) {
+    do_check_eq(CommonUtils.encodeBase64URL(k), v);
+  }
+
+  run_next_test();
+});
+
+add_test(function test_no_padding() {
+  do_check_eq(CommonUtils.encodeBase64URL("hello", false), "aGVsbG8");
+
+  run_next_test();
+});

services/common/tests/unit/test_utils_ensureMillisecondsTimestamp.js

@@ -0,0 +1,23 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp(null));
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp(0));
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp("0"));
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp("000"));
+
+    do_check_null(CommonUtils.ensureMillisecondsTimestamp(999 * 10000000000));
+
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(-1); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(1); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(1.5); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp(999 * 10000000000 + 0.5); });
+
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("-1"); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("1"); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("1.5"); });
+    do_check_throws(function err() { CommonUtils.ensureMillisecondsTimestamp("" + (999 * 10000000000 + 0.5)); });
+}

services/common/tests/unit/test_utils_json.js

@@ -0,0 +1,40 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://gre/modules/osfile.jsm");
+
+function run_test() {
+  initTestLogging();
+  run_next_test();
+}
+
+add_test(function test_writeJSON_readJSON() {
+  _("Round-trip some JSON through the promise-based JSON writer.");
+
+  let contents = {
+    "a": 12345.67,
+    "b": {
+      "c": "héllö",
+    },
+    "d": undefined,
+    "e": null,
+  };
+
+  function checkJSON(json) {
+    do_check_eq(contents.a, json.a);
+    do_check_eq(contents.b.c, json.b.c);
+    do_check_eq(contents.d, json.d);
+    do_check_eq(contents.e, json.e);
+    run_next_test();
+  };
+
+  function doRead() {
+    CommonUtils.readJSON(path)
+               .then(checkJSON, do_throw);
+  }
+
+  let path = OS.Path.join(OS.Constants.Path.profileDir, "bar.json");
+  CommonUtils.writeJSON(contents, path)
+             .then(doRead, do_throw);
+});

services/common/tests/unit/test_utils_makeURI.js

@@ -0,0 +1,66 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+_("Make sure uri strings are converted to nsIURIs");
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  _test_makeURI();
+}
+
+function _test_makeURI() {
+  _("Check http uris");
+  let uri1 = "http://mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uri1).spec, uri1);
+  let uri2 = "http://www.mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uri2).spec, uri2);
+  let uri3 = "http://mozillalabs.com/path";
+  do_check_eq(CommonUtils.makeURI(uri3).spec, uri3);
+  let uri4 = "http://mozillalabs.com/multi/path";
+  do_check_eq(CommonUtils.makeURI(uri4).spec, uri4);
+  let uri5 = "http://mozillalabs.com/?query";
+  do_check_eq(CommonUtils.makeURI(uri5).spec, uri5);
+  let uri6 = "http://mozillalabs.com/#hash";
+  do_check_eq(CommonUtils.makeURI(uri6).spec, uri6);
+
+  _("Check https uris");
+  let uris1 = "https://mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uris1).spec, uris1);
+  let uris2 = "https://www.mozillalabs.com/";
+  do_check_eq(CommonUtils.makeURI(uris2).spec, uris2);
+  let uris3 = "https://mozillalabs.com/path";
+  do_check_eq(CommonUtils.makeURI(uris3).spec, uris3);
+  let uris4 = "https://mozillalabs.com/multi/path";
+  do_check_eq(CommonUtils.makeURI(uris4).spec, uris4);
+  let uris5 = "https://mozillalabs.com/?query";
+  do_check_eq(CommonUtils.makeURI(uris5).spec, uris5);
+  let uris6 = "https://mozillalabs.com/#hash";
+  do_check_eq(CommonUtils.makeURI(uris6).spec, uris6);
+
+  _("Check chrome uris");
+  let uric1 = "chrome://browser/content/browser.xul";
+  do_check_eq(CommonUtils.makeURI(uric1).spec, uric1);
+  let uric2 = "chrome://browser/skin/browser.css";
+  do_check_eq(CommonUtils.makeURI(uric2).spec, uric2);
+  let uric3 = "chrome://browser/locale/browser.dtd";
+  do_check_eq(CommonUtils.makeURI(uric3).spec, uric3);
+
+  _("Check about uris");
+  let uria1 = "about:weave";
+  do_check_eq(CommonUtils.makeURI(uria1).spec, uria1);
+  let uria2 = "about:weave/";
+  do_check_eq(CommonUtils.makeURI(uria2).spec, uria2);
+  let uria3 = "about:weave/path";
+  do_check_eq(CommonUtils.makeURI(uria3).spec, uria3);
+  let uria4 = "about:weave/multi/path";
+  do_check_eq(CommonUtils.makeURI(uria4).spec, uria4);
+  let uria5 = "about:weave/?query";
+  do_check_eq(CommonUtils.makeURI(uria5).spec, uria5);
+  let uria6 = "about:weave/#hash";
+  do_check_eq(CommonUtils.makeURI(uria6).spec, uria6);
+
+  _("Invalid uris are undefined");
+  do_check_eq(CommonUtils.makeURI("mozillalabs.com"), undefined);
+  do_check_eq(CommonUtils.makeURI("chrome://badstuff"), undefined);
+  do_check_eq(CommonUtils.makeURI("this is a test"), undefined);
+}

services/common/tests/unit/test_utils_namedTimer.js

@@ -0,0 +1,69 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_required_args() {
+  try {
+    CommonUtils.namedTimer(function callback() {
+      do_throw("Shouldn't fire.");
+    }, 0);
+    do_throw("Should have thrown!");
+  } catch(ex) {
+    run_next_test();
+  }
+});
+
+add_test(function test_simple() {
+  _("Test basic properties of CommonUtils.namedTimer.");
+
+  const delay = 200;
+  let that = {};
+  let t0 = Date.now();
+  CommonUtils.namedTimer(function callback(timer) {
+    do_check_eq(this, that);
+    do_check_eq(this._zetimer, null);
+    do_check_true(timer instanceof Ci.nsITimer);
+    // Difference should be ~delay, but hard to predict on all platforms,
+    // particularly Windows XP.
+    do_check_true(Date.now() > t0);
+    run_next_test();
+  }, delay, that, "_zetimer");
+});
+
+add_test(function test_delay() {
+  _("Test delaying a timer that hasn't fired yet.");
+
+  const delay = 100;
+  let that = {};
+  let t0 = Date.now();
+  function callback(timer) {
+    // Difference should be ~2*delay, but hard to predict on all platforms,
+    // particularly Windows XP.
+    do_check_true((Date.now() - t0) > delay);
+    run_next_test();
+  }
+  CommonUtils.namedTimer(callback, delay, that, "_zetimer");
+  CommonUtils.namedTimer(callback, 2 * delay, that, "_zetimer");
+  run_next_test();
+});
+
+add_test(function test_clear() {
+  _("Test clearing a timer that hasn't fired yet.");
+
+  const delay = 0;
+  let that = {};
+  CommonUtils.namedTimer(function callback(timer) {
+    do_throw("Shouldn't fire!");
+  }, delay, that, "_zetimer");
+
+  that._zetimer.clear();
+  do_check_eq(that._zetimer, null);
+  CommonUtils.nextTick(run_next_test);
+
+  run_next_test();
+});

services/common/tests/unit/test_utils_sets.js

@@ -0,0 +1,72 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-common/utils.js");
+
+const EMPTY = new Set();
+const A = new Set(["a"]);
+const ABC = new Set(["a", "b", "c"]);
+const ABCD = new Set(["a", "b", "c", "d"]);
+const BC = new Set(["b", "c"]);
+const BCD = new Set(["b", "c", "d"]);
+const FGH = new Set(["f", "g", "h"]);
+const BCDFGH = new Set(["b", "c", "d", "f", "g", "h"]);
+
+var union = CommonUtils.union;
+var difference = CommonUtils.difference;
+var intersection = CommonUtils.intersection;
+var setEqual = CommonUtils.setEqual;
+
+function do_check_setEqual(a, b) {
+  do_check_true(setEqual(a, b));
+}
+
+function do_check_not_setEqual(a, b) {
+  do_check_false(setEqual(a, b));
+}
+
+function run_test() {
+  run_next_test();
+}
+
+add_test(function test_setEqual() {
+  do_check_setEqual(EMPTY, EMPTY);
+  do_check_setEqual(EMPTY, new Set());
+  do_check_setEqual(A, A);
+  do_check_setEqual(A, new Set(["a"]));
+  do_check_setEqual(new Set(["a"]), A);
+  do_check_not_setEqual(A, EMPTY);
+  do_check_not_setEqual(EMPTY, A);
+  do_check_not_setEqual(ABC, A);
+  run_next_test();
+});
+
+add_test(function test_union() {
+  do_check_setEqual(EMPTY, union(EMPTY, EMPTY));
+  do_check_setEqual(ABC, union(EMPTY, ABC));
+  do_check_setEqual(ABC, union(ABC, ABC));
+  do_check_setEqual(ABCD, union(ABC, BCD));
+  do_check_setEqual(ABCD, union(BCD, ABC));
+  do_check_setEqual(BCDFGH, union(BCD, FGH));
+  run_next_test();
+});
+
+add_test(function test_difference() {
+  do_check_setEqual(EMPTY, difference(EMPTY, EMPTY));
+  do_check_setEqual(EMPTY, difference(EMPTY, A));
+  do_check_setEqual(EMPTY, difference(A, A));
+  do_check_setEqual(ABC, difference(ABC, EMPTY));
+  do_check_setEqual(ABC, difference(ABC, FGH));
+  do_check_setEqual(A, difference(ABC, BCD));
+  run_next_test();
+});
+
+add_test(function test_intersection() {
+  do_check_setEqual(EMPTY, intersection(EMPTY, EMPTY));
+  do_check_setEqual(EMPTY, intersection(ABC, EMPTY));
+  do_check_setEqual(EMPTY, intersection(ABC, FGH));
+  do_check_setEqual(BC, intersection(ABC, BCD));
+  run_next_test();
+});

services/common/tests/unit/test_utils_utf8.js

@@ -0,0 +1,11 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/utils.js");
+
+function run_test() {
+  let str = "Umlaute: \u00FC \u00E4\n"; // Umlaute: ü ä
+  let encoded = CommonUtils.encodeUTF8(str);
+  let decoded = CommonUtils.decodeUTF8(encoded);
+  do_check_eq(decoded, str);
+}

services/common/tests/unit/test_utils_uuid.js

@@ -0,0 +1,12 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+function run_test() {
+  let uuid = CommonUtils.generateUUID();
+  do_check_eq(uuid.length, 36);
+  do_check_eq(uuid[8], "-");
+
+  run_next_test();
+}

services/common/tests/unit/xpcshell.ini

@@ -0,0 +1,53 @@
+[DEFAULT]
+head = head_global.js head_helpers.js head_http.js
+tail =
+firefox-appdir = browser
+support-files =
+  test_storage_adapter/**
+  test_blocklist_signatures/**
+
+# Test load modules first so syntax failures are caught early.
+[test_load_modules.js]
+
+[test_blocklist_certificates.js]
+[test_blocklist_clients.js]
+[test_blocklist_updater.js]
+
+[test_kinto.js]
+[test_blocklist_signatures.js]
+[test_storage_adapter.js]
+
+[test_utils_atob.js]
+[test_utils_convert_string.js]
+[test_utils_dateprefs.js]
+[test_utils_deepCopy.js]
+[test_utils_encodeBase32.js]
+[test_utils_encodeBase64URL.js]
+[test_utils_ensureMillisecondsTimestamp.js]
+[test_utils_json.js]
+[test_utils_makeURI.js]
+[test_utils_namedTimer.js]
+[test_utils_sets.js]
+[test_utils_utf8.js]
+[test_utils_uuid.js]
+
+[test_async_chain.js]
+[test_async_querySpinningly.js]
+
+[test_hawkclient.js]
+skip-if = os == "android"
+[test_hawkrequest.js]
+skip-if = os == "android"
+
+[test_logmanager.js]
+[test_observers.js]
+[test_restrequest.js]
+
+[test_tokenauthenticatedrequest.js]
+skip-if = os == "android"
+
+[test_tokenserverclient.js]
+skip-if = os == "android"
+
+[test_storage_server.js]
+skip-if = os == "android"

services/common/tokenserverclient.js

@@ -0,0 +1,462 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "TokenServerClient",
+  "TokenServerClientError",
+  "TokenServerClientNetworkError",
+  "TokenServerClientServerError",
+];
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/observers.js");
+
+const PREF_LOG_LEVEL = "services.common.log.logger.tokenserverclient";
+
+/**
+ * Represents a TokenServerClient error that occurred on the client.
+ *
+ * This is the base type for all errors raised by client operations.
+ *
+ * @param message
+ *        (string) Error message.
+ */
+this.TokenServerClientError = function TokenServerClientError(message) {
+  this.name = "TokenServerClientError";
+  this.message = message || "Client error.";
+  // Without explicitly setting .stack, all stacks from these errors will point
+  // to the "new Error()" call a few lines down, which isn't helpful.
+  this.stack = Error().stack;
+}
+TokenServerClientError.prototype = new Error();
+TokenServerClientError.prototype.constructor = TokenServerClientError;
+TokenServerClientError.prototype._toStringFields = function() {
+  return {message: this.message};
+}
+TokenServerClientError.prototype.toString = function() {
+  return this.name + "(" + JSON.stringify(this._toStringFields()) + ")";
+}
+TokenServerClientError.prototype.toJSON = function() {
+  let result = this._toStringFields();
+  result["name"] = this.name;
+  return result;
+}
+
+/**
+ * Represents a TokenServerClient error that occurred in the network layer.
+ *
+ * @param error
+ *        The underlying error thrown by the network layer.
+ */
+this.TokenServerClientNetworkError =
+ function TokenServerClientNetworkError(error) {
+  this.name = "TokenServerClientNetworkError";
+  this.error = error;
+  this.stack = Error().stack;
+}
+TokenServerClientNetworkError.prototype = new TokenServerClientError();
+TokenServerClientNetworkError.prototype.constructor =
+  TokenServerClientNetworkError;
+TokenServerClientNetworkError.prototype._toStringFields = function() {
+  return {error: this.error};
+}
+
+/**
+ * Represents a TokenServerClient error that occurred on the server.
+ *
+ * This type will be encountered for all non-200 response codes from the
+ * server. The type of error is strongly enumerated and is stored in the
+ * `cause` property. This property can have the following string values:
+ *
+ *   conditions-required -- The server is requesting that the client
+ *     agree to service conditions before it can obtain a token. The
+ *     conditions that must be presented to the user and agreed to are in
+ *     the `urls` mapping on the instance. Keys of this mapping are
+ *     identifiers. Values are string URLs.
+ *
+ *   invalid-credentials -- A token could not be obtained because
+ *     the credentials presented by the client were invalid.
+ *
+ *   unknown-service -- The requested service was not found.
+ *
+ *   malformed-request -- The server rejected the request because it
+ *     was invalid. If you see this, code in this file is likely wrong.
+ *
+ *   malformed-response -- The response from the server was not what was
+ *     expected.
+ *
+ *   general -- A general server error has occurred. Clients should
+ *     interpret this as an opaque failure.
+ *
+ * @param message
+ *        (string) Error message.
+ */
+this.TokenServerClientServerError =
+ function TokenServerClientServerError(message, cause="general") {
+  this.now = new Date().toISOString(); // may be useful to diagnose time-skew issues.
+  this.name = "TokenServerClientServerError";
+  this.message = message || "Server error.";
+  this.cause = cause;
+  this.stack = Error().stack;
+}
+TokenServerClientServerError.prototype = new TokenServerClientError();
+TokenServerClientServerError.prototype.constructor =
+  TokenServerClientServerError;
+
+TokenServerClientServerError.prototype._toStringFields = function() {
+  let fields = {
+    now: this.now,
+    message: this.message,
+    cause: this.cause,
+  };
+  if (this.response) {
+    fields.response_body = this.response.body;
+    fields.response_headers = this.response.headers;
+    fields.response_status = this.response.status;
+  }
+  return fields;
+};
+
+/**
+ * Represents a client to the Token Server.
+ *
+ * http://docs.services.mozilla.com/token/index.html
+ *
+ * The Token Server supports obtaining tokens for arbitrary apps by
+ * constructing URI paths of the form <app>/<app_version>. However, the service
+ * discovery mechanism emphasizes the use of full URIs and tries to not force
+ * the client to manipulate URIs. This client currently enforces this practice
+ * by not implementing an API which would perform URI manipulation.
+ *
+ * If you are tempted to implement this API in the future, consider this your
+ * warning that you may be doing it wrong and that you should store full URIs
+ * instead.
+ *
+ * Areas to Improve:
+ *
+ *  - The server sends a JSON response on error. The client does not currently
+ *    parse this. It might be convenient if it did.
+ *  - Currently most non-200 status codes are rolled into one error type. It
+ *    might be helpful if callers had a richer API that communicated who was
+ *    at fault (e.g. differentiating a 503 from a 401).
+ */
+this.TokenServerClient = function TokenServerClient() {
+  this._log = Log.repository.getLogger("Common.TokenServerClient");
+  let level = "Debug";
+  try {
+    level = Services.prefs.getCharPref(PREF_LOG_LEVEL);
+  } catch (ex) {}
+  this._log.level = Log.Level[level];
+}
+TokenServerClient.prototype = {
+  /**
+   * Logger instance.
+   */
+  _log: null,
+
+  /**
+   * Obtain a token from a BrowserID assertion against a specific URL.
+   *
+   * This asynchronously obtains the token. The callback receives 2 arguments:
+   *
+   *   (TokenServerClientError | null) If no token could be obtained, this
+   *     will be a TokenServerClientError instance describing why. The
+   *     type seen defines the type of error encountered. If an HTTP response
+   *     was seen, a RESTResponse instance will be stored in the `response`
+   *     property of this object. If there was no error and a token is
+   *     available, this will be null.
+   *
+   *   (map | null) On success, this will be a map containing the results from
+   *     the server. If there was an error, this will be null. The map has the
+   *     following properties:
+   *
+   *       id       (string) HTTP MAC public key identifier.
+   *       key      (string) HTTP MAC shared symmetric key.
+   *       endpoint (string) URL where service can be connected to.
+   *       uid      (string) user ID for requested service.
+   *       duration (string) the validity duration of the issued token.
+   *
+   * Terms of Service Acceptance
+   * ---------------------------
+   *
+   * Some services require users to accept terms of service before they can
+   * obtain a token. If a service requires ToS acceptance, the error passed
+   * to the callback will be a `TokenServerClientServerError` with the
+   * `cause` property set to "conditions-required". The `urls` property of that
+   * instance will be a map of string keys to string URL values. The user-agent
+   * should prompt the user to accept the content at these URLs.
+   *
+   * Clients signify acceptance of the terms of service by sending a token
+   * request with additional metadata. This is controlled by the
+   * `conditionsAccepted` argument to this function. Clients only need to set
+   * this flag once per service and the server remembers acceptance. If
+   * the conditions for the service change, the server may request
+   * clients agree to terms again. Therefore, clients should always be
+   * prepared to handle a conditions required response.
+   *
+   * Clients should not blindly send acceptance to conditions. Instead, clients
+   * should set `conditionsAccepted` if and only if the server asks for
+   * acceptance, the conditions are displayed to the user, and the user agrees
+   * to them.
+   *
+   * Example Usage
+   * -------------
+   *
+   *   let client = new TokenServerClient();
+   *   let assertion = getBrowserIDAssertionFromSomewhere();
+   *   let url = "https://token.services.mozilla.com/1.0/sync/2.0";
+   *
+   *   client.getTokenFromBrowserIDAssertion(url, assertion,
+   *                                         function onResponse(error, result) {
+   *     if (error) {
+   *       if (error.cause == "conditions-required") {
+   *         promptConditionsAcceptance(error.urls, function onAccept() {
+   *           client.getTokenFromBrowserIDAssertion(url, assertion,
+   *           onResponse, true);
+   *         }
+   *         return;
+   *       }
+   *
+   *       // Do other error handling.
+   *       return;
+   *     }
+   *
+   *     let {
+   *       id: id, key: key, uid: uid, endpoint: endpoint, duration: duration
+   *     } = result;
+   *     // Do stuff with data and carry on.
+   *   });
+   *
+   * @param  url
+   *         (string) URL to fetch token from.
+   * @param  assertion
+   *         (string) BrowserID assertion to exchange token for.
+   * @param  cb
+   *         (function) Callback to be invoked with result of operation.
+   * @param  conditionsAccepted
+   *         (bool) Whether to send acceptance to service conditions.
+   */
+  getTokenFromBrowserIDAssertion:
+    function getTokenFromBrowserIDAssertion(url, assertion, cb, addHeaders={}) {
+    if (!url) {
+      throw new TokenServerClientError("url argument is not valid.");
+    }
+
+    if (!assertion) {
+      throw new TokenServerClientError("assertion argument is not valid.");
+    }
+
+    if (!cb) {
+      throw new TokenServerClientError("cb argument is not valid.");
+    }
+
+    this._log.debug("Beginning BID assertion exchange: " + url);
+
+    let req = this.newRESTRequest(url);
+    req.setHeader("Accept", "application/json");
+    req.setHeader("Authorization", "BrowserID " + assertion);
+
+    for (let header in addHeaders) {
+      req.setHeader(header, addHeaders[header]);
+    }
+
+    let client = this;
+    req.get(function onResponse(error) {
+      if (error) {
+        cb(new TokenServerClientNetworkError(error), null);
+        return;
+      }
+
+      let self = this;
+      function callCallback(error, result) {
+        if (!cb) {
+          self._log.warn("Callback already called! Did it throw?");
+          return;
+        }
+
+        try {
+          cb(error, result);
+        } catch (ex) {
+          self._log.warn("Exception when calling user-supplied callback", ex);
+        }
+
+        cb = null;
+      }
+
+      try {
+        client._processTokenResponse(this.response, callCallback);
+      } catch (ex) {
+        this._log.warn("Error processing token server response", ex);
+
+        let error = new TokenServerClientError(ex);
+        error.response = this.response;
+        callCallback(error, null);
+      }
+    });
+  },
+
+  /**
+   * Handler to process token request responses.
+   *
+   * @param response
+   *        RESTResponse from token HTTP request.
+   * @param cb
+   *        The original callback passed to the public API.
+   */
+  _processTokenResponse: function processTokenResponse(response, cb) {
+    this._log.debug("Got token response: " + response.status);
+
+    // Responses should *always* be JSON, even in the case of 4xx and 5xx
+    // errors. If we don't see JSON, the server is likely very unhappy.
+    let ct = response.headers["content-type"] || "";
+    if (ct != "application/json" && !ct.startsWith("application/json;")) {
+      this._log.warn("Did not receive JSON response. Misconfigured server?");
+      this._log.debug("Content-Type: " + ct);
+      this._log.debug("Body: " + response.body);
+
+      let error = new TokenServerClientServerError("Non-JSON response.",
+                                                   "malformed-response");
+      error.response = response;
+      cb(error, null);
+      return;
+    }
+
+    let result;
+    try {
+      result = JSON.parse(response.body);
+    } catch (ex) {
+      this._log.warn("Invalid JSON returned by server: " + response.body);
+      let error = new TokenServerClientServerError("Malformed JSON.",
+                                                   "malformed-response");
+      error.response = response;
+      cb(error, null);
+      return;
+    }
+
+    // Any response status can have X-Backoff or X-Weave-Backoff headers.
+    this._maybeNotifyBackoff(response, "x-weave-backoff");
+    this._maybeNotifyBackoff(response, "x-backoff");
+
+    // The service shouldn't have any 3xx, so we don't need to handle those.
+    if (response.status != 200) {
+      // We /should/ have a Cornice error report in the JSON. We log that to
+      // help with debugging.
+      if ("errors" in result) {
+        // This could throw, but this entire function is wrapped in a try. If
+        // the server is sending something not an array of objects, it has
+        // failed to keep its contract with us and there is little we can do.
+        for (let error of result.errors) {
+          this._log.info("Server-reported error: " + JSON.stringify(error));
+        }
+      }
+
+      let error = new TokenServerClientServerError();
+      error.response = response;
+
+      if (response.status == 400) {
+        error.message = "Malformed request.";
+        error.cause = "malformed-request";
+      } else if (response.status == 401) {
+        // Cause can be invalid-credentials, invalid-timestamp, or
+        // invalid-generation.
+        error.message = "Authentication failed.";
+        error.cause = result.status;
+      }
+
+      // 403 should represent a "condition acceptance needed" response.
+      //
+      // The extra validation of "urls" is important. We don't want to signal
+      // conditions required unless we are absolutely sure that is what the
+      // server is asking for.
+      else if (response.status == 403) {
+        if (!("urls" in result)) {
+          this._log.warn("403 response without proper fields!");
+          this._log.warn("Response body: " + response.body);
+
+          error.message = "Missing JSON fields.";
+          error.cause = "malformed-response";
+        } else if (typeof(result.urls) != "object") {
+          error.message = "urls field is not a map.";
+          error.cause = "malformed-response";
+        } else {
+          error.message = "Conditions must be accepted.";
+          error.cause = "conditions-required";
+          error.urls = result.urls;
+        }
+      } else if (response.status == 404) {
+        error.message = "Unknown service.";
+        error.cause = "unknown-service";
+      }
+
+      // A Retry-After header should theoretically only appear on a 503, but
+      // we'll look for it on any error response.
+      this._maybeNotifyBackoff(response, "retry-after");
+
+      cb(error, null);
+      return;
+    }
+
+    for (let k of ["id", "key", "api_endpoint", "uid", "duration"]) {
+      if (!(k in result)) {
+        let error = new TokenServerClientServerError("Expected key not " +
+                                                     " present in result: " +
+                                                     k);
+        error.cause = "malformed-response";
+        error.response = response;
+        cb(error, null);
+        return;
+      }
+    }
+
+    this._log.debug("Successful token response");
+    cb(null, {
+      id:             result.id,
+      key:            result.key,
+      endpoint:       result.api_endpoint,
+      uid:            result.uid,
+      duration:       result.duration,
+      hashed_fxa_uid: result.hashed_fxa_uid,
+    });
+  },
+
+  /*
+   * The prefix used for all notifications sent by this module.  This
+   * allows the handler of notifications to be sure they are handling
+   * notifications for the service they expect.
+   *
+   * If not set, no notifications will be sent.
+   */
+  observerPrefix: null,
+
+  // Given an optional header value, notify that a backoff has been requested.
+  _maybeNotifyBackoff: function (response, headerName) {
+    if (!this.observerPrefix) {
+      return;
+    }
+    let headerVal = response.headers[headerName];
+    if (!headerVal) {
+      return;
+    }
+    let backoffInterval;
+    try {
+      backoffInterval = parseInt(headerVal, 10);
+    } catch (ex) {
+      this._log.error("TokenServer response had invalid backoff value in '" +
+                      headerName + "' header: " + headerVal);
+      return;
+    }
+    Observers.notify(this.observerPrefix + ":backoff:interval", backoffInterval);
+  },
+
+  // override points for testing.
+  newRESTRequest: function(url) {
+    return new RESTRequest(url);
+  }
+};

services/common/utils.js

@@ -0,0 +1,645 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+this.EXPORTED_SYMBOLS = ["CommonUtils"];
+
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/osfile.jsm")
+Cu.import("resource://gre/modules/Log.jsm");
+
+this.CommonUtils = {
+  /*
+   * Set manipulation methods. These should be lifted into toolkit, or added to
+   * `Set` itself.
+   */
+
+  /**
+   * Return elements of `a` or `b`.
+   */
+  union: function (a, b) {
+    let out = new Set(a);
+    for (let x of b) {
+      out.add(x);
+    }
+    return out;
+  },
+
+  /**
+   * Return elements of `a` that are not present in `b`.
+   */
+  difference: function (a, b) {
+    let out = new Set(a);
+    for (let x of b) {
+      out.delete(x);
+    }
+    return out;
+  },
+
+  /**
+   * Return elements of `a` that are also in `b`.
+   */
+  intersection: function (a, b) {
+    let out = new Set();
+    for (let x of a) {
+      if (b.has(x)) {
+        out.add(x);
+      }
+    }
+    return out;
+  },
+
+  /**
+   * Return true if `a` and `b` are the same size, and
+   * every element of `a` is in `b`.
+   */
+  setEqual: function (a, b) {
+    if (a.size != b.size) {
+      return false;
+    }
+    for (let x of a) {
+      if (!b.has(x)) {
+        return false;
+      }
+    }
+    return true;
+  },
+
+  /**
+   * Encode byte string as base64URL (RFC 4648).
+   *
+   * @param bytes
+   *        (string) Raw byte string to encode.
+   * @param pad
+   *        (bool) Whether to include padding characters (=). Defaults
+   *        to true for historical reasons.
+   */
+  encodeBase64URL: function encodeBase64URL(bytes, pad=true) {
+    let s = btoa(bytes).replace(/\+/g, "-").replace(/\//g, "_");
+
+    if (!pad) {
+      return s.replace(/=+$/, "");
+    }
+
+    return s;
+  },
+
+  /**
+   * Create a nsIURI instance from a string.
+   */
+  makeURI: function makeURI(URIString) {
+    if (!URIString)
+      return null;
+    try {
+      return Services.io.newURI(URIString, null, null);
+    } catch (e) {
+      let log = Log.repository.getLogger("Common.Utils");
+      log.debug("Could not create URI", e);
+      return null;
+    }
+  },
+
+  /**
+   * Execute a function on the next event loop tick.
+   *
+   * @param callback
+   *        Function to invoke.
+   * @param thisObj [optional]
+   *        Object to bind the callback to.
+   */
+  nextTick: function nextTick(callback, thisObj) {
+    if (thisObj) {
+      callback = callback.bind(thisObj);
+    }
+    Services.tm.currentThread.dispatch(callback, Ci.nsIThread.DISPATCH_NORMAL);
+  },
+
+  /**
+   * Return a promise resolving on some later tick.
+   *
+   * This a wrapper around Promise.resolve() that prevents stack
+   * accumulation and prevents callers from accidentally relying on
+   * same-tick promise resolution.
+   */
+  laterTickResolvingPromise: function (value, prototype) {
+    let deferred = Promise.defer(prototype);
+    this.nextTick(deferred.resolve.bind(deferred, value));
+    return deferred.promise;
+  },
+
+  /**
+   * Spin the event loop and return once the next tick is executed.
+   *
+   * This is an evil function and should not be used in production code. It
+   * exists in this module for ease-of-use.
+   */
+  waitForNextTick: function waitForNextTick() {
+    let cb = Async.makeSyncCallback();
+    this.nextTick(cb);
+    Async.waitForSyncCallback(cb);
+
+    return;
+  },
+
+  /**
+   * Return a timer that is scheduled to call the callback after waiting the
+   * provided time or as soon as possible. The timer will be set as a property
+   * of the provided object with the given timer name.
+   */
+  namedTimer: function namedTimer(callback, wait, thisObj, name) {
+    if (!thisObj || !name) {
+      throw "You must provide both an object and a property name for the timer!";
+    }
+
+    // Delay an existing timer if it exists
+    if (name in thisObj && thisObj[name] instanceof Ci.nsITimer) {
+      thisObj[name].delay = wait;
+      return;
+    }
+
+    // Create a special timer that we can add extra properties
+    let timer = Object.create(Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer));
+
+    // Provide an easy way to clear out the timer
+    timer.clear = function() {
+      thisObj[name] = null;
+      timer.cancel();
+    };
+
+    // Initialize the timer with a smart callback
+    timer.initWithCallback({
+      notify: function notify() {
+        // Clear out the timer once it's been triggered
+        timer.clear();
+        callback.call(thisObj, timer);
+      }
+    }, wait, timer.TYPE_ONE_SHOT);
+
+    return thisObj[name] = timer;
+  },
+
+  encodeUTF8: function encodeUTF8(str) {
+    try {
+      str = this._utf8Converter.ConvertFromUnicode(str);
+      return str + this._utf8Converter.Finish();
+    } catch (ex) {
+      return null;
+    }
+  },
+
+  decodeUTF8: function decodeUTF8(str) {
+    try {
+      str = this._utf8Converter.ConvertToUnicode(str);
+      return str + this._utf8Converter.Finish();
+    } catch (ex) {
+      return null;
+    }
+  },
+
+  byteArrayToString: function byteArrayToString(bytes) {
+    return bytes.map(byte => String.fromCharCode(byte)).join("");
+  },
+
+  stringToByteArray: function stringToByteArray(bytesString) {
+    return Array.prototype.slice.call(bytesString).map(c => c.charCodeAt(0));
+  },
+
+  bytesAsHex: function bytesAsHex(bytes) {
+    return Array.prototype.slice.call(bytes).map(c => ("0" + c.charCodeAt(0).toString(16)).slice(-2)).join("");
+  },
+
+  stringAsHex: function stringAsHex(str) {
+    return CommonUtils.bytesAsHex(CommonUtils.encodeUTF8(str));
+  },
+
+  stringToBytes: function stringToBytes(str) {
+    return CommonUtils.hexToBytes(CommonUtils.stringAsHex(str));
+  },
+
+  hexToBytes: function hexToBytes(str) {
+    let bytes = [];
+    for (let i = 0; i < str.length - 1; i += 2) {
+      bytes.push(parseInt(str.substr(i, 2), 16));
+    }
+    return String.fromCharCode.apply(String, bytes);
+  },
+
+  hexAsString: function hexAsString(hex) {
+    return CommonUtils.decodeUTF8(CommonUtils.hexToBytes(hex));
+  },
+
+  /**
+   * Base32 encode (RFC 4648) a string
+   */
+  encodeBase32: function encodeBase32(bytes) {
+    const key = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+    let quanta = Math.floor(bytes.length / 5);
+    let leftover = bytes.length % 5;
+
+    // Pad the last quantum with zeros so the length is a multiple of 5.
+    if (leftover) {
+      quanta += 1;
+      for (let i = leftover; i < 5; i++)
+        bytes += "\0";
+    }
+
+    // Chop the string into quanta of 5 bytes (40 bits). Each quantum
+    // is turned into 8 characters from the 32 character base.
+    let ret = "";
+    for (let i = 0; i < bytes.length; i += 5) {
+      let c = Array.prototype.slice.call(bytes.slice(i, i + 5)).map(byte => byte.charCodeAt(0));
+      ret += key[c[0] >> 3]
+           + key[((c[0] << 2) & 0x1f) | (c[1] >> 6)]
+           + key[(c[1] >> 1) & 0x1f]
+           + key[((c[1] << 4) & 0x1f) | (c[2] >> 4)]
+           + key[((c[2] << 1) & 0x1f) | (c[3] >> 7)]
+           + key[(c[3] >> 2) & 0x1f]
+           + key[((c[3] << 3) & 0x1f) | (c[4] >> 5)]
+           + key[c[4] & 0x1f];
+    }
+
+    switch (leftover) {
+      case 1:
+        return ret.slice(0, -6) + "======";
+      case 2:
+        return ret.slice(0, -4) + "====";
+      case 3:
+        return ret.slice(0, -3) + "===";
+      case 4:
+        return ret.slice(0, -1) + "=";
+      default:
+        return ret;
+    }
+  },
+
+  /**
+   * Base32 decode (RFC 4648) a string.
+   */
+  decodeBase32: function decodeBase32(str) {
+    const key = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
+
+    let padChar = str.indexOf("=");
+    let chars = (padChar == -1) ? str.length : padChar;
+    let bytes = Math.floor(chars * 5 / 8);
+    let blocks = Math.ceil(chars / 8);
+
+    // Process a chunk of 5 bytes / 8 characters.
+    // The processing of this is known in advance,
+    // so avoid arithmetic!
+    function processBlock(ret, cOffset, rOffset) {
+      let c, val;
+
+      // N.B., this relies on
+      //   undefined | foo == foo.
+      function accumulate(val) {
+        ret[rOffset] |= val;
+      }
+
+      function advance() {
+        c  = str[cOffset++];
+        if (!c || c == "" || c == "=") // Easier than range checking.
+          throw "Done";                // Will be caught far away.
+        val = key.indexOf(c);
+        if (val == -1)
+          throw "Unknown character in base32: " + c;
+      }
+
+      // Handle a left shift, restricted to bytes.
+      function left(octet, shift) {
+        return (octet << shift) & 0xff;
+      }
+
+      advance();
+      accumulate(left(val, 3));
+      advance();
+      accumulate(val >> 2);
+      ++rOffset;
+      accumulate(left(val, 6));
+      advance();
+      accumulate(left(val, 1));
+      advance();
+      accumulate(val >> 4);
+      ++rOffset;
+      accumulate(left(val, 4));
+      advance();
+      accumulate(val >> 1);
+      ++rOffset;
+      accumulate(left(val, 7));
+      advance();
+      accumulate(left(val, 2));
+      advance();
+      accumulate(val >> 3);
+      ++rOffset;
+      accumulate(left(val, 5));
+      advance();
+      accumulate(val);
+      ++rOffset;
+    }
+
+    // Our output. Define to be explicit (and maybe the compiler will be smart).
+    let ret  = new Array(bytes);
+    let i    = 0;
+    let cOff = 0;
+    let rOff = 0;
+
+    for (; i < blocks; ++i) {
+      try {
+        processBlock(ret, cOff, rOff);
+      } catch (ex) {
+        // Handle the detection of padding.
+        if (ex == "Done")
+          break;
+        throw ex;
+      }
+      cOff += 8;
+      rOff += 5;
+    }
+
+    // Slice in case our shift overflowed to the right.
+    return CommonUtils.byteArrayToString(ret.slice(0, bytes));
+  },
+
+  /**
+   * Trim excess padding from a Base64 string and atob().
+   *
+   * See bug 562431 comment 4.
+   */
+  safeAtoB: function safeAtoB(b64) {
+    let len = b64.length;
+    let over = len % 4;
+    return over ? atob(b64.substr(0, len - over)) : atob(b64);
+  },
+
+  /**
+   * Parses a JSON file from disk using OS.File and promises.
+   *
+   * @param path the file to read. Will be passed to `OS.File.read()`.
+   * @return a promise that resolves to the JSON contents of the named file.
+   */
+  readJSON: function(path) {
+    return OS.File.read(path, { encoding: "utf-8" }).then((data) => {
+      return JSON.parse(data);
+    });
+  },
+
+  /**
+   * Write a JSON object to the named file using OS.File and promises.
+   *
+   * @param contents a JS object. Will be serialized.
+   * @param path the path of the file to write.
+   * @return a promise, as produced by OS.File.writeAtomic.
+   */
+  writeJSON: function(contents, path) {
+    let data = JSON.stringify(contents);
+    return OS.File.writeAtomic(path, data, {encoding: "utf-8", tmpPath: path + ".tmp"});
+  },
+
+
+  /**
+   * Ensure that the specified value is defined in integer milliseconds since
+   * UNIX epoch.
+   *
+   * This throws an error if the value is not an integer, is negative, or looks
+   * like seconds, not milliseconds.
+   *
+   * If the value is null or 0, no exception is raised.
+   *
+   * @param value
+   *        Value to validate.
+   */
+  ensureMillisecondsTimestamp: function ensureMillisecondsTimestamp(value) {
+    if (!value) {
+      return;
+    }
+
+    if (!/^[0-9]+$/.test(value)) {
+      throw new Error("Timestamp value is not a positive integer: " + value);
+    }
+
+    let intValue = parseInt(value, 10);
+
+    if (!intValue) {
+       return;
+    }
+
+    // Catch what looks like seconds, not milliseconds.
+    if (intValue < 10000000000) {
+      throw new Error("Timestamp appears to be in seconds: " + intValue);
+    }
+  },
+
+  /**
+   * Read bytes from an nsIInputStream into a string.
+   *
+   * @param stream
+   *        (nsIInputStream) Stream to read from.
+   * @param count
+   *        (number) Integer number of bytes to read. If not defined, or
+   *        0, all available input is read.
+   */
+  readBytesFromInputStream: function readBytesFromInputStream(stream, count) {
+    let BinaryInputStream = Components.Constructor(
+        "@mozilla.org/binaryinputstream;1",
+        "nsIBinaryInputStream",
+        "setInputStream");
+    if (!count) {
+      count = stream.available();
+    }
+
+    return new BinaryInputStream(stream).readBytes(count);
+  },
+
+  /**
+   * Generate a new UUID using nsIUUIDGenerator.
+   *
+   * Example value: "1e00a2e2-1570-443e-bf5e-000354124234"
+   *
+   * @return string A hex-formatted UUID string.
+   */
+  generateUUID: function generateUUID() {
+    let uuid = Cc["@mozilla.org/uuid-generator;1"]
+                 .getService(Ci.nsIUUIDGenerator)
+                 .generateUUID()
+                 .toString();
+
+    return uuid.substring(1, uuid.length - 1);
+  },
+
+  /**
+   * Obtain an epoch value from a preference.
+   *
+   * This reads a string preference and returns an integer. The string
+   * preference is expected to contain the integer milliseconds since epoch.
+   * For best results, only read preferences that have been saved with
+   * setDatePref().
+   *
+   * We need to store times as strings because integer preferences are only
+   * 32 bits and likely overflow most dates.
+   *
+   * If the pref contains a non-integer value, the specified default value will
+   * be returned.
+   *
+   * @param branch
+   *        (Preferences) Branch from which to retrieve preference.
+   * @param pref
+   *        (string) The preference to read from.
+   * @param def
+   *        (Number) The default value to use if the preference is not defined.
+   * @param log
+   *        (Log.Logger) Logger to write warnings to.
+   */
+  getEpochPref: function getEpochPref(branch, pref, def=0, log=null) {
+    if (!Number.isInteger(def)) {
+      throw new Error("Default value is not a number: " + def);
+    }
+
+    let valueStr = branch.get(pref, null);
+
+    if (valueStr !== null) {
+      let valueInt = parseInt(valueStr, 10);
+      if (Number.isNaN(valueInt)) {
+        if (log) {
+          log.warn("Preference value is not an integer. Using default. " +
+                   pref + "=" + valueStr + " -> " + def);
+        }
+
+        return def;
+      }
+
+      return valueInt;
+    }
+
+    return def;
+  },
+
+  /**
+   * Obtain a Date from a preference.
+   *
+   * This is a wrapper around getEpochPref. It converts the value to a Date
+   * instance and performs simple range checking.
+   *
+   * The range checking ensures the date is newer than the oldestYear
+   * parameter.
+   *
+   * @param branch
+   *        (Preferences) Branch from which to read preference.
+   * @param pref
+   *        (string) The preference from which to read.
+   * @param def
+   *        (Number) The default value (in milliseconds) if the preference is
+   *        not defined or invalid.
+   * @param log
+   *        (Log.Logger) Logger to write warnings to.
+   * @param oldestYear
+   *        (Number) Oldest year to accept in read values.
+   */
+  getDatePref: function getDatePref(branch, pref, def=0, log=null,
+                                    oldestYear=2010) {
+
+    let valueInt = this.getEpochPref(branch, pref, def, log);
+    let date = new Date(valueInt);
+
+    if (valueInt == def || date.getFullYear() >= oldestYear) {
+      return date;
+    }
+
+    if (log) {
+      log.warn("Unexpected old date seen in pref. Returning default: " +
+               pref + "=" + date + " -> " + def);
+    }
+
+    return new Date(def);
+  },
+
+  /**
+   * Store a Date in a preference.
+   *
+   * This is the opposite of getDatePref(). The same notes apply.
+   *
+   * If the range check fails, an Error will be thrown instead of a default
+   * value silently being used.
+   *
+   * @param branch
+   *        (Preference) Branch from which to read preference.
+   * @param pref
+   *        (string) Name of preference to write to.
+   * @param date
+   *        (Date) The value to save.
+   * @param oldestYear
+   *        (Number) The oldest year to accept for values.
+   */
+  setDatePref: function setDatePref(branch, pref, date, oldestYear=2010) {
+    if (date.getFullYear() < oldestYear) {
+      throw new Error("Trying to set " + pref + " to a very old time: " +
+                      date + ". The current time is " + new Date() +
+                      ". Is the system clock wrong?");
+    }
+
+    branch.set(pref, "" + date.getTime());
+  },
+
+  /**
+   * Convert a string between two encodings.
+   *
+   * Output is only guaranteed if the input stream is composed of octets. If
+   * the input string has characters with values larger than 255, data loss
+   * will occur.
+   *
+   * The returned string is guaranteed to consist of character codes no greater
+   * than 255.
+   *
+   * @param s
+   *        (string) The source string to convert.
+   * @param source
+   *        (string) The current encoding of the string.
+   * @param dest
+   *        (string) The target encoding of the string.
+   *
+   * @return string
+   */
+  convertString: function convertString(s, source, dest) {
+    if (!s) {
+      throw new Error("Input string must be defined.");
+    }
+
+    let is = Cc["@mozilla.org/io/string-input-stream;1"]
+               .createInstance(Ci.nsIStringInputStream);
+    is.setData(s, s.length);
+
+    let listener = Cc["@mozilla.org/network/stream-loader;1"]
+                     .createInstance(Ci.nsIStreamLoader);
+
+    let result;
+
+    listener.init({
+      onStreamComplete: function onStreamComplete(loader, context, status,
+                                                  length, data) {
+        result = String.fromCharCode.apply(this, data);
+      },
+    });
+
+    let converter = this._converterService.asyncConvertData(source, dest,
+                                                            listener, null);
+    converter.onStartRequest(null, null);
+    converter.onDataAvailable(null, null, is, 0, s.length);
+    converter.onStopRequest(null, null, null);
+
+    return result;
+  },
+};
+
+XPCOMUtils.defineLazyGetter(CommonUtils, "_utf8Converter", function() {
+  let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                    .createInstance(Ci.nsIScriptableUnicodeConverter);
+  converter.charset = "UTF-8";
+  return converter;
+});
+
+XPCOMUtils.defineLazyGetter(CommonUtils, "_converterService", function() {
+  return Cc["@mozilla.org/streamConverters;1"]
+           .getService(Ci.nsIStreamConverterService);
+});

services/crypto/component/moz.build

@@ -0,0 +1,19 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/unit/xpcshell.ini']
+
+XPIDL_SOURCES += [
+    'nsISyncJPAKE.idl',
+]
+
+XPIDL_MODULE = 'services-crypto-component'
+
+SOURCES += [
+    'nsSyncJPAKE.cpp',
+]
+
+FINAL_LIBRARY = 'xul'

services/crypto/component/nsISyncJPAKE.idl

@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+[scriptable, uuid(5ab02a98-5122-4b90-93cd-f259c4b42e3a)]
+interface nsISyncJPAKE : nsISupports
+{
+  /**
+   * Perform first round of the JPAKE exchange.
+   *
+   * @param aSignerID
+   *        String identifying the signer.
+   * @param aGX1
+   *        Schnorr signature value g^x1, in hex representation.
+   * @param aGV1
+   *        Schnorr signature value g^v1 (v1 is a random value), in hex
+   *        representation.
+   * @param aR1
+   *        Schnorr signature value r1 = v1 - x1 * h, in hex representation.
+   * @param aGX2
+   *        Schnorr signature value g^x2, in hex representation.
+   * @param aGV2
+   *        Schnorr signature value g^v2 (v2 is a random value), in hex
+   *        representation.
+   * @param aR2
+   *        Schnorr signature value r2 = v2 - x2 * h, in hex representation.
+   */
+  void round1(in ACString aSignerID,
+              out ACString aGX1,
+              out ACString aGV1,
+              out ACString aR1,
+              out ACString aGX2,
+              out ACString aGV2,
+              out ACString aR2);
+
+  /**
+   * Perform second round of the JPAKE exchange.
+   *
+   * @param aPeerID
+   *        String identifying the peer.
+   * @param aPIN
+   *        String containing the weak secret (PIN).
+   * @param aGX3
+   *        Schnorr signature value g^x3, in hex representation.
+   * @param aGV3
+   *        Schnorr signature value g^v3 (v3 is a random value), in hex
+   *        representation.
+   * @param aR3
+   *        Schnorr signature value r3 = v3 - x3 * h, in hex representation.
+   * @param aGX4
+   *        Schnorr signature value g^x4, in hex representation.
+   * @param aGV4
+   *        Schnorr signature value g^v4 (v4 is a random value), in hex
+   *        representation.
+   * @param aR4
+   *        Schnorr signature value r4 = v4 - x4 * h, in hex representation.
+   * @param aA
+   *        Schnorr signature value A, in hex representation.
+   * @param aGVA
+   *        Schnorr signature value g^va (va is a random value), in hex
+   *        representation.
+   * @param aRA
+   *        Schnorr signature value ra = va - xa * h, in hex representation.
+   */
+  void round2(in ACString aPeerID,
+              in ACString aPIN,
+              in ACString aGX3,
+              in ACString aGV3,
+              in ACString aR3,
+              in ACString aGX4,
+              in ACString aGV4,
+              in ACString aR4,
+              out ACString aA,
+              out ACString aGVA,
+              out ACString aRA);
+
+  /**
+   * Perform the final step of the JPAKE exchange. This will compute
+   * the key and expand the key to two keys, an AES256 encryption key
+   * and a 256 bit HMAC key. It returns a key confirmation value
+   * (SHA256d of the key) and the encryption and HMAC keys.
+   *
+   * @param aB
+   *        Schnorr signature value B, in hex representation.
+   * @param aGVB
+   *        Schnorr signature value g^vb (vb is a random value), in hex
+   *        representation.
+   * @param aRB
+   *        Schnorr signature value rb = vb - xb * h, in hex representation.
+   * @param aAES256Key
+   *        The AES 256 encryption key, in base64 representation.
+   * @param aHMAC256Key
+   *        The 256 bit HMAC key, in base64 representation.
+   */
+  void final(in ACString aB,
+             in ACString aGVB,
+             in ACString aRB,
+             in ACString aHkdfInfo,
+             out ACString aAES256Key,
+             out ACString aHMAC256Key);
+};

services/crypto/component/nsSyncJPAKE.cpp

@@ -0,0 +1,484 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsSyncJPAKE.h"
+
+#include "base64.h"
+#include "keyhi.h"
+#include "mozilla/ModuleUtils.h"
+#include "mozilla/Move.h"
+#include "nsDebug.h"
+#include "nsError.h"
+#include "nsString.h"
+#include "nscore.h"
+#include "pk11pub.h"
+#include "pkcs11.h"
+#include "secerr.h"
+#include "secmodt.h"
+#include "secport.h"
+
+using mozilla::fallible;
+
+static bool
+hex_from_2char(const unsigned char *c2, unsigned char *byteval)
+{
+  int i;
+  unsigned char offset;
+  *byteval = 0;
+  for (i=0; i<2; i++) {
+    if (c2[i] >= '0' && c2[i] <= '9') {
+      offset = c2[i] - '0';
+      *byteval |= offset << 4*(1-i);
+    } else if (c2[i] >= 'a' && c2[i] <= 'f') {
+      offset = c2[i] - 'a';
+      *byteval |= (offset + 10) << 4*(1-i);
+    } else if (c2[i] >= 'A' && c2[i] <= 'F') {
+      offset = c2[i] - 'A';
+      *byteval |= (offset + 10) << 4*(1-i);
+    } else {
+      return false;
+    }
+  }
+  return true;
+}
+
+static bool
+fromHex(const char * str, unsigned char * p, size_t sLen)
+{
+  size_t i;
+  if (sLen & 1)
+    return false;
+
+  for (i = 0; i < sLen / 2; ++i) {
+    if (!hex_from_2char((const unsigned char *) str + (2*i),
+                        (unsigned char *) p + i)) {
+      return false;
+    }
+  }
+  return true;
+}
+
+static nsresult
+fromHexString(const nsACString & str, unsigned char * p, size_t pMaxLen)
+{
+  char * strData = (char *) str.Data();
+  unsigned len = str.Length();
+  NS_ENSURE_ARG(len / 2 <= pMaxLen);
+  if (!fromHex(strData, p, len)) {
+    return NS_ERROR_INVALID_ARG;
+  }
+  return NS_OK;
+}
+
+static bool
+toHexString(const unsigned char * str, unsigned len, nsACString & out)
+{
+  static const char digits[] = "0123456789ABCDEF";
+  if (!out.SetCapacity(2 * len, fallible))
+    return false;
+  out.SetLength(0);
+  for (unsigned i = 0; i < len; ++i) {
+    out.Append(digits[str[i] >> 4]);
+    out.Append(digits[str[i] & 0x0f]);
+  }
+  return true;
+}
+
+static nsresult
+mapErrno()
+{
+  int err = PORT_GetError();
+  switch (err) {
+    case SEC_ERROR_NO_MEMORY:   return NS_ERROR_OUT_OF_MEMORY;
+    default:                    return NS_ERROR_UNEXPECTED;
+  }
+}
+
+#define NUM_ELEM(x) (sizeof(x) / sizeof (x)[0])
+
+static const char p[] = 
+  "90066455B5CFC38F9CAA4A48B4281F292C260FEEF01FD61037E56258A7795A1C"
+  "7AD46076982CE6BB956936C6AB4DCFE05E6784586940CA544B9B2140E1EB523F"
+  "009D20A7E7880E4E5BFA690F1B9004A27811CD9904AF70420EEFD6EA11EF7DA1"
+  "29F58835FF56B89FAA637BC9AC2EFAAB903402229F491D8D3485261CD068699B"
+  "6BA58A1DDBBEF6DB51E8FE34E8A78E542D7BA351C21EA8D8F1D29F5D5D159394"
+  "87E27F4416B0CA632C59EFD1B1EB66511A5A0FBF615B766C5862D0BD8A3FE7A0"
+  "E0DA0FB2FE1FCB19E8F9996A8EA0FCCDE538175238FC8B0EE6F29AF7F642773E"
+  "BE8CD5402415A01451A840476B2FCEB0E388D30D4B376C37FE401C2A2C2F941D"
+  "AD179C540C1C8CE030D460C4D983BE9AB0B20F69144C1AE13F9383EA1C08504F"
+  "B0BF321503EFE43488310DD8DC77EC5B8349B8BFE97C2C560EA878DE87C11E3D"
+  "597F1FEA742D73EEC7F37BE43949EF1A0D15C3F3E3FC0A8335617055AC91328E"
+  "C22B50FC15B941D3D1624CD88BC25F3E941FDDC6200689581BFEC416B4B2CB73";
+static const char q[] =
+  "CFA0478A54717B08CE64805B76E5B14249A77A4838469DF7F7DC987EFCCFB11D";
+static const char g[] = 
+  "5E5CBA992E0A680D885EB903AEA78E4A45A469103D448EDE3B7ACCC54D521E37"
+  "F84A4BDD5B06B0970CC2D2BBB715F7B82846F9A0C393914C792E6A923E2117AB"
+  "805276A975AADB5261D91673EA9AAFFEECBFA6183DFCB5D3B7332AA19275AFA1"
+  "F8EC0B60FB6F66CC23AE4870791D5982AAD1AA9485FD8F4A60126FEB2CF05DB8"
+  "A7F0F09B3397F3937F2E90B9E5B9C9B6EFEF642BC48351C46FB171B9BFA9EF17"
+  "A961CE96C7E7A7CC3D3D03DFAD1078BA21DA425198F07D2481622BCE45969D9C"
+  "4D6063D72AB7A0F08B2F49A7CC6AF335E08C4720E31476B67299E231F8BD90B3"
+  "9AC3AE3BE0C6B6CACEF8289A2E2873D58E51E029CAFBD55E6841489AB66B5B4B"
+  "9BA6E2F784660896AFF387D92844CCB8B69475496DE19DA2E58259B090489AC8"
+  "E62363CDF82CFD8EF2A427ABCD65750B506F56DDE3B988567A88126B914D7828"
+  "E2B63A6D7ED0747EC59E0E0A23CE7D8A74C1D2C2A7AFB6A29799620F00E11C33"
+  "787F7DED3B30E1A22D09F1FBDA1ABBBFBF25CAE05A13F812E34563F99410E73B";
+
+NS_IMETHODIMP nsSyncJPAKE::Round1(const nsACString & aSignerID,
+                                  nsACString & aGX1,
+                                  nsACString & aGV1,
+                                  nsACString & aR1,
+                                  nsACString & aGX2,
+                                  nsACString & aGV2,
+                                  nsACString & aR2)
+{
+  nsNSSShutDownPreventionLock locker;
+  if (isAlreadyShutDown()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  NS_ENSURE_STATE(round == JPAKENotStarted);
+  NS_ENSURE_STATE(key == nullptr);
+
+  static CK_MECHANISM_TYPE mechanisms[] = {
+    CKM_NSS_JPAKE_ROUND1_SHA256,
+    CKM_NSS_JPAKE_ROUND2_SHA256,
+    CKM_NSS_JPAKE_FINAL_SHA256
+  };
+
+  UniquePK11SlotInfo slot(PK11_GetBestSlotMultiple(mechanisms,
+                                                   NUM_ELEM(mechanisms),
+                                                   nullptr));
+  NS_ENSURE_STATE(slot != nullptr);
+    
+  CK_BYTE pBuf[(NUM_ELEM(p) - 1) / 2];
+  CK_BYTE qBuf[(NUM_ELEM(q) - 1) / 2];
+  CK_BYTE gBuf[(NUM_ELEM(g) - 1) / 2];
+    
+  CK_KEY_TYPE keyType = CKK_NSS_JPAKE_ROUND1;
+  NS_ENSURE_STATE(fromHex(p, pBuf, (NUM_ELEM(p) - 1)));
+  NS_ENSURE_STATE(fromHex(q, qBuf, (NUM_ELEM(q) - 1)));
+  NS_ENSURE_STATE(fromHex(g, gBuf, (NUM_ELEM(g) - 1)));
+  CK_ATTRIBUTE keyTemplate[] = {
+    { CKA_NSS_JPAKE_SIGNERID, (CK_BYTE *) aSignerID.Data(),
+                              aSignerID.Length() },
+    { CKA_KEY_TYPE, &keyType, sizeof keyType },
+    { CKA_PRIME,    pBuf,     sizeof pBuf },
+    { CKA_SUBPRIME, qBuf,     sizeof qBuf },
+    { CKA_BASE,     gBuf,     sizeof gBuf }
+  };
+
+  CK_BYTE gx1Buf[NUM_ELEM(p) / 2];
+  CK_BYTE gv1Buf[NUM_ELEM(p) / 2];
+  CK_BYTE r1Buf [NUM_ELEM(p) / 2];
+  CK_BYTE gx2Buf[NUM_ELEM(p) / 2];
+  CK_BYTE gv2Buf[NUM_ELEM(p) / 2];
+  CK_BYTE r2Buf [NUM_ELEM(p) / 2];
+  CK_NSS_JPAKERound1Params rp = {
+      { gx1Buf, sizeof gx1Buf, gv1Buf, sizeof gv1Buf, r1Buf, sizeof r1Buf },
+      { gx2Buf, sizeof gx2Buf, gv2Buf, sizeof gv2Buf, r2Buf, sizeof r2Buf }
+  };
+  SECItem paramsItem;
+  paramsItem.data = (unsigned char *) &rp;
+  paramsItem.len = sizeof rp;
+  key = UniquePK11SymKey(
+    PK11_KeyGenWithTemplate(slot.get(), CKM_NSS_JPAKE_ROUND1_SHA256,
+                            CKM_NSS_JPAKE_ROUND1_SHA256, &paramsItem,
+                            keyTemplate, NUM_ELEM(keyTemplate), nullptr));
+  nsresult rv = key != nullptr
+              ? NS_OK
+              : mapErrno();
+  if (rv == NS_OK) {
+    NS_ENSURE_TRUE(toHexString(rp.gx1.pGX, rp.gx1.ulGXLen, aGX1) &&
+                   toHexString(rp.gx1.pGV, rp.gx1.ulGVLen, aGV1) &&
+                   toHexString(rp.gx1.pR,  rp.gx1.ulRLen,  aR1)  &&
+                   toHexString(rp.gx2.pGX, rp.gx2.ulGXLen, aGX2) &&
+                   toHexString(rp.gx2.pGV, rp.gx2.ulGVLen, aGV2) &&
+                   toHexString(rp.gx2.pR,  rp.gx2.ulRLen,  aR2),
+                   NS_ERROR_OUT_OF_MEMORY);
+    round = JPAKEBeforeRound2;
+  }
+  return rv;
+}
+
+NS_IMETHODIMP nsSyncJPAKE::Round2(const nsACString & aPeerID,
+                                  const nsACString & aPIN,
+                                  const nsACString & aGX3,
+                                  const nsACString & aGV3,
+                                  const nsACString & aR3,
+                                  const nsACString & aGX4,
+                                  const nsACString & aGV4,
+                                  const nsACString & aR4,
+                                  nsACString & aA,
+                                  nsACString & aGVA,
+                                  nsACString & aRA)
+{
+  nsNSSShutDownPreventionLock locker;
+  if (isAlreadyShutDown()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  NS_ENSURE_STATE(round == JPAKEBeforeRound2);
+  NS_ENSURE_STATE(key != nullptr);
+  NS_ENSURE_ARG(!aPeerID.IsEmpty());
+
+  /* PIN cannot be equal to zero when converted to a bignum. NSS 3.12.9 J-PAKE
+     assumes that the caller has already done this check. Future versions of 
+     NSS J-PAKE will do this check internally. See Bug 609068 Comment 4 */
+  bool foundNonZero = false;
+  for (size_t i = 0; i < aPIN.Length(); ++i) {
+    if (aPIN[i] != 0) {
+      foundNonZero = true;
+      break;
+    }
+  }
+  NS_ENSURE_ARG(foundNonZero);
+
+  CK_BYTE gx3Buf[NUM_ELEM(p)/2], gv3Buf[NUM_ELEM(p)/2], r3Buf [NUM_ELEM(p)/2];
+  CK_BYTE gx4Buf[NUM_ELEM(p)/2], gv4Buf[NUM_ELEM(p)/2], r4Buf [NUM_ELEM(p)/2];
+  CK_BYTE gxABuf[NUM_ELEM(p)/2], gvABuf[NUM_ELEM(p)/2], rABuf [NUM_ELEM(p)/2];
+  nsresult         rv = fromHexString(aGX3, gx3Buf, sizeof gx3Buf);
+  if (rv == NS_OK) rv = fromHexString(aGV3, gv3Buf, sizeof gv3Buf);
+  if (rv == NS_OK) rv = fromHexString(aR3,  r3Buf,  sizeof r3Buf);
+  if (rv == NS_OK) rv = fromHexString(aGX4, gx4Buf, sizeof gx4Buf);
+  if (rv == NS_OK) rv = fromHexString(aGV4, gv4Buf, sizeof gv4Buf);
+  if (rv == NS_OK) rv = fromHexString(aR4,  r4Buf,  sizeof r4Buf);
+  if (rv != NS_OK)
+    return rv;
+
+  CK_NSS_JPAKERound2Params rp;
+  rp.pSharedKey = (CK_BYTE *) aPIN.Data();
+  rp.ulSharedKeyLen = aPIN.Length();
+  rp.gx3.pGX = gx3Buf; rp.gx3.ulGXLen = aGX3.Length() / 2;
+  rp.gx3.pGV = gv3Buf; rp.gx3.ulGVLen = aGV3.Length() / 2;
+  rp.gx3.pR  = r3Buf;  rp.gx3.ulRLen  = aR3 .Length() / 2;
+  rp.gx4.pGX = gx4Buf; rp.gx4.ulGXLen = aGX4.Length() / 2;
+  rp.gx4.pGV = gv4Buf; rp.gx4.ulGVLen = aGV4.Length() / 2;
+  rp.gx4.pR  = r4Buf;  rp.gx4.ulRLen  = aR4 .Length() / 2;
+  rp.A.pGX   = gxABuf; rp.A  .ulGXLen = sizeof gxABuf;
+  rp.A.pGV   = gvABuf; rp.A  .ulGVLen = sizeof gxABuf;
+  rp.A.pR    = rABuf;  rp.A  .ulRLen  = sizeof gxABuf;
+
+  // Bug 629090: NSS 3.12.9 J-PAKE fails to check that gx^4 != 1, so check here.
+  bool gx4Good = false;
+  for (unsigned i = 0; i < rp.gx4.ulGXLen; ++i) {
+    if (rp.gx4.pGX[i] > 1 || (rp.gx4.pGX[i] != 0 && i < rp.gx4.ulGXLen - 1)) {
+      gx4Good = true;
+      break;
+    }
+  }
+  NS_ENSURE_ARG(gx4Good);
+
+  SECItem paramsItem;
+  paramsItem.data = (unsigned char *) &rp;
+  paramsItem.len = sizeof rp;
+  CK_KEY_TYPE keyType = CKK_NSS_JPAKE_ROUND2;
+  CK_ATTRIBUTE keyTemplate[] = {
+    { CKA_NSS_JPAKE_PEERID, (CK_BYTE *) aPeerID.Data(), aPeerID.Length(), },
+    { CKA_KEY_TYPE, &keyType, sizeof keyType }
+  };
+  UniquePK11SymKey newKey(PK11_DeriveWithTemplate(key.get(),
+                                                  CKM_NSS_JPAKE_ROUND2_SHA256,
+                                                  &paramsItem,
+                                                  CKM_NSS_JPAKE_FINAL_SHA256,
+                                                  CKA_DERIVE, 0,
+                                                  keyTemplate,
+                                                  NUM_ELEM(keyTemplate),
+                                                  false));
+  if (newKey != nullptr) {
+    if (toHexString(rp.A.pGX, rp.A.ulGXLen, aA) &&
+        toHexString(rp.A.pGV, rp.A.ulGVLen, aGVA) &&
+        toHexString(rp.A.pR, rp.A.ulRLen, aRA)) {
+      round = JPAKEAfterRound2;
+      key = Move(newKey);
+      return NS_OK;
+    } else {
+      rv = NS_ERROR_OUT_OF_MEMORY;
+    }
+  } else {
+    rv = mapErrno();
+  }
+
+  return rv;
+}
+
+static nsresult
+setBase64(const unsigned char * data, unsigned len, nsACString & out)
+{
+  nsresult rv = NS_OK;
+  const char * base64 = BTOA_DataToAscii(data, len);
+  
+  if (base64 != nullptr) {
+    size_t len = PORT_Strlen(base64);
+    if (out.SetCapacity(len, fallible)) {
+      out.SetLength(0);
+      out.Append(base64, len);
+    } else {
+      rv = NS_ERROR_OUT_OF_MEMORY;
+    }
+    PORT_Free((void*) base64);
+  } else {
+    rv = NS_ERROR_OUT_OF_MEMORY;
+  }
+  return rv;
+}
+
+static nsresult
+base64KeyValue(PK11SymKey * key, nsACString & keyString)
+{
+  nsresult rv = NS_OK;
+  if (PK11_ExtractKeyValue(key) == SECSuccess) {
+    const SECItem * value = PK11_GetKeyData(key);
+    rv = value != nullptr && value->data != nullptr && value->len > 0
+       ? setBase64(value->data, value->len, keyString)
+       : NS_ERROR_UNEXPECTED;
+  } else {
+    rv = mapErrno();
+  }
+  return rv;
+}
+
+static nsresult
+extractBase64KeyValue(UniquePK11SymKey & keyBlock, CK_ULONG bitPosition,
+                      CK_MECHANISM_TYPE destMech, int keySize,
+                      nsACString & keyString)
+{
+  SECItem paramsItem;
+  paramsItem.data = (CK_BYTE *) &bitPosition;
+  paramsItem.len = sizeof bitPosition;
+  PK11SymKey * key = PK11_Derive(keyBlock.get(), CKM_EXTRACT_KEY_FROM_KEY,
+                                 &paramsItem, destMech,
+                                 CKA_SIGN, keySize);
+  if (key == nullptr)
+    return mapErrno();
+  nsresult rv = base64KeyValue(key, keyString);
+  PK11_FreeSymKey(key);
+  return rv;
+}
+
+
+NS_IMETHODIMP nsSyncJPAKE::Final(const nsACString & aB,
+                                 const nsACString & aGVB,
+                                 const nsACString & aRB,
+                                 const nsACString & aHKDFInfo,
+                                 nsACString & aAES256Key,
+                                 nsACString & aHMAC256Key)
+{
+  nsNSSShutDownPreventionLock locker;
+  if (isAlreadyShutDown()) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  static const unsigned AES256_KEY_SIZE = 256 / 8;
+  static const unsigned HMAC_SHA256_KEY_SIZE = 256 / 8;
+  CK_EXTRACT_PARAMS aesBitPosition = 0;
+  CK_EXTRACT_PARAMS hmacBitPosition = aesBitPosition + (AES256_KEY_SIZE * 8);
+
+  NS_ENSURE_STATE(round == JPAKEAfterRound2);
+  NS_ENSURE_STATE(key != nullptr);
+
+  CK_BYTE gxBBuf[NUM_ELEM(p)/2], gvBBuf[NUM_ELEM(p)/2], rBBuf [NUM_ELEM(p)/2];
+  nsresult         rv = fromHexString(aB,   gxBBuf, sizeof gxBBuf);
+  if (rv == NS_OK) rv = fromHexString(aGVB, gvBBuf, sizeof gvBBuf);
+  if (rv == NS_OK) rv = fromHexString(aRB,  rBBuf,  sizeof rBBuf);
+  if (rv != NS_OK)
+    return rv;
+
+  CK_NSS_JPAKEFinalParams rp;
+  rp.B.pGX = gxBBuf; rp.B.ulGXLen = aB  .Length() / 2;
+  rp.B.pGV = gvBBuf; rp.B.ulGVLen = aGVB.Length() / 2;
+  rp.B.pR  = rBBuf;  rp.B.ulRLen  = aRB .Length() / 2;
+  SECItem paramsItem;
+  paramsItem.data = (unsigned char *) &rp;
+  paramsItem.len = sizeof rp;
+  UniquePK11SymKey keyMaterial(PK11_Derive(key.get(), CKM_NSS_JPAKE_FINAL_SHA256,
+                                           &paramsItem, CKM_NSS_HKDF_SHA256,
+                                           CKA_DERIVE, 0));
+  UniquePK11SymKey keyBlock;
+
+  if (keyMaterial == nullptr)
+    rv = mapErrno();
+
+  if (rv == NS_OK) {
+    CK_NSS_HKDFParams hkdfParams;
+    hkdfParams.bExtract = CK_TRUE;
+    hkdfParams.pSalt = nullptr;
+    hkdfParams.ulSaltLen = 0;
+    hkdfParams.bExpand = CK_TRUE;
+    hkdfParams.pInfo = (CK_BYTE *) aHKDFInfo.Data();
+    hkdfParams.ulInfoLen = aHKDFInfo.Length();
+    paramsItem.data = (unsigned char *) &hkdfParams;
+    paramsItem.len = sizeof hkdfParams;
+    keyBlock = UniquePK11SymKey(
+      PK11_Derive(keyMaterial.get(), CKM_NSS_HKDF_SHA256, &paramsItem,
+                  CKM_EXTRACT_KEY_FROM_KEY, CKA_DERIVE,
+                  AES256_KEY_SIZE + HMAC_SHA256_KEY_SIZE));
+    if (keyBlock == nullptr)
+      rv = mapErrno();
+  }
+
+  if (rv == NS_OK) {
+    rv = extractBase64KeyValue(keyBlock, aesBitPosition, CKM_AES_CBC,
+                               AES256_KEY_SIZE, aAES256Key);
+  }
+  if (rv == NS_OK) {
+    rv = extractBase64KeyValue(keyBlock, hmacBitPosition, CKM_SHA256_HMAC,
+                               HMAC_SHA256_KEY_SIZE, aHMAC256Key);
+  }
+
+  if (rv == NS_OK) {
+    SECStatus srv = PK11_ExtractKeyValue(keyMaterial.get());
+    NS_ENSURE_TRUE(srv == SECSuccess, NS_ERROR_UNEXPECTED);
+    SECItem * keyMaterialBytes = PK11_GetKeyData(keyMaterial.get());
+    NS_ENSURE_TRUE(keyMaterialBytes != nullptr, NS_ERROR_UNEXPECTED);
+  }
+
+  return rv;
+}
+
+NS_GENERIC_FACTORY_CONSTRUCTOR(nsSyncJPAKE)
+NS_DEFINE_NAMED_CID(NS_SYNCJPAKE_CID);
+
+nsSyncJPAKE::nsSyncJPAKE() : round(JPAKENotStarted), key(nullptr) { }
+
+nsSyncJPAKE::~nsSyncJPAKE()
+{
+  nsNSSShutDownPreventionLock locker;
+  if (isAlreadyShutDown()) {
+    return;
+  }
+  destructorSafeDestroyNSSReference();
+  shutdown(ShutdownCalledFrom::Object);
+}
+
+void
+nsSyncJPAKE::virtualDestroyNSSReference()
+{
+  destructorSafeDestroyNSSReference();
+}
+
+void
+nsSyncJPAKE::destructorSafeDestroyNSSReference()
+{
+  key = nullptr;
+}
+
+static const mozilla::Module::CIDEntry kServicesCryptoCIDs[] = {
+  { &kNS_SYNCJPAKE_CID, false, nullptr, nsSyncJPAKEConstructor },
+  { nullptr }
+};
+
+static const mozilla::Module::ContractIDEntry kServicesCryptoContracts[] = {
+  { NS_SYNCJPAKE_CONTRACTID, &kNS_SYNCJPAKE_CID },
+  { nullptr }
+};
+
+static const mozilla::Module kServicesCryptoModule = {
+  mozilla::Module::kVersion,
+  kServicesCryptoCIDs,
+  kServicesCryptoContracts
+};
+
+NSMODULE_DEFN(nsServicesCryptoModule) = &kServicesCryptoModule;

services/crypto/component/nsSyncJPAKE.h

@@ -0,0 +1,38 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+#ifndef nsSyncJPAKE_h__
+#define nsSyncJPAKE_h__
+
+#include "ScopedNSSTypes.h"
+#include "nsISyncJPAKE.h"
+#include "nsNSSShutDown.h"
+
+#define NS_SYNCJPAKE_CONTRACTID \
+  "@mozilla.org/services-crypto/sync-jpake;1"
+
+#define NS_SYNCJPAKE_CID \
+  {0x0b9721c0, 0x1805, 0x47c3, {0x86, 0xce, 0x68, 0x13, 0x79, 0x5a, 0x78, 0x3f}}
+
+using namespace mozilla;
+
+class nsSyncJPAKE : public nsISyncJPAKE
+                  , public nsNSSShutDownObject
+{
+public:
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSISYNCJPAKE
+  nsSyncJPAKE();
+protected:
+  virtual ~nsSyncJPAKE();
+private:
+  virtual void virtualDestroyNSSReference() override;
+  void destructorSafeDestroyNSSReference();
+
+  enum { JPAKENotStarted, JPAKEBeforeRound2, JPAKEAfterRound2 } round;
+  UniquePK11SymKey key;
+};
+
+NS_IMPL_ISUPPORTS(nsSyncJPAKE, nsISyncJPAKE)
+
+#endif // nsSyncJPAKE_h__

services/crypto/component/tests/unit/test_jpake.js

@@ -0,0 +1,289 @@
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+
+// Ensure PSM is initialized.
+Cc["@mozilla.org/psm;1"].getService(Ci.nsISupports);
+
+function do_check_throws(func) {
+  let have_error = false;
+  try {
+    func();
+  } catch(ex) {
+    dump("Was expecting an exception. Caught: " + ex + "\n");
+    have_error = true;
+  }
+  do_check_true(have_error);
+}
+
+function test_success() {
+  let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+  let b = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+
+  let a_gx1 = {};
+  let a_gv1 = {};
+  let a_r1 = {};
+  let a_gx2 = {};
+  let a_gv2 = {};
+  let a_r2 = {};
+
+  let b_gx1 = {};
+  let b_gv1 = {};
+  let b_r1 = {};
+  let b_gx2 = {};
+  let b_gv2 = {};
+  let b_r2 = {};
+
+  a.round1("alice", a_gx1, a_gv1, a_r1, a_gx2, a_gv2, a_r2);
+  b.round1("bob", b_gx1, b_gv1, b_r1, b_gx2, b_gv2, b_r2);
+
+  let a_A = {};
+  let a_gva = {};
+  let a_ra = {};
+
+  let b_A = {};
+  let b_gva = {};
+  let b_ra = {};
+
+  a.round2("bob", "sekrit", b_gx1.value, b_gv1.value, b_r1.value,
+           b_gx2.value, b_gv2.value, b_r2.value, a_A, a_gva, a_ra);
+  b.round2("alice", "sekrit", a_gx1.value, a_gv1.value, a_r1.value,
+           a_gx2.value, a_gv2.value, a_r2.value, b_A, b_gva, b_ra);
+
+  let a_aes = {};
+  let a_hmac = {};
+  let b_aes = {};
+  let b_hmac = {};
+
+  a.final(b_A.value, b_gva.value, b_ra.value, "ohai", a_aes, a_hmac);
+  b.final(a_A.value, a_gva.value, a_ra.value, "ohai", b_aes, b_hmac);
+
+  do_check_eq(a_aes.value, b_aes.value);
+  do_check_eq(a_hmac.value, b_hmac.value);
+}
+
+function test_failure(modlen) {
+  let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+  let b = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+
+  let a_gx1 = {};
+  let a_gv1 = {};
+  let a_r1 = {};
+  let a_gx2 = {};
+  let a_gv2 = {};
+  let a_r2 = {};
+
+  let b_gx1 = {};
+  let b_gv1 = {};
+  let b_r1 = {};
+  let b_gx2 = {};
+  let b_gv2 = {};
+  let b_r2 = {};
+
+  a.round1("alice", a_gx1, a_gv1, a_r1, a_gx2, a_gv2, a_r2);
+  b.round1("bob", b_gx1, b_gv1, b_r1, b_gx2, b_gv2, b_r2);
+
+  let a_A = {};
+  let a_gva = {};
+  let a_ra = {};
+
+  let b_A = {};
+  let b_gva = {};
+  let b_ra = {};
+
+  // Note how the PINs are different (secret vs. sekrit)
+  a.round2("bob", "secret", b_gx1.value, b_gv1.value, b_r1.value,
+           b_gx2.value, b_gv2.value, b_r2.value, a_A, a_gva, a_ra);
+  b.round2("alice", "sekrit", a_gx1.value, a_gv1.value, a_r1.value,
+           a_gx2.value, a_gv2.value, a_r2.value, b_A, b_gva, b_ra);
+
+  let a_aes = {};
+  let a_hmac = {};
+  let b_aes = {};
+  let b_hmac = {};
+
+  a.final(b_A.value, b_gva.value, b_ra.value, "ohai", a_aes, a_hmac);
+  b.final(a_A.value, a_gva.value, a_ra.value, "ohai", b_aes, b_hmac);
+
+  do_check_neq(a_aes.value, b_aes.value);
+  do_check_neq(a_hmac.value, b_hmac.value);
+}
+
+function test_same_signerids() {
+  let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+  let b = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+
+  let gx1 = {};
+  let gv1 = {};
+  let r1 = {};
+  let gx2 = {};
+  let gv2 = {};
+  let r2 = {};
+
+  a.round1("alice", {}, {}, {}, {}, {}, {});
+  b.round1("alice", gx1, gv1, r1, gx2, gv2, r2);
+  do_check_throws(function() {
+    a.round2("alice", "sekrit", gx1.value, gv1.value, r1.value,
+             gx2.value, gv2.value, r2.value, {}, {}, {});
+  });
+}
+
+function test_bad_zkp() {
+  let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+  let b = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+
+  let gx1 = {};
+  let gv1 = {};
+  let r1 = {};
+  let gx2 = {};
+  let gv2 = {};
+  let r2 = {};
+
+  a.round1("alice", {}, {}, {}, {}, {}, {});
+  b.round1("bob", gx1, gv1, r1, gx2, gv2, r2);
+  do_check_throws(function() {
+    a.round2("invalid", "sekrit", gx1.value, gv1.value, r1.value,
+             gx2.value, gv2.value, r2.value, {}, {}, {});
+  });
+}
+
+function test_x4_zero() {
+  // The PKCS#11 API for J-PAKE does not allow us to choose any of the nonces.
+  // In order to test the defence against x4 (mod p) == 1, we had to generate
+  // our own signed nonces using a the FreeBL JPAKE_Sign function directly.
+  // To verify the signatures are accurate, pass the given value of R as the
+  // "testRandom" parameter to FreeBL's JPAKE_Sign, along with the given values
+  // for X and GX, using signerID "alice". Then verify that each GV returned
+  // from JPAKE_Sign matches the value specified here.
+  let test = function(badGX, badX_GV, badX_R) {
+    let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+    let b = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+
+    let a_gx1 = {};
+    let a_gv1 = {};
+    let a_r1 = {};
+    let a_gx2 = {};
+    let a_gv2 = {};
+    let a_r2 = {};
+
+    let b_gx1 = {};
+    let b_gv1 = {};
+    let b_r1 = {};
+    let b_gx2 = {};
+    let b_gv2 = {};
+    let b_r2 = {};
+
+    a.round1("alice", a_gx1, a_gv1, a_r1, a_gx2, a_gv2, a_r2);
+    b.round1("bob", b_gx1, b_gv1, b_r1, b_gx2, b_gv2, b_r2);
+
+    // Replace the g^x2 generated by A with the given illegal value.
+    a_gx2.value = badGX;
+    a_gv2.value = badX_GV;
+    a_r2.value = badX_R;
+
+    let b_A = {};
+    let b_gva = {};
+    let b_ra = {};
+
+    do_check_throws(function() {
+        b.round2("alice", "secret", a_gx1.value, a_gv1.value, a_r1.value,
+                    a_gx2.value, a_gv2.value, a_r2.value, b_A, b_gva, b_ra);
+    });
+  };
+  
+  // g^x is NIST 3072's p + 1, (p + 1) mod p == 1, x == 0
+  test("90066455B5CFC38F9CAA4A48B4281F292C260FEEF01FD61037E56258A7795A1C"
+         + "7AD46076982CE6BB956936C6AB4DCFE05E6784586940CA544B9B2140E1EB523F"
+         + "009D20A7E7880E4E5BFA690F1B9004A27811CD9904AF70420EEFD6EA11EF7DA1"
+         + "29F58835FF56B89FAA637BC9AC2EFAAB903402229F491D8D3485261CD068699B"
+         + "6BA58A1DDBBEF6DB51E8FE34E8A78E542D7BA351C21EA8D8F1D29F5D5D159394"
+         + "87E27F4416B0CA632C59EFD1B1EB66511A5A0FBF615B766C5862D0BD8A3FE7A0"
+         + "E0DA0FB2FE1FCB19E8F9996A8EA0FCCDE538175238FC8B0EE6F29AF7F642773E"
+         + "BE8CD5402415A01451A840476B2FCEB0E388D30D4B376C37FE401C2A2C2F941D"
+         + "AD179C540C1C8CE030D460C4D983BE9AB0B20F69144C1AE13F9383EA1C08504F"
+         + "B0BF321503EFE43488310DD8DC77EC5B8349B8BFE97C2C560EA878DE87C11E3D"
+         + "597F1FEA742D73EEC7F37BE43949EF1A0D15C3F3E3FC0A8335617055AC91328E"
+         + "C22B50FC15B941D3D1624CD88BC25F3E941FDDC6200689581BFEC416B4B2CB74",
+       "5386107A0DD4A96ECF8D9BCF864BDE23AAEF13351F5550D777A32C1FEC165ED67AE51"
+         + "66C3876AABC1FED1A0993754F3AEE256530F529548F8FE010BC0D070175569845"
+         + "CF009AD24BC897A9CA1F18E1A9CE421DD54FD93AB528BC2594B47791713165276"
+         + "7B76903190C3DCD2076FEC1E61FFFC32D1B07273B06EA2889E66FCBFD41FE8984"
+         + "5FCE36056B09D1F20E58BB6BAA07A32796F11998BEF0AB3D387E2FB4FE3073FEB"
+         + "634BA91709010A70DA29C06F8F92D638C4F158680EAFEB5E0E323BD7DACB671C0"
+         + "BA3EDEEAB5CAA243CABAB28E7205AC9A0AAEAFE132635DAC7FE001C19F880A96E"
+         + "395C42536D694F81B4F44DC66D7D6FBE933C56ABF585837291D8751C18EB1F3FB"
+         + "620582E6A7B795D699E38C270863A289583CB9D07651E6BA3B82BC656B49BD09B"
+         + "6B8C27F370120C7CB89D0829BE51D56356EA836012E9204FF4D1CA8B1B7F9C768"
+         + "4BB2B0F226FD4042EEBAD931FDBD4F81F8425B305752F5E37FFA2B73BB5A034EC"
+         + "7EEF5AAC92EA212897E3A2B8961D2147710ECCE127B942AB2",
+       "05CC4DF005FE006C11111624E14806E4A904A4D1D6A53E795AC7867A960CD4FD");
+
+  // x == 0 implies g^x == 1
+  test("01",
+       "488759644532FA7C53E5239F2A365D4B9189582BDD2967A1852FE56568382B65"
+         + "C66BDFCD9B581EAEF4BB497CAF1290ECDFA47A1D1658DC5DC9248D9A4135"
+         + "DC70B6A8497CDF117236841FA18500DC696A92EEF5000ABE68E9C75B37BC"
+         + "6A722126BE728163AA90A6B03D5585994D3403557EEF08E819C72D143BBC"
+         + "CDF74559645066CB3607E1B0430365356389FC8FB3D66FD2B6E2E834EC23"
+         + "0B0234956752D07F983C918488C8E5A124B062D50B44C5E6FB36BCB03E39"
+         + "0385B17CF8062B6688371E6AF5915C2B1AAA31C9294943CC6DC1B994FC09"
+         + "49CA31828B83F3D6DFB081B26045DFD9F10092588B63F1D6E68881A06522"
+         + "5A417CA9555B036DE89D349AC794A43EB28FE320F9A321F06A9364C88B54"
+         + "99EEF4816375B119824ACC9AA56D1340B6A49D05F855DE699B351012028C"
+         + "CA43001F708CC61E71CA3849935BEEBABC0D268CD41B8D2B8DCA705FDFF8"
+         + "1DAA772DA96EDEA0B291FD5C0C1B8EFE5318D37EBC1BFF53A9DDEC4171A6"
+         + "479E341438970058E25C8F2BCDA6166C8BF1B065C174",
+       "8B2BACE575179D762F6F2FFDBFF00B497C07766AB3EED9961447CF6F43D06A97");
+}
+
+function test_invalid_input_round2() {
+  let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]   
+            .createInstance(Ci.nsISyncJPAKE);
+
+  a.round1("alice", {}, {}, {}, {}, {}, {});
+  do_check_throws(function() {
+    a.round2("invalid", "sekrit", "some", "real", "garbage",
+             "even", "more", "garbage", {}, {}, {});
+  });
+}
+
+function test_invalid_input_final() {
+  let a = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+  let b = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+            .createInstance(Ci.nsISyncJPAKE);
+
+  let gx1 = {};
+  let gv1 = {};
+  let r1 = {};
+  let gx2 = {};
+  let gv2 = {};
+  let r2 = {};
+
+  a.round1("alice", {}, {}, {}, {}, {}, {});
+  b.round1("bob", gx1, gv1, r1, gx2, gv2, r2);
+  a.round2("bob", "sekrit", gx1.value, gv1.value, r1.value,
+           gx2.value, gv2.value, r2.value, {}, {}, {});
+  do_check_throws(function() {
+    a.final("some", "garbage", "alright", "foobar-info", {}, {});
+  });
+}
+
+function run_test() {
+  test_x4_zero();
+  test_success();
+  test_failure();
+  test_same_signerids();
+  test_bad_zkp();
+  test_invalid_input_round2();
+  test_invalid_input_final();
+}

services/crypto/component/tests/unit/xpcshell.ini

@@ -0,0 +1,6 @@
+[DEFAULT]
+head = 
+tail = 
+firefox-appdir = browser
+
+[test_jpake.js]

services/crypto/cryptoComponents.manifest

@@ -0,0 +1 @@
+resource services-crypto resource://gre/modules/services-crypto/

services/crypto/modules/WeaveCrypto.js

@@ -0,0 +1,266 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["WeaveCrypto"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://services-common/async.js");
+
+Cu.importGlobalProperties(['crypto']);
+
+const CRYPT_ALGO        = "AES-CBC";
+const CRYPT_ALGO_LENGTH = 256;
+const AES_CBC_IV_SIZE   = 16;
+const OPERATIONS        = { ENCRYPT: 0, DECRYPT: 1 };
+const UTF_LABEL          = "utf-8";
+
+const KEY_DERIVATION_ALGO         = "PBKDF2";
+const KEY_DERIVATION_HASHING_ALGO = "SHA-1";
+const KEY_DERIVATION_ITERATIONS   = 4096; // PKCS#5 recommends at least 1000.
+const DERIVED_KEY_ALGO            = CRYPT_ALGO;
+
+this.WeaveCrypto = function WeaveCrypto() {
+    this.init();
+};
+
+WeaveCrypto.prototype = {
+    prefBranch : null,
+    debug      : true,  // services.sync.log.cryptoDebug
+
+    observer : {
+        _self : null,
+
+        QueryInterface : XPCOMUtils.generateQI([Ci.nsIObserver,
+                                                Ci.nsISupportsWeakReference]),
+
+        observe(subject, topic, data) {
+            let self = this._self;
+            self.log("Observed " + topic + " topic.");
+            if (topic == "nsPref:changed") {
+                self.debug = self.prefBranch.getBoolPref("cryptoDebug");
+            }
+        }
+    },
+
+    init() {
+        // Preferences. Add observer so we get notified of changes.
+        this.prefBranch = Services.prefs.getBranch("services.sync.log.");
+        this.prefBranch.addObserver("cryptoDebug", this.observer, false);
+        this.observer._self = this;
+        try {
+          this.debug = this.prefBranch.getBoolPref("cryptoDebug");
+        } catch (x) {
+          this.debug = false;
+        }
+        XPCOMUtils.defineLazyGetter(this, 'encoder', () => new TextEncoder(UTF_LABEL));
+        XPCOMUtils.defineLazyGetter(this, 'decoder', () => new TextDecoder(UTF_LABEL, { fatal: true }));
+    },
+
+    log(message) {
+        if (!this.debug) {
+            return;
+        }
+        dump("WeaveCrypto: " + message + "\n");
+        Services.console.logStringMessage("WeaveCrypto: " + message);
+    },
+
+    // /!\ Only use this for tests! /!\
+    _getCrypto() {
+        return crypto;
+    },
+
+    encrypt(clearTextUCS2, symmetricKey, iv) {
+        this.log("encrypt() called");
+        let clearTextBuffer = this.encoder.encode(clearTextUCS2).buffer;
+        let encrypted = this._commonCrypt(clearTextBuffer, symmetricKey, iv, OPERATIONS.ENCRYPT);
+        return this.encodeBase64(encrypted);
+    },
+
+    decrypt(cipherText, symmetricKey, iv) {
+        this.log("decrypt() called");
+        if (cipherText.length) {
+            cipherText = atob(cipherText);
+        }
+        let cipherTextBuffer = this.byteCompressInts(cipherText);
+        let decrypted = this._commonCrypt(cipherTextBuffer, symmetricKey, iv, OPERATIONS.DECRYPT);
+        return this.decoder.decode(decrypted);
+    },
+
+    /**
+     * _commonCrypt
+     *
+     * @args
+     * data: data to encrypt/decrypt (ArrayBuffer)
+     * symKeyStr: symmetric key (Base64 String)
+     * ivStr: initialization vector (Base64 String)
+     * operation: operation to apply (either OPERATIONS.ENCRYPT or OPERATIONS.DECRYPT)
+     * @returns
+     * the encrypted/decrypted data (ArrayBuffer)
+    */
+    _commonCrypt(data, symKeyStr, ivStr, operation) {
+        this.log("_commonCrypt() called");
+        ivStr = atob(ivStr);
+
+        if (operation !== OPERATIONS.ENCRYPT && operation !== OPERATIONS.DECRYPT) {
+            throw new Error("Unsupported operation in _commonCrypt.");
+        }
+        // We never want an IV longer than the block size, which is 16 bytes
+        // for AES, neither do we want one smaller; throw in both cases.
+        if (ivStr.length !== AES_CBC_IV_SIZE) {
+            throw "Invalid IV size; must be " + AES_CBC_IV_SIZE + " bytes.";
+        }
+
+        let iv = this.byteCompressInts(ivStr);
+        let symKey = this.importSymKey(symKeyStr, operation);
+        let cryptMethod = (operation === OPERATIONS.ENCRYPT
+                           ? crypto.subtle.encrypt
+                           : crypto.subtle.decrypt)
+                          .bind(crypto.subtle);
+        let algo = { name: CRYPT_ALGO, iv: iv };
+
+
+        return Async.promiseSpinningly(
+            cryptMethod(algo, symKey, data)
+            .then(keyBytes => new Uint8Array(keyBytes))
+        );
+    },
+
+
+    generateRandomKey() {
+        this.log("generateRandomKey() called");
+        let algo = {
+            name: CRYPT_ALGO,
+            length: CRYPT_ALGO_LENGTH
+        };
+        return Async.promiseSpinningly(
+            crypto.subtle.generateKey(algo, true, [])
+            .then(key => crypto.subtle.exportKey("raw", key))
+            .then(keyBytes => {
+                keyBytes = new Uint8Array(keyBytes);
+                return this.encodeBase64(keyBytes);
+            })
+        );
+    },
+
+    generateRandomIV() {
+      return this.generateRandomBytes(AES_CBC_IV_SIZE);
+    },
+
+    generateRandomBytes(byteCount) {
+        this.log("generateRandomBytes() called");
+
+        let randBytes = new Uint8Array(byteCount);
+        crypto.getRandomValues(randBytes);
+
+        return this.encodeBase64(randBytes);
+    },
+
+    //
+    // SymKey CryptoKey memoization.
+    //
+
+    // Memoize the import of symmetric keys. We do this by using the base64
+    // string itself as a key.
+    _encryptionSymKeyMemo: {},
+    _decryptionSymKeyMemo: {},
+    importSymKey(encodedKeyString, operation) {
+        let memo;
+
+        // We use two separate memos for thoroughness: operation is an input to
+        // key import.
+        switch (operation) {
+            case OPERATIONS.ENCRYPT:
+                memo = this._encryptionSymKeyMemo;
+                break;
+            case OPERATIONS.DECRYPT:
+                memo = this._decryptionSymKeyMemo;
+                break;
+            default:
+                throw "Unsupported operation in importSymKey.";
+        }
+
+        if (encodedKeyString in memo)
+            return memo[encodedKeyString];
+
+        let symmetricKeyBuffer = this.makeUint8Array(encodedKeyString, true);
+        let algo = { name: CRYPT_ALGO };
+        let usages = [operation === OPERATIONS.ENCRYPT ? "encrypt" : "decrypt"];
+
+        return Async.promiseSpinningly(
+            crypto.subtle.importKey("raw", symmetricKeyBuffer, algo, false, usages)
+            .then(symKey => {
+                memo[encodedKeyString] = symKey;
+                return symKey;
+            })
+        );
+    },
+
+
+    //
+    // Utility functions
+    //
+
+    /**
+     * Returns an Uint8Array filled with a JS string,
+     * which means we only keep utf-16 characters from 0x00 to 0xFF.
+     */
+    byteCompressInts(str) {
+        let arrayBuffer = new Uint8Array(str.length);
+        for (let i = 0; i < str.length; i++) {
+            arrayBuffer[i] = str.charCodeAt(i) & 0xFF;
+        }
+        return arrayBuffer;
+    },
+
+    expandData(data) {
+        let expanded = "";
+        for (let i = 0; i < data.length; i++) {
+            expanded += String.fromCharCode(data[i]);
+        }
+        return expanded;
+    },
+
+    encodeBase64(data) {
+        return btoa(this.expandData(data));
+    },
+
+    makeUint8Array(input, isEncoded) {
+        if (isEncoded) {
+            input = atob(input);
+        }
+        return this.byteCompressInts(input);
+    },
+
+    /**
+     * Returns the expanded data string for the derived key.
+     */
+    deriveKeyFromPassphrase(passphrase, saltStr, keyLength = 32) {
+        this.log("deriveKeyFromPassphrase() called.");
+        let keyData = this.makeUint8Array(passphrase, false);
+        let salt = this.makeUint8Array(saltStr, true);
+        let importAlgo = { name: KEY_DERIVATION_ALGO };
+        let deriveAlgo = {
+            name: KEY_DERIVATION_ALGO,
+            salt: salt,
+            iterations: KEY_DERIVATION_ITERATIONS,
+            hash: { name: KEY_DERIVATION_HASHING_ALGO },
+        };
+        let derivedKeyType = {
+            name: DERIVED_KEY_ALGO,
+            length: keyLength * 8,
+        };
+        return Async.promiseSpinningly(
+            crypto.subtle.importKey("raw", keyData, importAlgo, false, ["deriveKey"])
+            .then(key => crypto.subtle.deriveKey(deriveAlgo, key, derivedKeyType, true, []))
+            .then(derivedKey => crypto.subtle.exportKey("raw", derivedKey))
+            .then(keyBytes => {
+                keyBytes = new Uint8Array(keyBytes);
+                return this.expandData(keyBytes);
+            })
+        );
+    },
+};

services/crypto/modules/utils.js

@@ -0,0 +1,584 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+this.EXPORTED_SYMBOLS = ["CryptoUtils"];
+
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+this.CryptoUtils = {
+  xor: function xor(a, b) {
+    let bytes = [];
+
+    if (a.length != b.length) {
+      throw new Error("can't xor unequal length strings: "+a.length+" vs "+b.length);
+    }
+
+    for (let i = 0; i < a.length; i++) {
+      bytes[i] = a.charCodeAt(i) ^ b.charCodeAt(i);
+    }
+
+    return String.fromCharCode.apply(String, bytes);
+  },
+
+  /**
+   * Generate a string of random bytes.
+   */
+  generateRandomBytes: function generateRandomBytes(length) {
+    let rng = Cc["@mozilla.org/security/random-generator;1"]
+                .createInstance(Ci.nsIRandomGenerator);
+    let bytes = rng.generateRandomBytes(length);
+    return CommonUtils.byteArrayToString(bytes);
+  },
+
+  /**
+   * UTF8-encode a message and hash it with the given hasher. Returns a
+   * string containing bytes. The hasher is reset if it's an HMAC hasher.
+   */
+  digestUTF8: function digestUTF8(message, hasher) {
+    let data = this._utf8Converter.convertToByteArray(message, {});
+    hasher.update(data, data.length);
+    let result = hasher.finish(false);
+    if (hasher instanceof Ci.nsICryptoHMAC) {
+      hasher.reset();
+    }
+    return result;
+  },
+
+  /**
+   * Treat the given message as a bytes string and hash it with the given
+   * hasher. Returns a string containing bytes. The hasher is reset if it's
+   * an HMAC hasher.
+   */
+  digestBytes: function digestBytes(message, hasher) {
+    // No UTF-8 encoding for you, sunshine.
+    let bytes = Array.prototype.slice.call(message).map(b => b.charCodeAt(0));
+    hasher.update(bytes, bytes.length);
+    let result = hasher.finish(false);
+    if (hasher instanceof Ci.nsICryptoHMAC) {
+      hasher.reset();
+    }
+    return result;
+  },
+
+  /**
+   * Encode the message into UTF-8 and feed the resulting bytes into the
+   * given hasher. Does not return a hash. This can be called multiple times
+   * with a single hasher, but eventually you must extract the result
+   * yourself.
+   */
+  updateUTF8: function(message, hasher) {
+    let bytes = this._utf8Converter.convertToByteArray(message, {});
+    hasher.update(bytes, bytes.length);
+  },
+
+  /**
+   * UTF-8 encode a message and perform a SHA-1 over it.
+   *
+   * @param message
+   *        (string) Buffer to perform operation on. Should be a JS string.
+   *                 It is possible to pass in a string representing an array
+   *                 of bytes. But, you probably don't want to UTF-8 encode
+   *                 such data and thus should not be using this function.
+   *
+   * @return string
+   *         Raw bytes constituting SHA-1 hash. Value is a JS string. Each
+   *         character is the byte value for that offset. Returned string
+   *         always has .length == 20.
+   */
+  UTF8AndSHA1: function UTF8AndSHA1(message) {
+    let hasher = Cc["@mozilla.org/security/hash;1"]
+                 .createInstance(Ci.nsICryptoHash);
+    hasher.init(hasher.SHA1);
+
+    return CryptoUtils.digestUTF8(message, hasher);
+  },
+
+  sha1: function sha1(message) {
+    return CommonUtils.bytesAsHex(CryptoUtils.UTF8AndSHA1(message));
+  },
+
+  sha1Base32: function sha1Base32(message) {
+    return CommonUtils.encodeBase32(CryptoUtils.UTF8AndSHA1(message));
+  },
+
+  sha256(message) {
+    let hasher = Cc["@mozilla.org/security/hash;1"]
+                 .createInstance(Ci.nsICryptoHash);
+    hasher.init(hasher.SHA256);
+    return CommonUtils.bytesAsHex(CryptoUtils.digestUTF8(message, hasher));
+  },
+
+  /**
+   * Produce an HMAC key object from a key string.
+   */
+  makeHMACKey: function makeHMACKey(str) {
+    return Svc.KeyFactory.keyFromString(Ci.nsIKeyObject.HMAC, str);
+  },
+
+  /**
+   * Produce an HMAC hasher and initialize it with the given HMAC key.
+   */
+  makeHMACHasher: function makeHMACHasher(type, key) {
+    let hasher = Cc["@mozilla.org/security/hmac;1"]
+                   .createInstance(Ci.nsICryptoHMAC);
+    hasher.init(type, key);
+    return hasher;
+  },
+
+  /**
+   * HMAC-based Key Derivation (RFC 5869).
+   */
+  hkdf: function hkdf(ikm, xts, info, len) {
+    const BLOCKSIZE = 256 / 8;
+    if (typeof xts === undefined)
+      xts = String.fromCharCode(0, 0, 0, 0,  0, 0, 0, 0,
+                                0, 0, 0, 0,  0, 0, 0, 0,
+                                0, 0, 0, 0,  0, 0, 0, 0,
+                                0, 0, 0, 0,  0, 0, 0, 0);
+    let h = CryptoUtils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256,
+                                       CryptoUtils.makeHMACKey(xts));
+    let prk = CryptoUtils.digestBytes(ikm, h);
+    return CryptoUtils.hkdfExpand(prk, info, len);
+  },
+
+  /**
+   * HMAC-based Key Derivation Step 2 according to RFC 5869.
+   */
+  hkdfExpand: function hkdfExpand(prk, info, len) {
+    const BLOCKSIZE = 256 / 8;
+    let h = CryptoUtils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256,
+                                       CryptoUtils.makeHMACKey(prk));
+    let T = "";
+    let Tn = "";
+    let iterations = Math.ceil(len/BLOCKSIZE);
+    for (let i = 0; i < iterations; i++) {
+      Tn = CryptoUtils.digestBytes(Tn + info + String.fromCharCode(i + 1), h);
+      T += Tn;
+    }
+    return T.slice(0, len);
+  },
+
+  /**
+   * PBKDF2 implementation in Javascript.
+   *
+   * The arguments to this function correspond to items in
+   * PKCS #5, v2.0 pp. 9-10
+   *
+   * P: the passphrase, an octet string:              e.g., "secret phrase"
+   * S: the salt, an octet string:                    e.g., "DNXPzPpiwn"
+   * c: the number of iterations, a positive integer: e.g., 4096
+   * dkLen: the length in octets of the destination
+   *        key, a positive integer:                  e.g., 16
+   * hmacAlg: The algorithm to use for hmac
+   * hmacLen: The hmac length
+   *
+   * The default value of 20 for hmacLen is appropriate for SHA1.  For SHA256,
+   * hmacLen should be 32.
+   *
+   * The output is an octet string of length dkLen, which you
+   * can encode as you wish.
+   */
+  pbkdf2Generate : function pbkdf2Generate(P, S, c, dkLen,
+                       hmacAlg=Ci.nsICryptoHMAC.SHA1, hmacLen=20) {
+
+    // We don't have a default in the algo itself, as NSS does.
+    // Use the constant.
+    if (!dkLen) {
+      dkLen = SYNC_KEY_DECODED_LENGTH;
+    }
+
+    function F(S, c, i, h) {
+
+      function XOR(a, b, isA) {
+        if (a.length != b.length) {
+          return false;
+        }
+
+        let val = [];
+        for (let i = 0; i < a.length; i++) {
+          if (isA) {
+            val[i] = a[i] ^ b[i];
+          } else {
+            val[i] = a.charCodeAt(i) ^ b.charCodeAt(i);
+          }
+        }
+
+        return val;
+      }
+
+      let ret;
+      let U = [];
+
+      /* Encode i into 4 octets: _INT */
+      let I = [];
+      I[0] = String.fromCharCode((i >> 24) & 0xff);
+      I[1] = String.fromCharCode((i >> 16) & 0xff);
+      I[2] = String.fromCharCode((i >> 8) & 0xff);
+      I[3] = String.fromCharCode(i & 0xff);
+
+      U[0] = CryptoUtils.digestBytes(S + I.join(''), h);
+      for (let j = 1; j < c; j++) {
+        U[j] = CryptoUtils.digestBytes(U[j - 1], h);
+      }
+
+      ret = U[0];
+      for (let j = 1; j < c; j++) {
+        ret = CommonUtils.byteArrayToString(XOR(ret, U[j]));
+      }
+
+      return ret;
+    }
+
+    let l = Math.ceil(dkLen / hmacLen);
+    let r = dkLen - ((l - 1) * hmacLen);
+
+    // Reuse the key and the hasher. Remaking them 4096 times is 'spensive.
+    let h = CryptoUtils.makeHMACHasher(hmacAlg,
+                                       CryptoUtils.makeHMACKey(P));
+
+    let T = [];
+    for (let i = 0; i < l;) {
+      T[i] = F(S, c, ++i, h);
+    }
+
+    let ret = "";
+    for (let i = 0; i < l-1;) {
+      ret += T[i++];
+    }
+    ret += T[l - 1].substr(0, r);
+
+    return ret;
+  },
+
+  deriveKeyFromPassphrase: function deriveKeyFromPassphrase(passphrase,
+                                                            salt,
+                                                            keyLength,
+                                                            forceJS) {
+    if (Svc.Crypto.deriveKeyFromPassphrase && !forceJS) {
+      return Svc.Crypto.deriveKeyFromPassphrase(passphrase, salt, keyLength);
+    }
+    else {
+      // Fall back to JS implementation.
+      // 4096 is hardcoded in WeaveCrypto, so do so here.
+      return CryptoUtils.pbkdf2Generate(passphrase, atob(salt), 4096,
+                                        keyLength);
+    }
+  },
+
+  /**
+   * Compute the HTTP MAC SHA-1 for an HTTP request.
+   *
+   * @param  identifier
+   *         (string) MAC Key Identifier.
+   * @param  key
+   *         (string) MAC Key.
+   * @param  method
+   *         (string) HTTP request method.
+   * @param  URI
+   *         (nsIURI) HTTP request URI.
+   * @param  extra
+   *         (object) Optional extra parameters. Valid keys are:
+   *           nonce_bytes - How many bytes the nonce should be. This defaults
+   *             to 8. Note that this many bytes are Base64 encoded, so the
+   *             string length of the nonce will be longer than this value.
+   *           ts - Timestamp to use. Should only be defined for testing.
+   *           nonce - String nonce. Should only be defined for testing as this
+   *             function will generate a cryptographically secure random one
+   *             if not defined.
+   *           ext - Extra string to be included in MAC. Per the HTTP MAC spec,
+   *             the format is undefined and thus application specific.
+   * @returns
+   *         (object) Contains results of operation and input arguments (for
+   *           symmetry). The object has the following keys:
+   *
+   *           identifier - (string) MAC Key Identifier (from arguments).
+   *           key - (string) MAC Key (from arguments).
+   *           method - (string) HTTP request method (from arguments).
+   *           hostname - (string) HTTP hostname used (derived from arguments).
+   *           port - (string) HTTP port number used (derived from arguments).
+   *           mac - (string) Raw HMAC digest bytes.
+   *           getHeader - (function) Call to obtain the string Authorization
+   *             header value for this invocation.
+   *           nonce - (string) Nonce value used.
+   *           ts - (number) Integer seconds since Unix epoch that was used.
+   */
+  computeHTTPMACSHA1: function computeHTTPMACSHA1(identifier, key, method,
+                                                  uri, extra) {
+    let ts = (extra && extra.ts) ? extra.ts : Math.floor(Date.now() / 1000);
+    let nonce_bytes = (extra && extra.nonce_bytes > 0) ? extra.nonce_bytes : 8;
+
+    // We are allowed to use more than the Base64 alphabet if we want.
+    let nonce = (extra && extra.nonce)
+                ? extra.nonce
+                : btoa(CryptoUtils.generateRandomBytes(nonce_bytes));
+
+    let host = uri.asciiHost;
+    let port;
+    let usedMethod = method.toUpperCase();
+
+    if (uri.port != -1) {
+      port = uri.port;
+    } else if (uri.scheme == "http") {
+      port = "80";
+    } else if (uri.scheme == "https") {
+      port = "443";
+    } else {
+      throw new Error("Unsupported URI scheme: " + uri.scheme);
+    }
+
+    let ext = (extra && extra.ext) ? extra.ext : "";
+
+    let requestString = ts.toString(10) + "\n" +
+                        nonce           + "\n" +
+                        usedMethod      + "\n" +
+                        uri.path        + "\n" +
+                        host            + "\n" +
+                        port            + "\n" +
+                        ext             + "\n";
+
+    let hasher = CryptoUtils.makeHMACHasher(Ci.nsICryptoHMAC.SHA1,
+                                            CryptoUtils.makeHMACKey(key));
+    let mac = CryptoUtils.digestBytes(requestString, hasher);
+
+    function getHeader() {
+      return CryptoUtils.getHTTPMACSHA1Header(this.identifier, this.ts,
+                                              this.nonce, this.mac, this.ext);
+    }
+
+    return {
+      identifier: identifier,
+      key:        key,
+      method:     usedMethod,
+      hostname:   host,
+      port:       port,
+      mac:        mac,
+      nonce:      nonce,
+      ts:         ts,
+      ext:        ext,
+      getHeader:  getHeader
+    };
+  },
+
+
+  /**
+   * Obtain the HTTP MAC Authorization header value from fields.
+   *
+   * @param  identifier
+   *         (string) MAC key identifier.
+   * @param  ts
+   *         (number) Integer seconds since Unix epoch.
+   * @param  nonce
+   *         (string) Nonce value.
+   * @param  mac
+   *         (string) Computed HMAC digest (raw bytes).
+   * @param  ext
+   *         (optional) (string) Extra string content.
+   * @returns
+   *         (string) Value to put in Authorization header.
+   */
+  getHTTPMACSHA1Header: function getHTTPMACSHA1Header(identifier, ts, nonce,
+                                                      mac, ext) {
+    let header ='MAC id="' + identifier + '", ' +
+                'ts="'     + ts         + '", ' +
+                'nonce="'  + nonce      + '", ' +
+                'mac="'    + btoa(mac)  + '"';
+
+    if (!ext) {
+      return header;
+    }
+
+    return header += ', ext="' + ext +'"';
+  },
+
+  /**
+   * Given an HTTP header value, strip out any attributes.
+   */
+
+  stripHeaderAttributes: function(value) {
+    value = value || "";
+    let i = value.indexOf(";");
+    return value.substring(0, (i >= 0) ? i : undefined).trim().toLowerCase();
+  },
+
+  /**
+   * Compute the HAWK client values (mostly the header) for an HTTP request.
+   *
+   * @param  URI
+   *         (nsIURI) HTTP request URI.
+   * @param  method
+   *         (string) HTTP request method.
+   * @param  options
+   *         (object) extra parameters (all but "credentials" are optional):
+   *           credentials - (object, mandatory) HAWK credentials object.
+   *             All three keys are required:
+   *             id - (string) key identifier
+   *             key - (string) raw key bytes
+   *             algorithm - (string) which hash to use: "sha1" or "sha256"
+   *           ext - (string) application-specific data, included in MAC
+   *           localtimeOffsetMsec - (number) local clock offset (vs server)
+   *           payload - (string) payload to include in hash, containing the
+   *                     HTTP request body. If not provided, the HAWK hash
+   *                     will not cover the request body, and the server
+   *                     should not check it either. This will be UTF-8
+   *                     encoded into bytes before hashing. This function
+   *                     cannot handle arbitrary binary data, sorry (the
+   *                     UTF-8 encoding process will corrupt any codepoints
+   *                     between U+0080 and U+00FF). Callers must be careful
+   *                     to use an HTTP client function which encodes the
+   *                     payload exactly the same way, otherwise the hash
+   *                     will not match.
+   *           contentType - (string) payload Content-Type. This is included
+   *                         (without any attributes like "charset=") in the
+   *                         HAWK hash. It does *not* affect interpretation
+   *                         of the "payload" property.
+   *           hash - (base64 string) pre-calculated payload hash. If
+   *                  provided, "payload" is ignored.
+   *           ts - (number) pre-calculated timestamp, secs since epoch
+   *           now - (number) current time, ms-since-epoch, for tests
+   *           nonce - (string) pre-calculated nonce. Should only be defined
+   *                   for testing as this function will generate a
+   *                   cryptographically secure random one if not defined.
+   * @returns
+   *         (object) Contains results of operation. The object has the
+   *         following keys:
+   *           field - (string) HAWK header, to use in Authorization: header
+   *           artifacts - (object) other generated values:
+   *             ts - (number) timestamp, in seconds since epoch
+   *             nonce - (string)
+   *             method - (string)
+   *             resource - (string) path plus querystring
+   *             host - (string)
+   *             port - (number)
+   *             hash - (string) payload hash (base64)
+   *             ext - (string) app-specific data
+   *             MAC - (string) request MAC (base64)
+   */
+  computeHAWK: function(uri, method, options) {
+    let credentials = options.credentials;
+    let ts = options.ts || Math.floor(((options.now || Date.now()) +
+                                       (options.localtimeOffsetMsec || 0))
+                                      / 1000);
+
+    let hash_algo, hmac_algo;
+    if (credentials.algorithm == "sha1") {
+      hash_algo = Ci.nsICryptoHash.SHA1;
+      hmac_algo = Ci.nsICryptoHMAC.SHA1;
+    } else if (credentials.algorithm == "sha256") {
+      hash_algo = Ci.nsICryptoHash.SHA256;
+      hmac_algo = Ci.nsICryptoHMAC.SHA256;
+    } else {
+      throw new Error("Unsupported algorithm: " + credentials.algorithm);
+    }
+
+    let port;
+    if (uri.port != -1) {
+      port = uri.port;
+    } else if (uri.scheme == "http") {
+      port = 80;
+    } else if (uri.scheme == "https") {
+      port = 443;
+    } else {
+      throw new Error("Unsupported URI scheme: " + uri.scheme);
+    }
+
+    let artifacts = {
+      ts: ts,
+      nonce: options.nonce || btoa(CryptoUtils.generateRandomBytes(8)),
+      method: method.toUpperCase(),
+      resource: uri.path, // This includes both path and search/queryarg.
+      host: uri.asciiHost.toLowerCase(), // This includes punycoding.
+      port: port.toString(10),
+      hash: options.hash,
+      ext: options.ext,
+    };
+
+    let contentType = CryptoUtils.stripHeaderAttributes(options.contentType);
+
+    if (!artifacts.hash && options.hasOwnProperty("payload")
+        && options.payload) {
+      let hasher = Cc["@mozilla.org/security/hash;1"]
+                     .createInstance(Ci.nsICryptoHash);
+      hasher.init(hash_algo);
+      CryptoUtils.updateUTF8("hawk.1.payload\n", hasher);
+      CryptoUtils.updateUTF8(contentType+"\n", hasher);
+      CryptoUtils.updateUTF8(options.payload, hasher);
+      CryptoUtils.updateUTF8("\n", hasher);
+      let hash = hasher.finish(false);
+      // HAWK specifies this .hash to use +/ (not _-) and include the
+      // trailing "==" padding.
+      let hash_b64 = btoa(hash);
+      artifacts.hash = hash_b64;
+    }
+
+    let requestString = ("hawk.1.header"        + "\n" +
+                         artifacts.ts.toString(10) + "\n" +
+                         artifacts.nonce        + "\n" +
+                         artifacts.method       + "\n" +
+                         artifacts.resource     + "\n" +
+                         artifacts.host         + "\n" +
+                         artifacts.port         + "\n" +
+                         (artifacts.hash || "") + "\n");
+    if (artifacts.ext) {
+      requestString += artifacts.ext.replace("\\", "\\\\").replace("\n", "\\n");
+    }
+    requestString += "\n";
+
+    let hasher = CryptoUtils.makeHMACHasher(hmac_algo,
+                                            CryptoUtils.makeHMACKey(credentials.key));
+    artifacts.mac = btoa(CryptoUtils.digestBytes(requestString, hasher));
+    // The output MAC uses "+" and "/", and padded== .
+
+    function escape(attribute) {
+      // This is used for "x=y" attributes inside HTTP headers.
+      return attribute.replace(/\\/g, "\\\\").replace(/\"/g, '\\"');
+    }
+    let header = ('Hawk id="' + credentials.id + '", ' +
+                  'ts="' + artifacts.ts + '", ' +
+                  'nonce="' + artifacts.nonce + '", ' +
+                  (artifacts.hash ? ('hash="' + artifacts.hash + '", ') : "") +
+                  (artifacts.ext ? ('ext="' + escape(artifacts.ext) + '", ') : "") +
+                  'mac="' + artifacts.mac + '"');
+    return {
+      artifacts: artifacts,
+      field: header,
+    };
+  },
+
+};
+
+XPCOMUtils.defineLazyGetter(CryptoUtils, "_utf8Converter", function() {
+  let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                    .createInstance(Ci.nsIScriptableUnicodeConverter);
+  converter.charset = "UTF-8";
+
+  return converter;
+});
+
+var Svc = {};
+
+XPCOMUtils.defineLazyServiceGetter(Svc,
+                                   "KeyFactory",
+                                   "@mozilla.org/security/keyobjectfactory;1",
+                                   "nsIKeyObjectFactory");
+
+Svc.__defineGetter__("Crypto", function() {
+  let ns = {};
+  Cu.import("resource://services-crypto/WeaveCrypto.js", ns);
+
+  let wc = new ns.WeaveCrypto();
+  delete Svc.Crypto;
+  return Svc.Crypto = wc;
+});
+
+Observers.add("xpcom-shutdown", function unloadServices() {
+  Observers.remove("xpcom-shutdown", unloadServices);
+
+  for (let k in Svc) {
+    delete Svc[k];
+  }
+});

services/crypto/moz.build

@@ -0,0 +1,21 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('**'):
+    BUG_COMPONENT = ('Mozilla Services', 'Firefox Sync: Crypto')
+
+DIRS += ['component']
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/unit/xpcshell.ini']
+
+EXTRA_JS_MODULES['services-crypto'] += [
+    'modules/utils.js',
+    'modules/WeaveCrypto.js',
+]
+
+EXTRA_COMPONENTS += [
+    'cryptoComponents.manifest',
+]

services/crypto/tests/unit/head_helpers.js

@@ -0,0 +1,55 @@
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+try {
+  // In the context of xpcshell tests, there won't be a default AppInfo
+  Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULAppInfo);
+}
+catch(ex) {
+
+// Make sure to provide the right OS so crypto loads the right binaries
+var OS = "XPCShell";
+if (mozinfo.os == "win")
+  OS = "WINNT";
+else if (mozinfo.os == "mac")
+  OS = "Darwin";
+else
+  OS = "Linux";
+
+Cu.import("resource://testing-common/AppInfo.jsm", this);
+updateAppInfo({
+  name: "XPCShell",
+  ID: "{3e3ba16c-1675-4e88-b9c8-afef81b3d2ef}",
+  version: "1",
+  platformVersion: "",
+  OS: OS,
+});
+}
+
+// Register resource alias. Normally done in SyncComponents.manifest.
+function addResourceAlias() {
+  Cu.import("resource://gre/modules/Services.jsm");
+  const resProt = Services.io.getProtocolHandler("resource")
+                          .QueryInterface(Ci.nsIResProtocolHandler);
+  let uri = Services.io.newURI("resource://gre/modules/services-crypto/",
+                               null, null);
+  resProt.setSubstitution("services-crypto", uri);
+}
+addResourceAlias();
+
+/**
+ * Print some debug message to the console. All arguments will be printed,
+ * separated by spaces.
+ *
+ * @param [arg0, arg1, arg2, ...]
+ *        Any number of arguments to print out
+ * @usage _("Hello World") -> prints "Hello World"
+ * @usage _(1, 2, 3) -> prints "1 2 3"
+ */
+var _ = function(some, debug, text, to) {
+  print(Array.slice(arguments).join(" "));
+};

services/crypto/tests/unit/test_crypto_crypt.js

@@ -0,0 +1,213 @@
+Cu.import("resource://services-crypto/WeaveCrypto.js");
+Cu.importGlobalProperties(['crypto']);
+
+var cryptoSvc = new WeaveCrypto();
+
+add_task(function* test_key_memoization() {
+  let cryptoGlobal = cryptoSvc._getCrypto();
+  let oldImport = cryptoGlobal.subtle.importKey;
+  if (!oldImport) {
+    _("Couldn't swizzle crypto.subtle.importKey; returning.");
+    return;
+  }
+
+  let iv  = cryptoSvc.generateRandomIV();
+  let key = cryptoSvc.generateRandomKey();
+  let c   = 0;
+  cryptoGlobal.subtle.importKey = function(format, keyData, algo, extractable, usages) {
+    c++;
+    return oldImport.call(cryptoGlobal.subtle, format, keyData, algo, extractable, usages);
+  }
+
+  // Encryption should cause a single counter increment.
+  do_check_eq(c, 0);
+  let cipherText = cryptoSvc.encrypt("Hello, world.", key, iv);
+  do_check_eq(c, 1);
+  cipherText = cryptoSvc.encrypt("Hello, world.", key, iv);
+  do_check_eq(c, 1);
+
+  // ... as should decryption.
+  cryptoSvc.decrypt(cipherText, key, iv);
+  cryptoSvc.decrypt(cipherText, key, iv);
+  cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(c, 2);
+
+  // Un-swizzle.
+  cryptoGlobal.subtle.importKey = oldImport;
+});
+
+// Just verify that it gets populated with the correct bytes.
+add_task(function* test_makeUint8Array() {
+  Components.utils.import("resource://gre/modules/ctypes.jsm");
+
+  let item1 = cryptoSvc.makeUint8Array("abcdefghi", false);
+  do_check_true(item1);
+  for (let i = 0; i < 8; ++i)
+    do_check_eq(item1[i], "abcdefghi".charCodeAt(i));
+});
+
+add_task(function* test_encrypt_decrypt() {
+  // First, do a normal run with expected usage... Generate a random key and
+  // iv, encrypt and decrypt a string.
+  var iv = cryptoSvc.generateRandomIV();
+  do_check_eq(iv.length, 24);
+
+  var key = cryptoSvc.generateRandomKey();
+  do_check_eq(key.length, 44);
+
+  var mySecret = "bacon is a vegetable";
+  var cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  do_check_eq(cipherText.length, 44);
+
+  var clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(clearText.length, 20);
+
+  // Did the text survive the encryption round-trip?
+  do_check_eq(clearText, mySecret);
+  do_check_neq(cipherText, mySecret); // just to be explicit
+
+
+  // Do some more tests with a fixed key/iv, to check for reproducable results.
+  key = "St1tFCor7vQEJNug/465dQ==";
+  iv  = "oLjkfrLIOnK2bDRvW4kXYA==";
+
+  _("Testing small IV.");
+  mySecret = "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=";
+  let shortiv  = "YWJj";
+  let err;
+  try {
+    cryptoSvc.encrypt(mySecret, key, shortiv);
+  } catch (ex) {
+    err = ex;
+  }
+  do_check_true(!!err);
+
+  _("Testing long IV.");
+  let longiv  = "gsgLRDaxWvIfKt75RjuvFWERt83FFsY2A0TW+0b2iVk=";
+  try {
+    cryptoSvc.encrypt(mySecret, key, longiv);
+  } catch (ex) {
+    err = ex;
+  }
+  do_check_true(!!err);
+
+  // Test small input sizes
+  mySecret = "";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "OGQjp6mK1a3fs9k9Ml4L3w==");
+  do_check_eq(clearText, mySecret);
+
+  mySecret = "x";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "96iMl4vhOxFUW/lVHHzVqg==");
+  do_check_eq(clearText, mySecret);
+
+  mySecret = "xx";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "olpPbETRYROCSqFWcH2SWg==");
+  do_check_eq(clearText, mySecret);
+
+  mySecret = "xxx";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "rRbpHGyVSZizLX/x43Wm+Q==");
+  do_check_eq(clearText, mySecret);
+
+  mySecret = "xxxx";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "HeC7miVGDcpxae9RmiIKAw==");
+  do_check_eq(clearText, mySecret);
+
+  // Test non-ascii input
+  // ("testuser1" using similar-looking glyphs)
+  mySecret = String.fromCharCode(355, 277, 349, 357, 533, 537, 101, 345, 185);
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "Pj4ixByXoH3SU3JkOXaEKPgwRAWplAWFLQZkpJd5Kr4=");
+  do_check_eq(clearText, mySecret);
+
+  // Tests input spanning a block boundary (AES block size is 16 bytes)
+  mySecret = "123456789012345";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "e6c5hwphe45/3VN/M0bMUA==");
+  do_check_eq(clearText, mySecret);
+
+  mySecret = "1234567890123456";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "V6aaOZw8pWlYkoIHNkhsP1JOIQF87E2vTUvBUQnyV04=");
+  do_check_eq(clearText, mySecret);
+
+  mySecret = "12345678901234567";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "V6aaOZw8pWlYkoIHNkhsP5GvxWJ9+GIAS6lXw+5fHTI=");
+  do_check_eq(clearText, mySecret);
+
+
+  key = "iz35tuIMq4/H+IYw2KTgow==";
+  iv  = "TJYrvva2KxvkM8hvOIvWp3==";
+  mySecret = "i like pie";
+
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "DLGx8BWqSCLGG7i/xwvvxg==");
+  do_check_eq(clearText, mySecret);
+
+  key = "c5hG3YG+NC61FFy8NOHQak1ZhMEWO79bwiAfar2euzI=";
+  iv  = "gsgLRDaxWvIfKt75RjuvFW==";
+  mySecret = "i like pie";
+
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  clearText = cryptoSvc.decrypt(cipherText, key, iv);
+  do_check_eq(cipherText, "o+ADtdMd8ubzNWurS6jt0Q==");
+  do_check_eq(clearText, mySecret);
+
+  key = "St1tFCor7vQEJNug/465dQ==";
+  iv  = "oLjkfrLIOnK2bDRvW4kXYA==";
+  mySecret = "does thunder read testcases?";
+  cipherText = cryptoSvc.encrypt(mySecret, key, iv);
+  do_check_eq(cipherText, "T6fik9Ros+DB2ablH9zZ8FWZ0xm/szSwJjIHZu7sjPs=");
+
+  var badkey    = "badkeybadkeybadkeybadk==";
+  var badiv     = "badivbadivbadivbadivbad=";
+  var badcipher = "crapinputcrapinputcrapinputcrapinputcrapinp=";
+  var failure;
+
+  try {
+    failure = false;
+    clearText = cryptoSvc.decrypt(cipherText, badkey, iv);
+  } catch (e) {
+    failure = true;
+  }
+  do_check_true(failure);
+
+  try {
+    failure = false;
+    clearText = cryptoSvc.decrypt(cipherText, key, badiv);
+  } catch (e) {
+    failure = true;
+  }
+  do_check_true(failure);
+
+  try {
+    failure = false;
+    clearText = cryptoSvc.decrypt(cipherText, badkey, badiv);
+  } catch (e) {
+    failure = true;
+  }
+  do_check_true(failure);
+
+  try {
+    failure = false;
+    clearText = cryptoSvc.decrypt(badcipher, key, iv);
+  } catch (e) {
+    failure = true;
+  }
+  do_check_true(failure);
+});

services/crypto/tests/unit/test_crypto_deriveKey.js

@@ -0,0 +1,28 @@
+Components.utils.import("resource://services-crypto/WeaveCrypto.js");
+
+function run_test() {
+  let cryptoSvc = new WeaveCrypto();
+  // Extracted from test_utils_deriveKey.
+  let pp = "secret phrase";
+  let salt = "RE5YUHpQcGl3bg==";   // btoa("DNXPzPpiwn")
+
+  // 16-byte, extract key data.
+  let k = cryptoSvc.deriveKeyFromPassphrase(pp, salt, 16);
+  do_check_eq(16, k.length);
+  do_check_eq(btoa(k), "d2zG0d2cBfXnRwMUGyMwyg==");
+
+  // Test different key lengths.
+  k = cryptoSvc.deriveKeyFromPassphrase(pp, salt, 32);
+  do_check_eq(32, k.length);
+  do_check_eq(btoa(k), "d2zG0d2cBfXnRwMUGyMwyroRXtnrSIeLwSDvReSfcyA=");
+  let encKey = btoa(k);
+
+  // Test via encryption.
+  let iv = cryptoSvc.generateRandomIV();
+  do_check_eq(cryptoSvc.decrypt(cryptoSvc.encrypt("bacon", encKey, iv), encKey, iv), "bacon");
+
+  // Test default length (32).
+  k = cryptoSvc.deriveKeyFromPassphrase(pp, salt);
+  do_check_eq(32, k.length);
+  do_check_eq(encKey, btoa(k));
+}

services/crypto/tests/unit/test_crypto_random.js

@@ -0,0 +1,58 @@
+var WeaveCryptoModule = Cu.import("resource://services-crypto/WeaveCrypto.js");
+
+var cryptoSvc = new WeaveCrypto();
+
+function run_test() {
+  if (this.gczeal) {
+    _("Running crypto random tests with gczeal(2).");
+    gczeal(2);
+  }
+
+  // Test salt generation.
+  var salt;
+
+  salt = cryptoSvc.generateRandomBytes(0);
+  do_check_eq(salt.length, 0);
+  salt = cryptoSvc.generateRandomBytes(1);
+  do_check_eq(salt.length, 4);
+  salt = cryptoSvc.generateRandomBytes(2);
+  do_check_eq(salt.length, 4);
+  salt = cryptoSvc.generateRandomBytes(3);
+  do_check_eq(salt.length, 4);
+  salt = cryptoSvc.generateRandomBytes(4);
+  do_check_eq(salt.length, 8);
+  salt = cryptoSvc.generateRandomBytes(8);
+  do_check_eq(salt.length, 12);
+
+  // sanity check to make sure salts seem random
+  var salt2 = cryptoSvc.generateRandomBytes(8);
+  do_check_eq(salt2.length, 12);
+  do_check_neq(salt, salt2);
+
+  salt = cryptoSvc.generateRandomBytes(1024);
+  do_check_eq(salt.length, 1368);
+  salt = cryptoSvc.generateRandomBytes(16);
+  do_check_eq(salt.length, 24);
+
+
+  // Test random key generation
+  var keydata, keydata2, iv;
+
+  keydata  = cryptoSvc.generateRandomKey();
+  do_check_eq(keydata.length, 44);
+  keydata2 = cryptoSvc.generateRandomKey();
+  do_check_neq(keydata, keydata2); // sanity check for randomness
+  iv = cryptoSvc.generateRandomIV();
+  do_check_eq(iv.length, 24);
+
+  cryptoSvc.algorithm = WeaveCryptoModule.AES_256_CBC;
+  keydata  = cryptoSvc.generateRandomKey();
+  do_check_eq(keydata.length, 44);
+  keydata2 = cryptoSvc.generateRandomKey();
+  do_check_neq(keydata, keydata2); // sanity check for randomness
+  iv = cryptoSvc.generateRandomIV();
+  do_check_eq(iv.length, 24);
+
+  if (this.gczeal)
+    gczeal(0);
+}

services/crypto/tests/unit/test_load_modules.js

@@ -0,0 +1,16 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const modules = [
+  "utils.js",
+  "WeaveCrypto.js",
+];
+
+function run_test() {
+  for (let m of modules) {
+    let resource = "resource://services-crypto/" + m;
+    _("Attempting to import: " + resource);
+    Components.utils.import(resource, {});
+  }
+}
+