import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

services/sync/SyncComponents.manifest

@@ -0,0 +1,21 @@
+# WeaveService has to restrict its registration for the app-startup category
+# to the specific list of apps that use it so it doesn't get loaded in xpcshell.
+# Thus we restrict it to these apps:
+#
+#   b2g:            {3c2e2abc-06d4-11e1-ac3b-374f68613e61}
+#   browser:        {ec8030f7-c20a-464f-9b0e-13a3a9e97384}
+#   mobile/android: {aa3c5121-dab2-40e2-81ca-7ea25febc110}
+#   mobile/xul:     {a23983c0-fd0e-11dc-95ff-0800200c9a66}
+#   suite (comm):   {92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}
+#   graphene:       {d1bfe7d9-c01e-4237-998b-7b5f960a4314}
+
+# Weave.js
+component {74b89fb0-f200-4ae8-a3ec-dd164117f6de} Weave.js
+contract @mozilla.org/weave/service;1 {74b89fb0-f200-4ae8-a3ec-dd164117f6de}
+category app-startup WeaveService service,@mozilla.org/weave/service;1 application={3c2e2abc-06d4-11e1-ac3b-374f68613e61} application={ec8030f7-c20a-464f-9b0e-13a3a9e97384} application={aa3c5121-dab2-40e2-81ca-7ea25febc110} application={a23983c0-fd0e-11dc-95ff-0800200c9a66} application={92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a} application={99bceaaa-e3c6-48c1-b981-ef9b46b67d60} application={d1bfe7d9-c01e-4237-998b-7b5f960a4314}
+component {d28f8a0b-95da-48f4-b712-caf37097be41} Weave.js
+contract @mozilla.org/network/protocol/about;1?what=sync-log {d28f8a0b-95da-48f4-b712-caf37097be41}
+
+# Register resource aliases
+# (Note, for tests these are also set up in addResourceAlias)
+resource services-sync resource://gre/modules/services-sync/

services/sync/Weave.js

@@ -0,0 +1,200 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://services-sync/util.js");
+
+const SYNC_PREFS_BRANCH = "services.sync.";
+
+
+/**
+ * Sync's XPCOM service.
+ *
+ * It is named "Weave" for historical reasons.
+ *
+ * It's worth noting how Sync is lazily loaded. We register a timer that
+ * loads Sync a few seconds after app startup. This is so Sync does not
+ * adversely affect application start time.
+ *
+ * If Sync is not configured, no extra Sync code is loaded. If an
+ * external component (say the UI) needs to interact with Sync, it
+ * should use the promise-base function whenLoaded() - something like the
+ * following:
+ *
+ * // 1. Grab a handle to the Sync XPCOM service.
+ * let service = Cc["@mozilla.org/weave/service;1"]
+ *                 .getService(Components.interfaces.nsISupports)
+ *                 .wrappedJSObject;
+ *
+ * // 2. Use the .then method of the promise.
+ * service.whenLoaded().then(() => {
+ *   // You are free to interact with "Weave." objects.
+ *   return;
+ * });
+ *
+ * And that's it!  However, if you really want to avoid promises and do it
+ * old-school, then
+ *
+ * // 1. Get a reference to the service as done in (1) above.
+ *
+ * // 2. Check if the service has been initialized.
+ * if (service.ready) {
+ *   // You are free to interact with "Weave." objects.
+ *   return;
+ * }
+ *
+ * // 3. Install "ready" listener.
+ * Services.obs.addObserver(function onReady() {
+ *   Services.obs.removeObserver(onReady, "weave:service:ready");
+ *
+ *   // You are free to interact with "Weave." objects.
+ * }, "weave:service:ready", false);
+ *
+ * // 4. Trigger loading of Sync.
+ * service.ensureLoaded();
+ */
+function WeaveService() {
+  this.wrappedJSObject = this;
+  this.ready = false;
+}
+WeaveService.prototype = {
+  classID: Components.ID("{74b89fb0-f200-4ae8-a3ec-dd164117f6de}"),
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver,
+                                         Ci.nsISupportsWeakReference]),
+
+  ensureLoaded: function () {
+    // If we are loaded and not using FxA, load the migration module.
+    if (!this.fxAccountsEnabled) {
+      Cu.import("resource://services-sync/FxaMigrator.jsm");
+    }
+
+    Components.utils.import("resource://services-sync/main.js");
+
+    // Side-effect of accessing the service is that it is instantiated.
+    Weave.Service;
+  },
+
+  whenLoaded: function() {
+    if (this.ready) {
+      return Promise.resolve();
+    }
+    let deferred = Promise.defer();
+
+    Services.obs.addObserver(function onReady() {
+      Services.obs.removeObserver(onReady, "weave:service:ready");
+      deferred.resolve();
+    }, "weave:service:ready", false);
+    this.ensureLoaded();
+    return deferred.promise;
+  },
+
+  /**
+   * Whether Firefox Accounts is enabled.
+   *
+   * @return bool
+   */
+  get fxAccountsEnabled() {
+    try {
+      // Old sync guarantees '@' will never appear in the username while FxA
+      // uses the FxA email address - so '@' is the flag we use.
+      let username = Services.prefs.getCharPref(SYNC_PREFS_BRANCH + "username");
+      return !username || username.includes('@');
+    } catch (_) {
+      return true; // No username == only allow FxA to be configured.
+    }
+  },
+
+  /**
+   * Whether Sync appears to be enabled.
+   *
+   * This returns true if all the Sync preferences for storing account
+   * and server configuration are populated.
+   *
+   * It does *not* perform a robust check to see if the client is working.
+   * For that, you'll want to check Weave.Status.checkSetup().
+   */
+  get enabled() {
+    let prefs = Services.prefs.getBranch(SYNC_PREFS_BRANCH);
+    return prefs.prefHasUserValue("username");
+  },
+
+  observe: function (subject, topic, data) {
+    switch (topic) {
+    case "app-startup":
+      let os = Cc["@mozilla.org/observer-service;1"].
+               getService(Ci.nsIObserverService);
+      os.addObserver(this, "final-ui-startup", true);
+      break;
+
+    case "final-ui-startup":
+      // Force Weave service to load if it hasn't triggered from overlays
+      this.timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+      this.timer.initWithCallback({
+        notify: function() {
+          let isConfigured = false;
+          // We only load more if it looks like Sync is configured.
+          let prefs = Services.prefs.getBranch(SYNC_PREFS_BRANCH);
+          if (prefs.prefHasUserValue("username")) {
+            // We have a username. So, do a more thorough check. This will
+            // import a number of modules and thus increase memory
+            // accordingly. We could potentially copy code performed by
+            // this check into this file if our above code is yielding too
+            // many false positives.
+            Components.utils.import("resource://services-sync/main.js");
+            isConfigured = Weave.Status.checkSetup() != Weave.CLIENT_NOT_CONFIGURED;
+          }
+          let getHistogramById = Services.telemetry.getHistogramById;
+          getHistogramById("WEAVE_CONFIGURED").add(isConfigured);
+          if (isConfigured) {
+            getHistogramById("WEAVE_CONFIGURED_MASTER_PASSWORD").add(Utils.mpEnabled());
+            this.ensureLoaded();
+          }
+        }.bind(this)
+      }, 10000, Ci.nsITimer.TYPE_ONE_SHOT);
+      break;
+    }
+  }
+};
+
+function AboutWeaveLog() {}
+AboutWeaveLog.prototype = {
+  classID: Components.ID("{d28f8a0b-95da-48f4-b712-caf37097be41}"),
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIAboutModule,
+                                         Ci.nsISupportsWeakReference]),
+
+  getURIFlags: function(aURI) {
+    return 0;
+  },
+
+  newChannel: function(aURI, aLoadInfo) {
+    let dir = FileUtils.getDir("ProfD", ["weave", "logs"], true);
+    let uri = Services.io.newFileURI(dir);
+    let channel = Services.io.newChannelFromURIWithLoadInfo(uri, aLoadInfo);
+
+    channel.originalURI = aURI;
+
+    // Ensure that the about page has the same privileges as a regular directory
+    // view. That way links to files can be opened. make sure we use the correct
+    // origin attributes when creating the principal for accessing the
+    // about:sync-log data.
+    let ssm = Cc["@mozilla.org/scriptsecuritymanager;1"]
+                .getService(Ci.nsIScriptSecurityManager);
+    let principal = ssm.createCodebasePrincipal(uri, aLoadInfo.originAttributes);
+
+    channel.owner = principal;
+    return channel;
+  }
+};
+
+const components = [WeaveService, AboutWeaveLog];
+this.NSGetFactory = XPCOMUtils.generateNSGetFactory(components);

services/sync/locales/en-US/errors.properties

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+error.login.reason.network      = Failed to connect to the server
+error.login.reason.recoverykey  = Wrong Recovery Key
+error.login.reason.account      = Incorrect account name or password
+error.login.reason.no_username  = Missing account name
+error.login.reason.no_password2 = Missing password
+error.login.reason.no_recoverykey= No saved Recovery Key to use
+error.login.reason.server       = Server incorrectly configured
+
+error.sync.failed_partial            = One or more data types could not be synced
+# LOCALIZATION NOTE (error.sync.reason.serverMaintenance): We removed the extraneous period from this string
+error.sync.reason.serverMaintenance  = Firefox Sync server maintenance is underway, syncing will resume automatically
+
+invalid-captcha = Incorrect words, try again
+weak-password   = Use a stronger password
+
+# this is the fallback, if we hit an error we didn't bother to localize
+error.reason.unknown          = Unknown error
+
+change.password.pwSameAsPassword     = Password can’t match current password
+change.password.pwSameAsUsername     = Password can’t match your user name
+change.password.pwSameAsEmail        = Password can’t match your email address
+change.password.mismatch             = The passwords entered do not match
+change.password.tooShort             = The password entered is too short

services/sync/locales/en-US/sync.properties

@@ -0,0 +1,27 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# %1: the user name (Ed), %2: the app name (Firefox), %3: the operating system (Android)
+client.name2 = %1$S’s %2$S on %3$S
+
+# %S is the date and time at which the last sync successfully completed
+lastSync2.label = Last sync: %S
+
+# signInToSync.description is the tooltip for the Sync buttons when Sync is
+# not configured.
+signInToSync.description = Sign In To Sync
+
+error.sync.title = Error While Syncing
+error.sync.description = Sync encountered an error while syncing: %1$S.  Sync will automatically retry this action.
+warning.sync.eol.label = Service Shutting Down
+# %1: the app name (Firefox)
+warning.sync.eol.description = Your Firefox Sync service is shutting down soon. Upgrade %1$S to keep syncing.
+error.sync.eol.label = Service Unavailable
+# %1: the app name (Firefox)
+error.sync.eol.description = Your Firefox Sync service is no longer available. You need to upgrade %1$S to keep syncing.
+sync.eol.learnMore.label = Learn more
+sync.eol.learnMore.accesskey = L
+
+syncnow.label = Sync Now
+syncing2.label = Syncing…

services/sync/locales/jar.mn

@@ -0,0 +1,10 @@
+#filter substitution
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+@AB_CD@.jar:
+% locale weave @AB_CD@ %locale/@AB_CD@/
+  locale/@AB_CD@/services/errors.properties  (%errors.properties)
+  locale/@AB_CD@/services/sync.properties    (%sync.properties)

services/sync/locales/l10n.ini

@@ -0,0 +1,9 @@
+; This Source Code Form is subject to the terms of the Mozilla Public
+; License, v. 2.0. If a copy of the MPL was not distributed with this
+; file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+[general]
+depth = ../../..
+
+[compare]
+dirs = services/sync

services/sync/locales/moz.build

@@ -0,0 +1,7 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+JAR_MANIFESTS += ['jar.mn']

services/sync/modules-testing/fakeservices.js

@@ -0,0 +1,131 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "FakeCryptoService",
+  "FakeFilesystemService",
+  "FakeGUIDService",
+  "fakeSHA256HMAC",
+];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+
+var btoa = Cu.import("resource://gre/modules/Log.jsm").btoa;
+
+this.FakeFilesystemService = function FakeFilesystemService(contents) {
+  this.fakeContents = contents;
+  let self = this;
+
+  // Save away the unmocked versions of the functions we replace here for tests
+  // that really want the originals. As this may be called many times per test,
+  // we must be careful to not replace them with ones we previously replaced.
+  // (And WTF are we bothering with these mocks in the first place? Is the
+  // performance of the filesystem *really* such that it outweighs the downside
+  // of not running our real JSON functions in the tests? Eg, these mocks don't
+  // always throw exceptions when the real ones do. Anyway...)
+  for (let name of ["jsonSave", "jsonLoad", "jsonMove", "jsonRemove"]) {
+    let origName = "_real_" + name;
+    if (!Utils[origName]) {
+      Utils[origName] = Utils[name];
+    }
+  }
+
+  Utils.jsonSave = function jsonSave(filePath, that, obj, callback) {
+    let json = typeof obj == "function" ? obj.call(that) : obj;
+    self.fakeContents["weave/" + filePath + ".json"] = JSON.stringify(json);
+    callback.call(that);
+  };
+
+  Utils.jsonLoad = function jsonLoad(filePath, that, cb) {
+    let obj;
+    let json = self.fakeContents["weave/" + filePath + ".json"];
+    if (json) {
+      obj = JSON.parse(json);
+    }
+    cb.call(that, obj);
+  };
+
+  Utils.jsonMove = function jsonMove(aFrom, aTo, that) {
+    const fromPath = "weave/" + aFrom + ".json";
+    self.fakeContents["weave/" + aTo + ".json"] = self.fakeContents[fromPath];
+    delete self.fakeContents[fromPath];
+    return Promise.resolve();
+  };
+
+  Utils.jsonRemove = function jsonRemove(filePath, that) {
+    delete self.fakeContents["weave/" + filePath + ".json"];
+    return Promise.resolve();
+  };
+};
+
+this.fakeSHA256HMAC = function fakeSHA256HMAC(message) {
+   message = message.substr(0, 64);
+   while (message.length < 64) {
+     message += " ";
+   }
+   return message;
+}
+
+this.FakeGUIDService = function FakeGUIDService() {
+  let latestGUID = 0;
+
+  Utils.makeGUID = function makeGUID() {
+    // ensure that this always returns a unique 12 character string
+    let nextGUID = "fake-guid-" + String(latestGUID++).padStart(2, "0");
+    return nextGUID.slice(nextGUID.length-12, nextGUID.length);
+  };
+}
+
+/*
+ * Mock implementation of WeaveCrypto. It does not encrypt or
+ * decrypt, merely returning the input verbatim.
+ */
+this.FakeCryptoService = function FakeCryptoService() {
+  this.counter = 0;
+
+  delete Svc.Crypto;  // get rid of the getter first
+  Svc.Crypto = this;
+
+  CryptoWrapper.prototype.ciphertextHMAC = function ciphertextHMAC(keyBundle) {
+    return fakeSHA256HMAC(this.ciphertext);
+  };
+}
+FakeCryptoService.prototype = {
+
+  encrypt: function encrypt(clearText, symmetricKey, iv) {
+    return clearText;
+  },
+
+  decrypt: function decrypt(cipherText, symmetricKey, iv) {
+    return cipherText;
+  },
+
+  generateRandomKey: function generateRandomKey() {
+    return btoa("fake-symmetric-key-" + this.counter++);
+  },
+
+  generateRandomIV: function generateRandomIV() {
+    // A base64-encoded IV is 24 characters long
+    return btoa("fake-fake-fake-random-iv");
+  },
+
+  expandData: function expandData(data, len) {
+    return data;
+  },
+
+  deriveKeyFromPassphrase: function deriveKeyFromPassphrase(passphrase,
+                                                            salt, keyLength) {
+    return "some derived key string composed of bytes";
+  },
+
+  generateRandomBytes: function generateRandomBytes(byteCount) {
+    return "not-so-random-now-are-we-HA-HA-HA! >:)".slice(byteCount);
+  }
+};
+

services/sync/modules-testing/fxa_utils.js

@@ -0,0 +1,58 @@
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "initializeIdentityWithTokenServerResponse",
+];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/main.js");
+Cu.import("resource://services-sync/browserid_identity.js");
+Cu.import("resource://services-common/tokenserverclient.js");
+Cu.import("resource://testing-common/services/common/logging.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+
+// Create a new browserid_identity object and initialize it with a
+// mocked TokenServerClient which always receives the specified response.
+this.initializeIdentityWithTokenServerResponse = function(response) {
+  // First create a mock "request" object that well' hack into the token server.
+  // A log for it
+  let requestLog = Log.repository.getLogger("testing.mock-rest");
+  if (!requestLog.appenders.length) { // might as well see what it says :)
+    requestLog.addAppender(new Log.DumpAppender());
+    requestLog.level = Log.Level.Trace;
+  }
+
+  // A mock request object.
+  function MockRESTRequest(url) {};
+  MockRESTRequest.prototype = {
+    _log: requestLog,
+    setHeader: function() {},
+    get: function(callback) {
+      this.response = response;
+      callback.call(this);
+    }
+  }
+  // The mocked TokenServer client which will get the response.
+  function MockTSC() { }
+  MockTSC.prototype = new TokenServerClient();
+  MockTSC.prototype.constructor = MockTSC;
+  MockTSC.prototype.newRESTRequest = function(url) {
+    return new MockRESTRequest(url);
+  }
+  // Arrange for the same observerPrefix as browserid_identity uses.
+  MockTSC.prototype.observerPrefix = "weave:service";
+
+  // tie it all together.
+  Weave.Status.__authManager = Weave.Service.identity = new BrowserIDManager();
+  Weave.Service._clusterManager = Weave.Service.identity.createClusterManager(Weave.Service);
+  let browseridManager = Weave.Service.identity;
+  // a sanity check
+  if (!(browseridManager instanceof BrowserIDManager)) {
+    throw new Error("sync isn't configured for browserid_identity");
+  }
+  let mockTSC = new MockTSC()
+  configureFxAccountIdentity(browseridManager);
+  browseridManager._tokenServerClient = mockTSC;
+}

services/sync/modules-testing/rotaryengine.js

@@ -0,0 +1,124 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "RotaryEngine",
+  "RotaryRecord",
+  "RotaryStore",
+  "RotaryTracker",
+];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+
+/*
+ * A fake engine implementation.
+ * This is used all over the place.
+ *
+ * Complete with record, store, and tracker implementations.
+ */
+
+this.RotaryRecord = function RotaryRecord(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+RotaryRecord.prototype = {
+  __proto__: CryptoWrapper.prototype
+};
+Utils.deferGetSet(RotaryRecord, "cleartext", ["denomination"]);
+
+this.RotaryStore = function RotaryStore(name, engine) {
+  Store.call(this, name, engine);
+  this.items = {};
+}
+RotaryStore.prototype = {
+  __proto__: Store.prototype,
+
+  create: function create(record) {
+    this.items[record.id] = record.denomination;
+  },
+
+  remove: function remove(record) {
+    delete this.items[record.id];
+  },
+
+  update: function update(record) {
+    this.items[record.id] = record.denomination;
+  },
+
+  itemExists: function itemExists(id) {
+    return (id in this.items);
+  },
+
+  createRecord: function createRecord(id, collection) {
+    let record = new RotaryRecord(collection, id);
+
+    if (!(id in this.items)) {
+      record.deleted = true;
+      return record;
+    }
+
+    record.denomination = this.items[id] || "Data for new record: " + id;
+    return record;
+  },
+
+  changeItemID: function changeItemID(oldID, newID) {
+    if (oldID in this.items) {
+      this.items[newID] = this.items[oldID];
+    }
+
+    delete this.items[oldID];
+  },
+
+  getAllIDs: function getAllIDs() {
+    let ids = {};
+    for (let id in this.items) {
+      ids[id] = true;
+    }
+    return ids;
+  },
+
+  wipe: function wipe() {
+    this.items = {};
+  }
+};
+
+this.RotaryTracker = function RotaryTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+RotaryTracker.prototype = {
+  __proto__: Tracker.prototype
+};
+
+
+this.RotaryEngine = function RotaryEngine(service) {
+  SyncEngine.call(this, "Rotary", service);
+  // Ensure that the engine starts with a clean slate.
+  this.toFetch        = [];
+  this.previousFailed = [];
+}
+RotaryEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _storeObj: RotaryStore,
+  _trackerObj: RotaryTracker,
+  _recordObj: RotaryRecord,
+
+  _findDupe: function _findDupe(item) {
+    // This is a semaphore used for testing proper reconciling on dupe
+    // detection.
+    if (item.id == "DUPE_INCOMING") {
+      return "DUPE_LOCAL";
+    }
+
+    for (let [id, value] of Object.entries(this._store.items)) {
+      if (item.denomination == value) {
+        return id;
+      }
+    }
+  }
+};

services/sync/modules-testing/utils.js

@@ -0,0 +1,350 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "btoa", // It comes from a module import.
+  "encryptPayload",
+  "isConfiguredWithLegacyIdentity",
+  "ensureLegacyIdentityManager",
+  "setBasicCredentials",
+  "makeIdentityConfig",
+  "makeFxAccountsInternalMock",
+  "configureFxAccountIdentity",
+  "configureIdentity",
+  "SyncTestingInfrastructure",
+  "waitForZeroTimer",
+  "Promise", // from a module import
+  "add_identity_test",
+  "MockFxaStorageManager",
+  "AccountState", // from a module import
+  "sumHistogram",
+];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://services-sync/status.js");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/browserid_identity.js");
+Cu.import("resource://testing-common/services/common/logging.js");
+Cu.import("resource://testing-common/services/sync/fakeservices.js");
+Cu.import("resource://gre/modules/FxAccounts.jsm");
+Cu.import("resource://gre/modules/FxAccountsClient.jsm");
+Cu.import("resource://gre/modules/FxAccountsCommon.js");
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+
+// and grab non-exported stuff via a backstage pass.
+const {AccountState} = Cu.import("resource://gre/modules/FxAccounts.jsm", {});
+
+// A mock "storage manager" for FxAccounts that doesn't actually write anywhere.
+function MockFxaStorageManager() {
+}
+
+MockFxaStorageManager.prototype = {
+  promiseInitialized: Promise.resolve(),
+
+  initialize(accountData) {
+    this.accountData = accountData;
+  },
+
+  finalize() {
+    return Promise.resolve();
+  },
+
+  getAccountData() {
+    return Promise.resolve(this.accountData);
+  },
+
+  updateAccountData(updatedFields) {
+    for (let [name, value] of Object.entries(updatedFields)) {
+      if (value == null) {
+        delete this.accountData[name];
+      } else {
+        this.accountData[name] = value;
+      }
+    }
+    return Promise.resolve();
+  },
+
+  deleteAccountData() {
+    this.accountData = null;
+    return Promise.resolve();
+  }
+}
+
+/**
+ * First wait >100ms (nsITimers can take up to that much time to fire, so
+ * we can account for the timer in delayedAutoconnect) and then two event
+ * loop ticks (to account for the Utils.nextTick() in autoConnect).
+ */
+this.waitForZeroTimer = function waitForZeroTimer(callback) {
+  let ticks = 2;
+  function wait() {
+    if (ticks) {
+      ticks -= 1;
+      CommonUtils.nextTick(wait);
+      return;
+    }
+    callback();
+  }
+  CommonUtils.namedTimer(wait, 150, {}, "timer");
+}
+
+/**
+ * Return true if Sync is configured with the "legacy" identity provider.
+ */
+this.isConfiguredWithLegacyIdentity = function() {
+  let ns = {};
+  Cu.import("resource://services-sync/service.js", ns);
+
+  // We can't use instanceof as BrowserIDManager (the "other" identity) inherits
+  // from IdentityManager so that would return true - so check the prototype.
+  return Object.getPrototypeOf(ns.Service.identity) === IdentityManager.prototype;
+}
+
+/**
+  * Ensure Sync is configured with the "legacy" identity provider.
+  */
+this.ensureLegacyIdentityManager = function() {
+  let ns = {};
+  Cu.import("resource://services-sync/service.js", ns);
+
+  Status.__authManager = ns.Service.identity = new IdentityManager();
+  ns.Service._clusterManager = ns.Service.identity.createClusterManager(ns.Service);
+}
+
+this.setBasicCredentials =
+ function setBasicCredentials(username, password, syncKey) {
+  let ns = {};
+  Cu.import("resource://services-sync/service.js", ns);
+
+  let auth = ns.Service.identity;
+  auth.username = username;
+  auth.basicPassword = password;
+  auth.syncKey = syncKey;
+}
+
+// Return an identity configuration suitable for testing with our identity
+// providers.  |overrides| can specify overrides for any default values.
+this.makeIdentityConfig = function(overrides) {
+  // first setup the defaults.
+  let result = {
+    // Username used in both fxaccount and sync identity configs.
+    username: "foo",
+    // fxaccount specific credentials.
+    fxaccount: {
+      user: {
+        assertion: 'assertion',
+        email: 'email',
+        kA: 'kA',
+        kB: 'kB',
+        sessionToken: 'sessionToken',
+        uid: "a".repeat(32),
+        verified: true,
+      },
+      token: {
+        endpoint: null,
+        duration: 300,
+        id: "id",
+        key: "key",
+        hashed_fxa_uid: "f".repeat(32), // used during telemetry validation
+        // uid will be set to the username.
+      }
+    },
+    sync: {
+      // username will come from the top-level username
+      password: "whatever",
+      syncKey: "abcdeabcdeabcdeabcdeabcdea",
+    }
+  };
+
+  // Now handle any specified overrides.
+  if (overrides) {
+    if (overrides.username) {
+      result.username = overrides.username;
+    }
+    if (overrides.sync) {
+      // TODO: allow just some attributes to be specified
+      result.sync = overrides.sync;
+    }
+    if (overrides.fxaccount) {
+      // TODO: allow just some attributes to be specified
+      result.fxaccount = overrides.fxaccount;
+    }
+  }
+  return result;
+}
+
+this.makeFxAccountsInternalMock = function(config) {
+  return {
+    newAccountState(credentials) {
+      // We only expect this to be called with null indicating the (mock)
+      // storage should be read.
+      if (credentials) {
+        throw new Error("Not expecting to have credentials passed");
+      }
+      let storageManager = new MockFxaStorageManager();
+      storageManager.initialize(config.fxaccount.user);
+      let accountState = new AccountState(storageManager);
+      return accountState;
+    },
+    _getAssertion(audience) {
+      return Promise.resolve("assertion");
+    },
+  };
+};
+
+// Configure an instance of an FxAccount identity provider with the specified
+// config (or the default config if not specified).
+this.configureFxAccountIdentity = function(authService,
+                                           config = makeIdentityConfig(),
+                                           fxaInternal = makeFxAccountsInternalMock(config)) {
+  // until we get better test infrastructure for bid_identity, we set the
+  // signedin user's "email" to the username, simply as many tests rely on this.
+  config.fxaccount.user.email = config.username;
+
+  let fxa = new FxAccounts(fxaInternal);
+
+  let MockFxAccountsClient = function() {
+    FxAccountsClient.apply(this);
+  };
+  MockFxAccountsClient.prototype = {
+    __proto__: FxAccountsClient.prototype,
+    accountStatus() {
+      return Promise.resolve(true);
+    }
+  };
+  let mockFxAClient = new MockFxAccountsClient();
+  fxa.internal._fxAccountsClient = mockFxAClient;
+
+  let mockTSC = { // TokenServerClient
+    getTokenFromBrowserIDAssertion: function(uri, assertion, cb) {
+      config.fxaccount.token.uid = config.username;
+      cb(null, config.fxaccount.token);
+    },
+  };
+  authService._fxaService = fxa;
+  authService._tokenServerClient = mockTSC;
+  // Set the "account" of the browserId manager to be the "email" of the
+  // logged in user of the mockFXA service.
+  authService._signedInUser = config.fxaccount.user;
+  authService._account = config.fxaccount.user.email;
+}
+
+this.configureIdentity = function(identityOverrides) {
+  let config = makeIdentityConfig(identityOverrides);
+  let ns = {};
+  Cu.import("resource://services-sync/service.js", ns);
+
+  if (ns.Service.identity instanceof BrowserIDManager) {
+    // do the FxAccounts thang...
+    configureFxAccountIdentity(ns.Service.identity, config);
+    return ns.Service.identity.initializeWithCurrentIdentity().then(() => {
+      // need to wait until this identity manager is readyToAuthenticate.
+      return ns.Service.identity.whenReadyToAuthenticate.promise;
+    });
+  }
+  // old style identity provider.
+  setBasicCredentials(config.username, config.sync.password, config.sync.syncKey);
+  let deferred = Promise.defer();
+  deferred.resolve();
+  return deferred.promise;
+}
+
+this.SyncTestingInfrastructure = function (server, username, password, syncKey) {
+  let ns = {};
+  Cu.import("resource://services-sync/service.js", ns);
+
+  ensureLegacyIdentityManager();
+  let config = makeIdentityConfig();
+  // XXX - hacks for the sync identity provider.
+  if (username)
+    config.username = username;
+  if (password)
+    config.sync.password = password;
+  if (syncKey)
+    config.sync.syncKey = syncKey;
+  let cb = Async.makeSpinningCallback();
+  configureIdentity(config).then(cb, cb);
+  cb.wait();
+
+  let i = server.identity;
+  let uri = i.primaryScheme + "://" + i.primaryHost + ":" +
+            i.primaryPort + "/";
+
+  ns.Service.serverURL = uri;
+  ns.Service.clusterURL = uri;
+
+  this.logStats = initTestLogging();
+  this.fakeFilesystem = new FakeFilesystemService({});
+  this.fakeGUIDService = new FakeGUIDService();
+  this.fakeCryptoService = new FakeCryptoService();
+}
+
+/**
+ * Turn WBO cleartext into fake "encrypted" payload as it goes over the wire.
+ */
+this.encryptPayload = function encryptPayload(cleartext) {
+  if (typeof cleartext == "object") {
+    cleartext = JSON.stringify(cleartext);
+  }
+
+  return {
+    ciphertext: cleartext, // ciphertext == cleartext with fake crypto
+    IV: "irrelevant",
+    hmac: fakeSHA256HMAC(cleartext, CryptoUtils.makeHMACKey("")),
+  };
+}
+
+// This helper can be used instead of 'add_test' or 'add_task' to run the
+// specified test function twice - once with the old-style sync identity
+// manager and once with the new-style BrowserID identity manager, to ensure
+// it works in both cases.
+//
+// * The test itself should be passed as 'test' - ie, test code will generally
+//   pass |this|.
+// * The test function is a regular test function - although note that it must
+//   be a generator - async operations should yield them, and run_next_test
+//   mustn't be called.
+this.add_identity_test = function(test, testFunction) {
+  function note(what) {
+    let msg = "running test " + testFunction.name + " with " + what + " identity manager";
+    test.do_print(msg);
+  }
+  let ns = {};
+  Cu.import("resource://services-sync/service.js", ns);
+  // one task for the "old" identity manager.
+  test.add_task(function* () {
+    note("sync");
+    let oldIdentity = Status._authManager;
+    ensureLegacyIdentityManager();
+    yield testFunction();
+    Status.__authManager = ns.Service.identity = oldIdentity;
+  });
+  // another task for the FxAccounts identity manager.
+  test.add_task(function* () {
+    note("FxAccounts");
+    let oldIdentity = Status._authManager;
+    Status.__authManager = ns.Service.identity = new BrowserIDManager();
+    yield testFunction();
+    Status.__authManager = ns.Service.identity = oldIdentity;
+  });
+}
+
+this.sumHistogram = function(name, options = {}) {
+  let histogram = options.key ? Services.telemetry.getKeyedHistogramById(name) :
+                  Services.telemetry.getHistogramById(name);
+  let snapshot = histogram.snapshot(options.key);
+  let sum = -Infinity;
+  if (snapshot) {
+    sum = snapshot.sum;
+  }
+  histogram.clear();
+  return sum;
+}

services/sync/modules/FxaMigrator.jsm

@@ -0,0 +1,99 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict;"
+
+// Note that this module used to supervise the step-by-step migration from
+// a legacy Sync account to a FxA-based Sync account. In bug 1205928, this
+// changed to automatically disconnect the legacy Sync account.
+
+const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+
+XPCOMUtils.defineLazyGetter(this, "WeaveService", function() {
+  return Cc["@mozilla.org/weave/service;1"]
+         .getService(Components.interfaces.nsISupports)
+         .wrappedJSObject;
+});
+
+XPCOMUtils.defineLazyModuleGetter(this, "Weave",
+  "resource://services-sync/main.js");
+
+// We send this notification when we perform the disconnection. The browser
+// window will show a one-off notification bar.
+const OBSERVER_STATE_CHANGE_TOPIC = "fxa-migration:state-changed";
+
+const OBSERVER_TOPICS = [
+  "xpcom-shutdown",
+  "weave:eol",
+];
+
+function Migrator() {
+  // Leave the log-level as Debug - Sync will setup log appenders such that
+  // these messages generally will not be seen unless other log related
+  // prefs are set.
+  this.log.level = Log.Level.Debug;
+
+  for (let topic of OBSERVER_TOPICS) {
+    Services.obs.addObserver(this, topic, false);
+  }
+}
+
+Migrator.prototype = {
+  log: Log.repository.getLogger("Sync.SyncMigration"),
+
+  finalize() {
+    for (let topic of OBSERVER_TOPICS) {
+      Services.obs.removeObserver(this, topic);
+    }
+  },
+
+  observe(subject, topic, data) {
+    this.log.debug("observed " + topic);
+    switch (topic) {
+      case "xpcom-shutdown":
+        this.finalize();
+        break;
+
+      default:
+        // this notification when configured with legacy Sync means we want to
+        // disconnect
+        if (!WeaveService.fxAccountsEnabled) {
+          this.log.info("Disconnecting from legacy Sync");
+          // Set up an observer for when the disconnection is complete.
+          let observe;
+          Services.obs.addObserver(observe = () => {
+            this.log.info("observed that startOver is complete");
+            Services.obs.removeObserver(observe, "weave:service:start-over:finish");
+            // Send the notification for the UI.
+            Services.obs.notifyObservers(null, OBSERVER_STATE_CHANGE_TOPIC, null);
+          }, "weave:service:start-over:finish", false);
+
+          // Do the disconnection.
+          Weave.Service.startOver();
+        }
+    }
+  },
+
+  get learnMoreLink() {
+    try {
+      var url = Services.prefs.getCharPref("app.support.baseURL");
+    } catch (err) {
+      return null;
+    }
+    url += "sync-upgrade";
+    let sb = Services.strings.createBundle("chrome://weave/locale/services/sync.properties");
+    return {
+      text: sb.GetStringFromName("sync.eol.learnMore.label"),
+      href: Services.urlFormatter.formatURL(url),
+    };
+  },
+};
+
+// We expose a singleton
+this.EXPORTED_SYMBOLS = ["fxaMigrator"];
+var fxaMigrator = new Migrator();

services/sync/modules/SyncedTabs.jsm

@@ -0,0 +1,301 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["SyncedTabs"];
+
+
+const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/PlacesUtils.jsm", this);
+Cu.import("resource://services-sync/main.js");
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+// The Sync XPCOM service
+XPCOMUtils.defineLazyGetter(this, "weaveXPCService", function() {
+  return Cc["@mozilla.org/weave/service;1"]
+           .getService(Ci.nsISupports)
+           .wrappedJSObject;
+});
+
+// from MDN...
+function escapeRegExp(string) {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// A topic we fire whenever we have new tabs available. This might be due
+// to a request made by this module to refresh the tab list, or as the result
+// of a regularly scheduled sync. The intent is that consumers just listen
+// for this notification and update their UI in response.
+const TOPIC_TABS_CHANGED = "services.sync.tabs.changed";
+
+// The interval, in seconds, before which we consider the existing list
+// of tabs "fresh enough" and don't force a new sync.
+const TABS_FRESH_ENOUGH_INTERVAL = 30;
+
+let log = Log.repository.getLogger("Sync.RemoteTabs");
+// A new scope to do the logging thang...
+(function() {
+  let level = Preferences.get("services.sync.log.logger.tabs");
+  if (level) {
+    let appender = new Log.DumpAppender();
+    log.level = appender.level = Log.Level[level] || Log.Level.Debug;
+    log.addAppender(appender);
+  }
+})();
+
+
+// A private singleton that does the work.
+let SyncedTabsInternal = {
+  /* Make a "tab" record. Returns a promise */
+  _makeTab: Task.async(function* (client, tab, url, showRemoteIcons) {
+    let icon;
+    if (showRemoteIcons) {
+      icon = tab.icon;
+    }
+    if (!icon) {
+      try {
+        icon = (yield PlacesUtils.promiseFaviconLinkUrl(url)).spec;
+      } catch (ex) { /* no favicon avaiable */ }
+    }
+    if (!icon) {
+      icon = "";
+    }
+    return {
+      type:  "tab",
+      title: tab.title || url,
+      url,
+      icon,
+      client: client.id,
+      lastUsed: tab.lastUsed,
+    };
+  }),
+
+  /* Make a "client" record. Returns a promise for consistency with _makeTab */
+  _makeClient: Task.async(function* (client) {
+    return {
+      id: client.id,
+      type: "client",
+      name: Weave.Service.clientsEngine.getClientName(client.id),
+      isMobile: Weave.Service.clientsEngine.isMobile(client.id),
+      lastModified: client.lastModified * 1000, // sec to ms
+      tabs: []
+    };
+  }),
+
+  _tabMatchesFilter(tab, filter) {
+    let reFilter = new RegExp(escapeRegExp(filter), "i");
+    return tab.url.match(reFilter) || tab.title.match(reFilter);
+  },
+
+  getTabClients: Task.async(function* (filter) {
+    log.info("Generating tab list with filter", filter);
+    let result = [];
+
+    // If Sync isn't ready, don't try and get anything.
+    if (!weaveXPCService.ready) {
+      log.debug("Sync isn't yet ready, so returning an empty tab list");
+      return result;
+    }
+
+    // A boolean that controls whether we should show the icon from the remote tab.
+    const showRemoteIcons = Preferences.get("services.sync.syncedTabs.showRemoteIcons", true);
+
+    let engine = Weave.Service.engineManager.get("tabs");
+
+    let seenURLs = new Set();
+    let parentIndex = 0;
+    let ntabs = 0;
+
+    for (let [guid, client] of Object.entries(engine.getAllClients())) {
+      if (!Weave.Service.clientsEngine.remoteClientExists(client.id)) {
+        continue;
+      }
+      let clientRepr = yield this._makeClient(client);
+      log.debug("Processing client", clientRepr);
+
+      for (let tab of client.tabs) {
+        let url = tab.urlHistory[0];
+        log.debug("remote tab", url);
+        // Note there are some issues with tracking "seen" tabs, including:
+        // * We really can't return the entire urlHistory record as we are
+        //   only checking the first entry - others might be different.
+        // * We don't update the |lastUsed| timestamp to reflect the
+        //   most-recently-seen time.
+        // In a followup we should consider simply dropping this |seenUrls|
+        // check and return duplicate records - it seems the user will be more
+        // confused by tabs not showing up on a device (because it was detected
+        // as a dupe so it only appears on a different device) than being
+        // confused by seeing the same tab on different clients.
+        if (!url || seenURLs.has(url)) {
+          continue;
+        }
+        let tabRepr = yield this._makeTab(client, tab, url, showRemoteIcons);
+        if (filter && !this._tabMatchesFilter(tabRepr, filter)) {
+          continue;
+        }
+        seenURLs.add(url);
+        clientRepr.tabs.push(tabRepr);
+      }
+      // We return all clients, even those without tabs - the consumer should
+      // filter it if they care.
+      ntabs += clientRepr.tabs.length;
+      result.push(clientRepr);
+    }
+    log.info(`Final tab list has ${result.length} clients with ${ntabs} tabs.`);
+    return result;
+  }),
+
+  syncTabs(force) {
+    if (!force) {
+      // Don't bother refetching tabs if we already did so recently
+      let lastFetch = Preferences.get("services.sync.lastTabFetch", 0);
+      let now = Math.floor(Date.now() / 1000);
+      if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL) {
+        log.info("_refetchTabs was done recently, do not doing it again");
+        return Promise.resolve(false);
+      }
+    }
+
+    // If Sync isn't configured don't try and sync, else we will get reports
+    // of a login failure.
+    if (Weave.Status.checkSetup() == Weave.CLIENT_NOT_CONFIGURED) {
+      log.info("Sync client is not configured, so not attempting a tab sync");
+      return Promise.resolve(false);
+    }
+    // Ask Sync to just do the tabs engine if it can.
+    // Sync is currently synchronous, so do it after an event-loop spin to help
+    // keep the UI responsive.
+    return new Promise((resolve, reject) => {
+      Services.tm.currentThread.dispatch(() => {
+        try {
+          log.info("Doing a tab sync.");
+          Weave.Service.sync(["tabs"]);
+          resolve(true);
+        } catch (ex) {
+          log.error("Sync failed", ex);
+          reject(ex);
+        };
+      }, Ci.nsIThread.DISPATCH_NORMAL);
+    });
+  },
+
+  observe(subject, topic, data) {
+    log.trace(`observed topic=${topic}, data=${data}, subject=${subject}`);
+    switch (topic) {
+      case "weave:engine:sync:finish":
+        if (data != "tabs") {
+          return;
+        }
+        // The tabs engine just finished syncing
+        // Set our lastTabFetch pref here so it tracks both explicit sync calls
+        // and normally scheduled ones.
+        Preferences.set("services.sync.lastTabFetch", Math.floor(Date.now() / 1000));
+        Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
+        break;
+      case "weave:service:start-over":
+        // start-over needs to notify so consumers find no tabs.
+        Preferences.reset("services.sync.lastTabFetch");
+        Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
+        break;
+      case "nsPref:changed":
+        Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
+        break;
+      default:
+        break;
+    }
+  },
+
+  // Returns true if Sync is configured to Sync tabs, false otherwise
+  get isConfiguredToSyncTabs() {
+    if (!weaveXPCService.ready) {
+      log.debug("Sync isn't yet ready; assuming tab engine is enabled");
+      return true;
+    }
+
+    let engine = Weave.Service.engineManager.get("tabs");
+    return engine && engine.enabled;
+  },
+
+  get hasSyncedThisSession() {
+    let engine = Weave.Service.engineManager.get("tabs");
+    return engine && engine.hasSyncedThisSession;
+  },
+};
+
+Services.obs.addObserver(SyncedTabsInternal, "weave:engine:sync:finish", false);
+Services.obs.addObserver(SyncedTabsInternal, "weave:service:start-over", false);
+// Observe the pref the indicates the state of the tabs engine has changed.
+// This will force consumers to re-evaluate the state of sync and update
+// accordingly.
+Services.prefs.addObserver("services.sync.engine.tabs", SyncedTabsInternal, false);
+
+// The public interface.
+this.SyncedTabs = {
+  // A mock-point for tests.
+  _internal: SyncedTabsInternal,
+
+  // We make the topic for the observer notification public.
+  TOPIC_TABS_CHANGED,
+
+  // Returns true if Sync is configured to Sync tabs, false otherwise
+  get isConfiguredToSyncTabs() {
+    return this._internal.isConfiguredToSyncTabs;
+  },
+
+  // Returns true if a tab sync has completed once this session. If this
+  // returns false, then getting back no clients/tabs possibly just means we
+  // are waiting for that first sync to complete.
+  get hasSyncedThisSession() {
+    return this._internal.hasSyncedThisSession;
+  },
+
+  // Return a promise that resolves with an array of client records, each with
+  // a .tabs array. Note that part of the contract for this module is that the
+  // returned objects are not shared between invocations, so callers are free
+  // to mutate the returned objects (eg, sort, truncate) however they see fit.
+  getTabClients(query) {
+    return this._internal.getTabClients(query);
+  },
+
+  // Starts a background request to start syncing tabs. Returns a promise that
+  // resolves when the sync is complete, but there's no resolved value -
+  // callers should be listening for TOPIC_TABS_CHANGED.
+  // If |force| is true we always sync. If false, we only sync if the most
+  // recent sync wasn't "recently".
+  syncTabs(force) {
+    return this._internal.syncTabs(force);
+  },
+
+  sortTabClientsByLastUsed(clients, maxTabs = Infinity) {
+    // First sort and filter the list of tabs for each client. Note that
+    // this module promises that the objects it returns are never
+    // shared, so we are free to mutate those objects directly.
+    for (let client of clients) {
+      let tabs = client.tabs;
+      tabs.sort((a, b) => b.lastUsed - a.lastUsed);
+      if (Number.isFinite(maxTabs)) {
+        client.tabs = tabs.slice(0, maxTabs);
+      }
+    }
+    // Now sort the clients - the clients are sorted in the order of the
+    // most recent tab for that client (ie, it is important the tabs for
+    // each client are already sorted.)
+    clients.sort((a, b) => {
+      if (a.tabs.length == 0) {
+        return 1; // b comes first.
+      }
+      if (b.tabs.length == 0) {
+        return -1; // a comes first.
+      }
+      return b.tabs[0].lastUsed - a.tabs[0].lastUsed;
+    });
+  },
+};
+

services/sync/modules/addonsreconciler.js

@@ -0,0 +1,676 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains middleware to reconcile state of AddonManager for
+ * purposes of tracking events for Sync. The content in this file exists
+ * because AddonManager does not have a getChangesSinceX() API and adding
+ * that functionality properly was deemed too time-consuming at the time
+ * add-on sync was originally written. If/when AddonManager adds this API,
+ * this file can go away and the add-ons engine can be rewritten to use it.
+ *
+ * It was decided to have this tracking functionality exist in a separate
+ * standalone file so it could be more easily understood, tested, and
+ * hopefully ported.
+ */
+
+"use strict";
+
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/AddonManager.jsm");
+
+const DEFAULT_STATE_FILE = "addonsreconciler";
+
+this.CHANGE_INSTALLED   = 1;
+this.CHANGE_UNINSTALLED = 2;
+this.CHANGE_ENABLED     = 3;
+this.CHANGE_DISABLED    = 4;
+
+this.EXPORTED_SYMBOLS = ["AddonsReconciler", "CHANGE_INSTALLED",
+                         "CHANGE_UNINSTALLED", "CHANGE_ENABLED",
+                         "CHANGE_DISABLED"];
+/**
+ * Maintains state of add-ons.
+ *
+ * State is maintained in 2 data structures, an object mapping add-on IDs
+ * to metadata and an array of changes over time. The object mapping can be
+ * thought of as a minimal copy of data from AddonManager which is needed for
+ * Sync. The array is effectively a log of changes over time.
+ *
+ * The data structures are persisted to disk by serializing to a JSON file in
+ * the current profile. The data structures are updated by 2 mechanisms. First,
+ * they can be refreshed from the global state of the AddonManager. This is a
+ * sure-fire way of ensuring the reconciler is up to date. Second, the
+ * reconciler adds itself as an AddonManager listener. When it receives change
+ * notifications, it updates its internal state incrementally.
+ *
+ * The internal state is persisted to a JSON file in the profile directory.
+ *
+ * An instance of this is bound to an AddonsEngine instance. In reality, it
+ * likely exists as a singleton. To AddonsEngine, it functions as a store and
+ * an entity which emits events for tracking.
+ *
+ * The usage pattern for instances of this class is:
+ *
+ *   let reconciler = new AddonsReconciler();
+ *   reconciler.loadState(null, function(error) { ... });
+ *
+ *   // At this point, your instance should be ready to use.
+ *
+ * When you are finished with the instance, please call:
+ *
+ *   reconciler.stopListening();
+ *   reconciler.saveState(...);
+ *
+ * There are 2 classes of listeners in the AddonManager: AddonListener and
+ * InstallListener. This class is a listener for both (member functions just
+ * get called directly).
+ *
+ * When an add-on is installed, listeners are called in the following order:
+ *
+ *  IL.onInstallStarted, AL.onInstalling, IL.onInstallEnded, AL.onInstalled
+ *
+ * For non-restartless add-ons, an application restart may occur between
+ * IL.onInstallEnded and AL.onInstalled. Unfortunately, Sync likely will
+ * not be loaded when AL.onInstalled is fired shortly after application
+ * start, so it won't see this event. Therefore, for add-ons requiring a
+ * restart, Sync treats the IL.onInstallEnded event as good enough to
+ * indicate an install. For restartless add-ons, Sync assumes AL.onInstalled
+ * will follow shortly after IL.onInstallEnded and thus it ignores
+ * IL.onInstallEnded.
+ *
+ * The listeners can also see events related to the download of the add-on.
+ * This class isn't interested in those. However, there are failure events,
+ * IL.onDownloadFailed and IL.onDownloadCanceled which get called if a
+ * download doesn't complete successfully.
+ *
+ * For uninstalls, we see AL.onUninstalling then AL.onUninstalled. Like
+ * installs, the events could be separated by an application restart and Sync
+ * may not see the onUninstalled event. Again, if we require a restart, we
+ * react to onUninstalling. If not, we assume we'll get onUninstalled.
+ *
+ * Enabling and disabling work by sending:
+ *
+ *   AL.onEnabling, AL.onEnabled
+ *   AL.onDisabling, AL.onDisabled
+ *
+ * Again, they may be separated by a restart, so we heed the requiresRestart
+ * flag.
+ *
+ * Actions can be undone. All undoable actions notify the same
+ * AL.onOperationCancelled event. We treat this event like any other.
+ *
+ * Restartless add-ons have interesting behavior during uninstall. These
+ * add-ons are first disabled then they are actually uninstalled. So, we will
+ * see AL.onDisabling and AL.onDisabled. The onUninstalling and onUninstalled
+ * events only come after the Addon Manager is closed or another view is
+ * switched to. In the case of Sync performing the uninstall, the uninstall
+ * events will occur immediately. However, we still see disabling events and
+ * heed them like they were normal. In the end, the state is proper.
+ */
+this.AddonsReconciler = function AddonsReconciler() {
+  this._log = Log.repository.getLogger("Sync.AddonsReconciler");
+  let level = Svc.Prefs.get("log.logger.addonsreconciler", "Debug");
+  this._log.level = Log.Level[level];
+
+  Svc.Obs.add("xpcom-shutdown", this.stopListening, this);
+};
+AddonsReconciler.prototype = {
+  /** Flag indicating whether we are listening to AddonManager events. */
+  _listening: false,
+
+  /**
+   * Whether state has been loaded from a file.
+   *
+   * State is loaded on demand if an operation requires it.
+   */
+  _stateLoaded: false,
+
+  /**
+   * Define this as false if the reconciler should not persist state
+   * to disk when handling events.
+   *
+   * This allows test code to avoid spinning to write during observer
+   * notifications and xpcom shutdown, which appears to cause hangs on WinXP
+   * (Bug 873861).
+   */
+  _shouldPersist: true,
+
+  /** Log logger instance */
+  _log: null,
+
+  /**
+   * Container for add-on metadata.
+   *
+   * Keys are add-on IDs. Values are objects which describe the state of the
+   * add-on. This is a minimal mirror of data that can be queried from
+   * AddonManager. In some cases, we retain data longer than AddonManager.
+   */
+  _addons: {},
+
+  /**
+   * List of add-on changes over time.
+   *
+   * Each element is an array of [time, change, id].
+   */
+  _changes: [],
+
+  /**
+   * Objects subscribed to changes made to this instance.
+   */
+  _listeners: [],
+
+  /**
+   * Accessor for add-ons in this object.
+   *
+   * Returns an object mapping add-on IDs to objects containing metadata.
+   */
+  get addons() {
+    this._ensureStateLoaded();
+    return this._addons;
+  },
+
+  /**
+   * Load reconciler state from a file.
+   *
+   * The path is relative to the weave directory in the profile. If no
+   * path is given, the default one is used.
+   *
+   * If the file does not exist or there was an error parsing the file, the
+   * state will be transparently defined as empty.
+   *
+   * @param path
+   *        Path to load. ".json" is appended automatically. If not defined,
+   *        a default path will be consulted.
+   * @param callback
+   *        Callback to be executed upon file load. The callback receives a
+   *        truthy error argument signifying whether an error occurred and a
+   *        boolean indicating whether data was loaded.
+   */
+  loadState: function loadState(path, callback) {
+    let file = path || DEFAULT_STATE_FILE;
+    Utils.jsonLoad(file, this, function(json) {
+      this._addons = {};
+      this._changes = [];
+
+      if (!json) {
+        this._log.debug("No data seen in loaded file: " + file);
+        if (callback) {
+          callback(null, false);
+        }
+
+        return;
+      }
+
+      let version = json.version;
+      if (!version || version != 1) {
+        this._log.error("Could not load JSON file because version not " +
+                        "supported: " + version);
+        if (callback) {
+          callback(null, false);
+        }
+
+        return;
+      }
+
+      this._addons = json.addons;
+      for (let id in this._addons) {
+        let record = this._addons[id];
+        record.modified = new Date(record.modified);
+      }
+
+      for (let [time, change, id] of json.changes) {
+        this._changes.push([new Date(time), change, id]);
+      }
+
+      if (callback) {
+        callback(null, true);
+      }
+    });
+  },
+
+  /**
+   * Saves the current state to a file in the local profile.
+   *
+   * @param  path
+   *         String path in profile to save to. If not defined, the default
+   *         will be used.
+   * @param  callback
+   *         Function to be invoked on save completion. No parameters will be
+   *         passed to callback.
+   */
+  saveState: function saveState(path, callback) {
+    let file = path || DEFAULT_STATE_FILE;
+    let state = {version: 1, addons: {}, changes: []};
+
+    for (let [id, record] of Object.entries(this._addons)) {
+      state.addons[id] = {};
+      for (let [k, v] of Object.entries(record)) {
+        if (k == "modified") {
+          state.addons[id][k] = v.getTime();
+        }
+        else {
+          state.addons[id][k] = v;
+        }
+      }
+    }
+
+    for (let [time, change, id] of this._changes) {
+      state.changes.push([time.getTime(), change, id]);
+    }
+
+    this._log.info("Saving reconciler state to file: " + file);
+    Utils.jsonSave(file, this, state, callback);
+  },
+
+  /**
+   * Registers a change listener with this instance.
+   *
+   * Change listeners are called every time a change is recorded. The listener
+   * is an object with the function "changeListener" that takes 3 arguments,
+   * the Date at which the change happened, the type of change (a CHANGE_*
+   * constant), and the add-on state object reflecting the current state of
+   * the add-on at the time of the change.
+   *
+   * @param listener
+   *        Object containing changeListener function.
+   */
+  addChangeListener: function addChangeListener(listener) {
+    if (this._listeners.indexOf(listener) == -1) {
+      this._log.debug("Adding change listener.");
+      this._listeners.push(listener);
+    }
+  },
+
+  /**
+   * Removes a previously-installed change listener from the instance.
+   *
+   * @param listener
+   *        Listener instance to remove.
+   */
+  removeChangeListener: function removeChangeListener(listener) {
+    this._listeners = this._listeners.filter(function(element) {
+      if (element == listener) {
+        this._log.debug("Removing change listener.");
+        return false;
+      } else {
+        return true;
+      }
+    }.bind(this));
+  },
+
+  /**
+   * Tells the instance to start listening for AddonManager changes.
+   *
+   * This is typically called automatically when Sync is loaded.
+   */
+  startListening: function startListening() {
+    if (this._listening) {
+      return;
+    }
+
+    this._log.info("Registering as Add-on Manager listener.");
+    AddonManager.addAddonListener(this);
+    AddonManager.addInstallListener(this);
+    this._listening = true;
+  },
+
+  /**
+   * Tells the instance to stop listening for AddonManager changes.
+   *
+   * The reconciler should always be listening. This should only be called when
+   * the instance is being destroyed.
+   *
+   * This function will get called automatically on XPCOM shutdown. However, it
+   * is a best practice to call it yourself.
+   */
+  stopListening: function stopListening() {
+    if (!this._listening) {
+      return;
+    }
+
+    this._log.debug("Stopping listening and removing AddonManager listeners.");
+    AddonManager.removeInstallListener(this);
+    AddonManager.removeAddonListener(this);
+    this._listening = false;
+  },
+
+  /**
+   * Refreshes the global state of add-ons by querying the AddonManager.
+   */
+  refreshGlobalState: function refreshGlobalState(callback) {
+    this._log.info("Refreshing global state from AddonManager.");
+    this._ensureStateLoaded();
+
+    let installs;
+
+    AddonManager.getAllAddons(function (addons) {
+      let ids = {};
+
+      for (let addon of addons) {
+        ids[addon.id] = true;
+        this.rectifyStateFromAddon(addon);
+      }
+
+      // Look for locally-defined add-ons that no longer exist and update their
+      // record.
+      for (let [id, addon] of Object.entries(this._addons)) {
+        if (id in ids) {
+          continue;
+        }
+
+        // If the id isn't in ids, it means that the add-on has been deleted or
+        // the add-on is in the process of being installed. We detect the
+        // latter by seeing if an AddonInstall is found for this add-on.
+
+        if (!installs) {
+          let cb = Async.makeSyncCallback();
+          AddonManager.getAllInstalls(cb);
+          installs = Async.waitForSyncCallback(cb);
+        }
+
+        let installFound = false;
+        for (let install of installs) {
+          if (install.addon && install.addon.id == id &&
+              install.state == AddonManager.STATE_INSTALLED) {
+
+            installFound = true;
+            break;
+          }
+        }
+
+        if (installFound) {
+          continue;
+        }
+
+        if (addon.installed) {
+          addon.installed = false;
+          this._log.debug("Adding change because add-on not present in " +
+                          "Add-on Manager: " + id);
+          this._addChange(new Date(), CHANGE_UNINSTALLED, addon);
+        }
+      }
+
+      // See note for _shouldPersist.
+      if (this._shouldPersist) {
+        this.saveState(null, callback);
+      } else {
+        callback();
+      }
+    }.bind(this));
+  },
+
+  /**
+   * Rectifies the state of an add-on from an Addon instance.
+   *
+   * This basically says "given an Addon instance, assume it is truth and
+   * apply changes to the local state to reflect it."
+   *
+   * This function could result in change listeners being called if the local
+   * state differs from the passed add-on's state.
+   *
+   * @param addon
+   *        Addon instance being updated.
+   */
+  rectifyStateFromAddon: function rectifyStateFromAddon(addon) {
+    this._log.debug(`Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})`);
+    this._ensureStateLoaded();
+
+    let id = addon.id;
+    let enabled = !addon.userDisabled;
+    let guid = addon.syncGUID;
+    let now = new Date();
+
+    if (!(id in this._addons)) {
+      let record = {
+        id: id,
+        guid: guid,
+        enabled: enabled,
+        installed: true,
+        modified: now,
+        type: addon.type,
+        scope: addon.scope,
+        foreignInstall: addon.foreignInstall,
+        isSyncable: addon.isSyncable,
+      };
+      this._addons[id] = record;
+      this._log.debug("Adding change because add-on not present locally: " +
+                      id);
+      this._addChange(now, CHANGE_INSTALLED, record);
+      return;
+    }
+
+    let record = this._addons[id];
+    record.isSyncable = addon.isSyncable;
+
+    if (!record.installed) {
+      // It is possible the record is marked as uninstalled because an
+      // uninstall is pending.
+      if (!(addon.pendingOperations & AddonManager.PENDING_UNINSTALL)) {
+        record.installed = true;
+        record.modified = now;
+      }
+    }
+
+    if (record.enabled != enabled) {
+      record.enabled = enabled;
+      record.modified = now;
+      let change = enabled ? CHANGE_ENABLED : CHANGE_DISABLED;
+      this._log.debug("Adding change because enabled state changed: " + id);
+      this._addChange(new Date(), change, record);
+    }
+
+    if (record.guid != guid) {
+      record.guid = guid;
+      // We don't record a change because the Sync engine rectifies this on its
+      // own. This is tightly coupled with Sync. If this code is ever lifted
+      // outside of Sync, this exception should likely be removed.
+    }
+  },
+
+  /**
+   * Record a change in add-on state.
+   *
+   * @param date
+   *        Date at which the change occurred.
+   * @param change
+   *        The type of the change. A CHANGE_* constant.
+   * @param state
+   *        The new state of the add-on. From this.addons.
+   */
+  _addChange: function _addChange(date, change, state) {
+    this._log.info("Change recorded for " + state.id);
+    this._changes.push([date, change, state.id]);
+
+    for (let listener of this._listeners) {
+      try {
+        listener.changeListener.call(listener, date, change, state);
+      } catch (ex) {
+        this._log.warn("Exception calling change listener", ex);
+      }
+    }
+  },
+
+  /**
+   * Obtain the set of changes to add-ons since the date passed.
+   *
+   * This will return an array of arrays. Each entry in the array has the
+   * elements [date, change_type, id], where
+   *
+   *   date - Date instance representing when the change occurred.
+   *   change_type - One of CHANGE_* constants.
+   *   id - ID of add-on that changed.
+   */
+  getChangesSinceDate: function getChangesSinceDate(date) {
+    this._ensureStateLoaded();
+
+    let length = this._changes.length;
+    for (let i = 0; i < length; i++) {
+      if (this._changes[i][0] >= date) {
+        return this._changes.slice(i);
+      }
+    }
+
+    return [];
+  },
+
+  /**
+   * Prunes all recorded changes from before the specified Date.
+   *
+   * @param date
+   *        Entries older than this Date will be removed.
+   */
+  pruneChangesBeforeDate: function pruneChangesBeforeDate(date) {
+    this._ensureStateLoaded();
+
+    this._changes = this._changes.filter(function test_age(change) {
+      return change[0] >= date;
+    });
+  },
+
+  /**
+   * Obtains the set of all known Sync GUIDs for add-ons.
+   *
+   * @return Object with guids as keys and values of true.
+   */
+  getAllSyncGUIDs: function getAllSyncGUIDs() {
+    let result = {};
+    for (let id in this.addons) {
+      result[id] = true;
+    }
+
+    return result;
+  },
+
+  /**
+   * Obtain the add-on state record for an add-on by Sync GUID.
+   *
+   * If the add-on could not be found, returns null.
+   *
+   * @param  guid
+   *         Sync GUID of add-on to retrieve.
+   * @return Object on success on null on failure.
+   */
+  getAddonStateFromSyncGUID: function getAddonStateFromSyncGUID(guid) {
+    for (let id in this.addons) {
+      let addon = this.addons[id];
+      if (addon.guid == guid) {
+        return addon;
+      }
+    }
+
+    return null;
+  },
+
+  /**
+   * Ensures that state is loaded before continuing.
+   *
+   * This is called internally by anything that accesses the internal data
+   * structures. It effectively just-in-time loads serialized state.
+   */
+  _ensureStateLoaded: function _ensureStateLoaded() {
+    if (this._stateLoaded) {
+      return;
+    }
+
+    let cb = Async.makeSpinningCallback();
+    this.loadState(null, cb);
+    cb.wait();
+    this._stateLoaded = true;
+  },
+
+  /**
+   * Handler that is invoked as part of the AddonManager listeners.
+   */
+  _handleListener: function _handlerListener(action, addon, requiresRestart) {
+    // Since this is called as an observer, we explicitly trap errors and
+    // log them to ourselves so we don't see errors reported elsewhere.
+    try {
+      let id = addon.id;
+      this._log.debug("Add-on change: " + action + " to " + id);
+
+      // We assume that every event for non-restartless add-ons is
+      // followed by another event and that this follow-up event is the most
+      // appropriate to react to. Currently we ignore onEnabling, onDisabling,
+      // and onUninstalling for non-restartless add-ons.
+      if (requiresRestart === false) {
+        this._log.debug("Ignoring " + action + " for restartless add-on.");
+        return;
+      }
+
+      switch (action) {
+        case "onEnabling":
+        case "onEnabled":
+        case "onDisabling":
+        case "onDisabled":
+        case "onInstalled":
+        case "onInstallEnded":
+        case "onOperationCancelled":
+          this.rectifyStateFromAddon(addon);
+          break;
+
+        case "onUninstalling":
+        case "onUninstalled":
+          let id = addon.id;
+          let addons = this.addons;
+          if (id in addons) {
+            let now = new Date();
+            let record = addons[id];
+            record.installed = false;
+            record.modified = now;
+            this._log.debug("Adding change because of uninstall listener: " +
+                            id);
+            this._addChange(now, CHANGE_UNINSTALLED, record);
+          }
+      }
+
+      // See note for _shouldPersist.
+      if (this._shouldPersist) {
+        let cb = Async.makeSpinningCallback();
+        this.saveState(null, cb);
+        cb.wait();
+      }
+    }
+    catch (ex) {
+      this._log.warn("Exception", ex);
+    }
+  },
+
+  // AddonListeners
+  onEnabling: function onEnabling(addon, requiresRestart) {
+    this._handleListener("onEnabling", addon, requiresRestart);
+  },
+  onEnabled: function onEnabled(addon) {
+    this._handleListener("onEnabled", addon);
+  },
+  onDisabling: function onDisabling(addon, requiresRestart) {
+    this._handleListener("onDisabling", addon, requiresRestart);
+  },
+  onDisabled: function onDisabled(addon) {
+    this._handleListener("onDisabled", addon);
+  },
+  onInstalling: function onInstalling(addon, requiresRestart) {
+    this._handleListener("onInstalling", addon, requiresRestart);
+  },
+  onInstalled: function onInstalled(addon) {
+    this._handleListener("onInstalled", addon);
+  },
+  onUninstalling: function onUninstalling(addon, requiresRestart) {
+    this._handleListener("onUninstalling", addon, requiresRestart);
+  },
+  onUninstalled: function onUninstalled(addon) {
+    this._handleListener("onUninstalled", addon);
+  },
+  onOperationCancelled: function onOperationCancelled(addon) {
+    this._handleListener("onOperationCancelled", addon);
+  },
+
+  // InstallListeners
+  onInstallEnded: function onInstallEnded(install, addon) {
+    this._handleListener("onInstallEnded", addon);
+  }
+};

services/sync/modules/addonutils.js

@@ -0,0 +1,506 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["AddonUtils"];
+
+var {interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/util.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
+  "resource://gre/modules/AddonManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository",
+  "resource://gre/modules/addons/AddonRepository.jsm");
+
+function AddonUtilsInternal() {
+  this._log = Log.repository.getLogger("Sync.AddonUtils");
+  this._log.Level = Log.Level[Svc.Prefs.get("log.logger.addonutils")];
+}
+AddonUtilsInternal.prototype = {
+  /**
+   * Obtain an AddonInstall object from an AddonSearchResult instance.
+   *
+   * The callback will be invoked with the result of the operation. The
+   * callback receives 2 arguments, error and result. Error will be falsy
+   * on success or some kind of error value otherwise. The result argument
+   * will be an AddonInstall on success or null on failure. It is possible
+   * for the error to be falsy but result to be null. This could happen if
+   * an install was not found.
+   *
+   * @param addon
+   *        AddonSearchResult to obtain install from.
+   * @param cb
+   *        Function to be called with result of operation.
+   */
+  getInstallFromSearchResult:
+    function getInstallFromSearchResult(addon, cb) {
+
+    this._log.debug("Obtaining install for " + addon.id);
+
+    // We should theoretically be able to obtain (and use) addon.install if
+    // it is available. However, the addon.sourceURI rewriting won't be
+    // reflected in the AddonInstall, so we can't use it. If we ever get rid
+    // of sourceURI rewriting, we can avoid having to reconstruct the
+    // AddonInstall.
+    AddonManager.getInstallForURL(
+      addon.sourceURI.spec,
+      function handleInstall(install) {
+        cb(null, install);
+      },
+      "application/x-xpinstall",
+      undefined,
+      addon.name,
+      addon.iconURL,
+      addon.version
+    );
+  },
+
+  /**
+   * Installs an add-on from an AddonSearchResult instance.
+   *
+   * The options argument defines extra options to control the install.
+   * Recognized keys in this map are:
+   *
+   *   syncGUID - Sync GUID to use for the new add-on.
+   *   enabled - Boolean indicating whether the add-on should be enabled upon
+   *             install.
+   *
+   * When complete it calls a callback with 2 arguments, error and result.
+   *
+   * If error is falsy, result is an object. If error is truthy, result is
+   * null.
+   *
+   * The result object has the following keys:
+   *
+   *   id      ID of add-on that was installed.
+   *   install AddonInstall that was installed.
+   *   addon   Addon that was installed.
+   *
+   * @param addon
+   *        AddonSearchResult to install add-on from.
+   * @param options
+   *        Object with additional metadata describing how to install add-on.
+   * @param cb
+   *        Function to be invoked with result of operation.
+   */
+  installAddonFromSearchResult:
+    function installAddonFromSearchResult(addon, options, cb) {
+    this._log.info("Trying to install add-on from search result: " + addon.id);
+
+    this.getInstallFromSearchResult(addon, function onResult(error, install) {
+      if (error) {
+        cb(error, null);
+        return;
+      }
+
+      if (!install) {
+        cb(new Error("AddonInstall not available: " + addon.id), null);
+        return;
+      }
+
+      try {
+        this._log.info("Installing " + addon.id);
+        let log = this._log;
+
+        let listener = {
+          onInstallStarted: function onInstallStarted(install) {
+            if (!options) {
+              return;
+            }
+
+            if (options.syncGUID) {
+              log.info("Setting syncGUID of " + install.name  +": " +
+                       options.syncGUID);
+              install.addon.syncGUID = options.syncGUID;
+            }
+
+            // We only need to change userDisabled if it is disabled because
+            // enabled is the default.
+            if ("enabled" in options && !options.enabled) {
+              log.info("Marking add-on as disabled for install: " +
+                       install.name);
+              install.addon.userDisabled = true;
+            }
+          },
+          onInstallEnded: function(install, addon) {
+            install.removeListener(listener);
+
+            cb(null, {id: addon.id, install: install, addon: addon});
+          },
+          onInstallFailed: function(install) {
+            install.removeListener(listener);
+
+            cb(new Error("Install failed: " + install.error), null);
+          },
+          onDownloadFailed: function(install) {
+            install.removeListener(listener);
+
+            cb(new Error("Download failed: " + install.error), null);
+          }
+        };
+        install.addListener(listener);
+        install.install();
+      }
+      catch (ex) {
+        this._log.error("Error installing add-on", ex);
+        cb(ex, null);
+      }
+    }.bind(this));
+  },
+
+  /**
+   * Uninstalls the Addon instance and invoke a callback when it is done.
+   *
+   * @param addon
+   *        Addon instance to uninstall.
+   * @param cb
+   *        Function to be invoked when uninstall has finished. It receives a
+   *        truthy value signifying error and the add-on which was uninstalled.
+   */
+  uninstallAddon: function uninstallAddon(addon, cb) {
+    let listener = {
+      onUninstalling: function(uninstalling, needsRestart) {
+        if (addon.id != uninstalling.id) {
+          return;
+        }
+
+        // We assume restartless add-ons will send the onUninstalled event
+        // soon.
+        if (!needsRestart) {
+          return;
+        }
+
+        // For non-restartless add-ons, we issue the callback on uninstalling
+        // because we will likely never see the uninstalled event.
+        AddonManager.removeAddonListener(listener);
+        cb(null, addon);
+      },
+      onUninstalled: function(uninstalled) {
+        if (addon.id != uninstalled.id) {
+          return;
+        }
+
+        AddonManager.removeAddonListener(listener);
+        cb(null, addon);
+      }
+    };
+    AddonManager.addAddonListener(listener);
+    addon.uninstall();
+  },
+
+  /**
+   * Installs multiple add-ons specified by metadata.
+   *
+   * The first argument is an array of objects. Each object must have the
+   * following keys:
+   *
+   *   id - public ID of the add-on to install.
+   *   syncGUID - syncGUID for new add-on.
+   *   enabled - boolean indicating whether the add-on should be enabled.
+   *   requireSecureURI - Boolean indicating whether to require a secure
+   *     URI when installing from a remote location. This defaults to
+   *     true.
+   *
+   * The callback will be called when activity on all add-ons is complete. The
+   * callback receives 2 arguments, error and result.
+   *
+   * If error is truthy, it contains a string describing the overall error.
+   *
+   * The 2nd argument to the callback is always an object with details on the
+   * overall execution state. It contains the following keys:
+   *
+   *   installedIDs  Array of add-on IDs that were installed.
+   *   installs      Array of AddonInstall instances that were installed.
+   *   addons        Array of Addon instances that were installed.
+   *   errors        Array of errors encountered. Only has elements if error is
+   *                 truthy.
+   *
+   * @param installs
+   *        Array of objects describing add-ons to install.
+   * @param cb
+   *        Function to be called when all actions are complete.
+   */
+  installAddons: function installAddons(installs, cb) {
+    if (!cb) {
+      throw new Error("Invalid argument: cb is not defined.");
+    }
+
+    let ids = [];
+    for (let addon of installs) {
+      ids.push(addon.id);
+    }
+
+    AddonRepository.getAddonsByIDs(ids, {
+      searchSucceeded: function searchSucceeded(addons, addonsLength, total) {
+        this._log.info("Found " + addonsLength + "/" + ids.length +
+                       " add-ons during repository search.");
+
+        let ourResult = {
+          installedIDs: [],
+          installs:     [],
+          addons:       [],
+          skipped:      [],
+          errors:       []
+        };
+
+        if (!addonsLength) {
+          cb(null, ourResult);
+          return;
+        }
+
+        let expectedInstallCount = 0;
+        let finishedCount = 0;
+        let installCallback = function installCallback(error, result) {
+          finishedCount++;
+
+          if (error) {
+            ourResult.errors.push(error);
+          } else {
+            ourResult.installedIDs.push(result.id);
+            ourResult.installs.push(result.install);
+            ourResult.addons.push(result.addon);
+          }
+
+          if (finishedCount >= expectedInstallCount) {
+            if (ourResult.errors.length > 0) {
+              cb(new Error("1 or more add-ons failed to install"), ourResult);
+            } else {
+              cb(null, ourResult);
+            }
+          }
+        }.bind(this);
+
+        let toInstall = [];
+
+        // Rewrite the "src" query string parameter of the source URI to note
+        // that the add-on was installed by Sync and not something else so
+        // server-side metrics aren't skewed (bug 708134). The server should
+        // ideally send proper URLs, but this solution was deemed too
+        // complicated at the time the functionality was implemented.
+        for (let addon of addons) {
+          // Find the specified options for this addon.
+          let options;
+          for (let install of installs) {
+            if (install.id == addon.id) {
+              options = install;
+              break;
+            }
+          }
+          if (!this.canInstallAddon(addon, options)) {
+            ourResult.skipped.push(addon.id);
+            continue;
+          }
+
+          // We can go ahead and attempt to install it.
+          toInstall.push(addon);
+
+          // We should always be able to QI the nsIURI to nsIURL. If not, we
+          // still try to install the add-on, but we don't rewrite the URL,
+          // potentially skewing metrics.
+          try {
+            addon.sourceURI.QueryInterface(Ci.nsIURL);
+          } catch (ex) {
+            this._log.warn("Unable to QI sourceURI to nsIURL: " +
+                           addon.sourceURI.spec);
+            continue;
+          }
+
+          let params = addon.sourceURI.query.split("&").map(
+            function rewrite(param) {
+
+            if (param.indexOf("src=") == 0) {
+              return "src=sync";
+            } else {
+              return param;
+            }
+          });
+
+          addon.sourceURI.query = params.join("&");
+        }
+
+        expectedInstallCount = toInstall.length;
+
+        if (!expectedInstallCount) {
+          cb(null, ourResult);
+          return;
+        }
+
+        // Start all the installs asynchronously. They will report back to us
+        // as they finish, eventually triggering the global callback.
+        for (let addon of toInstall) {
+          let options = {};
+          for (let install of installs) {
+            if (install.id == addon.id) {
+              options = install;
+              break;
+            }
+          }
+
+          this.installAddonFromSearchResult(addon, options, installCallback);
+        }
+
+      }.bind(this),
+
+      searchFailed: function searchFailed() {
+        cb(new Error("AddonRepository search failed"), null);
+      },
+    });
+  },
+
+  /**
+   * Returns true if we are able to install the specified addon, false
+   * otherwise. It is expected that this will log the reason if it returns
+   * false.
+   *
+   * @param addon
+   *        (Addon) Add-on instance to check.
+   * @param options
+   *        (object) The options specified for this addon. See installAddons()
+   *        for the valid elements.
+   */
+  canInstallAddon(addon, options) {
+    // sourceURI presence isn't enforced by AddonRepository. So, we skip
+    // add-ons without a sourceURI.
+    if (!addon.sourceURI) {
+      this._log.info("Skipping install of add-on because missing " +
+                     "sourceURI: " + addon.id);
+      return false;
+    }
+    // Verify that the source URI uses TLS. We don't allow installs from
+    // insecure sources for security reasons. The Addon Manager ensures
+    // that cert validation etc is performed.
+    // (We should also consider just dropping this entirely and calling
+    // XPIProvider.isInstallAllowed, but that has additional semantics we might
+    // need to think through...)
+    let requireSecureURI = true;
+    if (options && options.requireSecureURI !== undefined) {
+      requireSecureURI = options.requireSecureURI;
+    }
+
+    if (requireSecureURI) {
+      let scheme = addon.sourceURI.scheme;
+      if (scheme != "https") {
+        this._log.info(`Skipping install of add-on "${addon.id}" because sourceURI's scheme of "${scheme}" is not trusted`);
+        return false;
+      }
+    }
+    this._log.info(`Add-on "${addon.id}" is able to be installed`);
+    return true;
+  },
+
+
+  /**
+   * Update the user disabled flag for an add-on.
+   *
+   * The supplied callback will be called when the operation is
+   * complete. If the new flag matches the existing or if the add-on
+   * isn't currently active, the function will fire the callback
+   * immediately. Else, the callback is invoked when the AddonManager
+   * reports the change has taken effect or has been registered.
+   *
+   * The callback receives as arguments:
+   *
+   *   (Error) Encountered error during operation or null on success.
+   *   (Addon) The add-on instance being operated on.
+   *
+   * @param addon
+   *        (Addon) Add-on instance to operate on.
+   * @param value
+   *        (bool) New value for add-on's userDisabled property.
+   * @param cb
+   *        (function) Callback to be invoked on completion.
+   */
+  updateUserDisabled: function updateUserDisabled(addon, value, cb) {
+    if (addon.userDisabled == value) {
+      cb(null, addon);
+      return;
+    }
+
+    let listener = {
+      onEnabling: function onEnabling(wrapper, needsRestart) {
+        this._log.debug("onEnabling: " + wrapper.id);
+        if (wrapper.id != addon.id) {
+          return;
+        }
+
+        // We ignore the restartless case because we'll get onEnabled shortly.
+        if (!needsRestart) {
+          return;
+        }
+
+        AddonManager.removeAddonListener(listener);
+        cb(null, wrapper);
+      }.bind(this),
+
+      onEnabled: function onEnabled(wrapper) {
+        this._log.debug("onEnabled: " + wrapper.id);
+        if (wrapper.id != addon.id) {
+          return;
+        }
+
+        AddonManager.removeAddonListener(listener);
+        cb(null, wrapper);
+      }.bind(this),
+
+      onDisabling: function onDisabling(wrapper, needsRestart) {
+        this._log.debug("onDisabling: " + wrapper.id);
+        if (wrapper.id != addon.id) {
+          return;
+        }
+
+        if (!needsRestart) {
+          return;
+        }
+
+        AddonManager.removeAddonListener(listener);
+        cb(null, wrapper);
+      }.bind(this),
+
+      onDisabled: function onDisabled(wrapper) {
+        this._log.debug("onDisabled: " + wrapper.id);
+        if (wrapper.id != addon.id) {
+          return;
+        }
+
+        AddonManager.removeAddonListener(listener);
+        cb(null, wrapper);
+      }.bind(this),
+
+      onOperationCancelled: function onOperationCancelled(wrapper) {
+        this._log.debug("onOperationCancelled: " + wrapper.id);
+        if (wrapper.id != addon.id) {
+          return;
+        }
+
+        AddonManager.removeAddonListener(listener);
+        cb(new Error("Operation cancelled"), wrapper);
+      }.bind(this)
+    };
+
+    // The add-on listeners are only fired if the add-on is active. If not, the
+    // change is silently updated and made active when/if the add-on is active.
+
+    if (!addon.appDisabled) {
+      AddonManager.addAddonListener(listener);
+    }
+
+    this._log.info("Updating userDisabled flag: " + addon.id + " -> " + value);
+    addon.userDisabled = !!value;
+
+    if (!addon.appDisabled) {
+      cb(null, addon);
+      return;
+    }
+    // Else the listener will handle invoking the callback.
+  },
+
+};
+
+XPCOMUtils.defineLazyGetter(this, "AddonUtils", function() {
+  return new AddonUtilsInternal();
+});

services/sync/modules/bookmark_validator.js

@@ -0,0 +1,784 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+
+this.EXPORTED_SYMBOLS = ["BookmarkValidator", "BookmarkProblemData"];
+
+const LEFT_PANE_ROOT_ANNO = "PlacesOrganizer/OrganizerFolder";
+const LEFT_PANE_QUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
+
+// Indicates if a local bookmark tree node should be excluded from syncing.
+function isNodeIgnored(treeNode) {
+  return treeNode.annos && treeNode.annos.some(anno => anno.name == LEFT_PANE_ROOT_ANNO ||
+                                                       anno.name == LEFT_PANE_QUERY_ANNO);
+}
+const BOOKMARK_VALIDATOR_VERSION = 1;
+
+/**
+ * Result of bookmark validation. Contains the following fields which describe
+ * server-side problems unless otherwise specified.
+ *
+ * - missingIDs (number): # of objects with missing ids
+ * - duplicates (array of ids): ids seen more than once
+ * - parentChildMismatches (array of {parent: parentid, child: childid}):
+ *   instances where the child's parentid and the parent's children array
+ *   do not match
+ * - cycles (array of array of ids). List of cycles found in the server-side tree.
+ * - clientCycles (array of array of ids). List of cycles found in the client-side tree.
+ * - orphans (array of {id: string, parent: string}): List of nodes with
+ *   either no parentid, or where the parent could not be found.
+ * - missingChildren (array of {parent: id, child: id}):
+ *   List of parent/children where the child id couldn't be found
+ * - deletedChildren (array of { parent: id, child: id }):
+ *   List of parent/children where child id was a deleted item (but still showed up
+ *   in the children array)
+ * - multipleParents (array of {child: id, parents: array of ids}):
+ *   List of children that were part of multiple parent arrays
+ * - deletedParents (array of ids) : List of records that aren't deleted but
+ *   had deleted parents
+ * - childrenOnNonFolder (array of ids): list of non-folders that still have
+ *   children arrays
+ * - duplicateChildren (array of ids): list of records who have the same
+ *   child listed multiple times in their children array
+ * - parentNotFolder (array of ids): list of records that have parents that
+ *   aren't folders
+ * - rootOnServer (boolean): true if the root came from the server
+ * - badClientRoots (array of ids): Contains any client-side root ids where
+ *   the root is missing or isn't a (direct) child of the places root.
+ *
+ * - clientMissing: Array of ids on the server missing from the client
+ * - serverMissing: Array of ids on the client missing from the server
+ * - serverDeleted: Array of ids on the client that the server had marked as deleted.
+ * - serverUnexpected: Array of ids that appear on the server but shouldn't
+ *   because the client attempts to never upload them.
+ * - differences: Array of {id: string, differences: string array} recording
+ *   the non-structural properties that are differente between the client and server
+ * - structuralDifferences: As above, but contains the items where the differences were
+ *   structural, that is, they contained childGUIDs or parentid
+ */
+class BookmarkProblemData {
+  constructor() {
+    this.rootOnServer = false;
+    this.missingIDs = 0;
+
+    this.duplicates = [];
+    this.parentChildMismatches = [];
+    this.cycles = [];
+    this.clientCycles = [];
+    this.orphans = [];
+    this.missingChildren = [];
+    this.deletedChildren = [];
+    this.multipleParents = [];
+    this.deletedParents = [];
+    this.childrenOnNonFolder = [];
+    this.duplicateChildren = [];
+    this.parentNotFolder = [];
+
+    this.badClientRoots = [];
+    this.clientMissing = [];
+    this.serverMissing = [];
+    this.serverDeleted = [];
+    this.serverUnexpected = [];
+    this.differences = [];
+    this.structuralDifferences = [];
+  }
+
+  /**
+   * Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into
+   * [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc.
+   */
+  _summarizeDifferences(prefix, diffs) {
+    let diffCounts = new Map();
+    for (let { differences } of diffs) {
+      for (let type of differences) {
+        let name = prefix + ":" + type;
+        let count = diffCounts.get(name) || 0;
+        diffCounts.set(name, count + 1);
+      }
+    }
+    return [...diffCounts].map(([name, count]) => ({ name, count }));
+  }
+
+  /**
+   * Produce a list summarizing problems found. Each entry contains {name, count},
+   * where name is the field name for the problem, and count is the number of times
+   * the problem was encountered.
+   *
+   * Validation has failed if all counts are not 0.
+   *
+   * If the `full` argument is truthy, we also include information about which
+   * properties we saw structural differences in. Currently, this means either
+   * "sdiff:parentid" and "sdiff:childGUIDS" may be present.
+   */
+  getSummary(full) {
+    let result = [
+      { name: "clientMissing", count: this.clientMissing.length },
+      { name: "serverMissing", count: this.serverMissing.length },
+      { name: "serverDeleted", count: this.serverDeleted.length },
+      { name: "serverUnexpected", count: this.serverUnexpected.length },
+
+      { name: "structuralDifferences", count: this.structuralDifferences.length },
+      { name: "differences", count: this.differences.length },
+
+      { name: "missingIDs", count: this.missingIDs },
+      { name: "rootOnServer", count: this.rootOnServer ? 1 : 0 },
+
+      { name: "duplicates", count: this.duplicates.length },
+      { name: "parentChildMismatches", count: this.parentChildMismatches.length },
+      { name: "cycles", count: this.cycles.length },
+      { name: "clientCycles", count: this.clientCycles.length },
+      { name: "badClientRoots", count: this.badClientRoots.length },
+      { name: "orphans", count: this.orphans.length },
+      { name: "missingChildren", count: this.missingChildren.length },
+      { name: "deletedChildren", count: this.deletedChildren.length },
+      { name: "multipleParents", count: this.multipleParents.length },
+      { name: "deletedParents", count: this.deletedParents.length },
+      { name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length },
+      { name: "duplicateChildren", count: this.duplicateChildren.length },
+      { name: "parentNotFolder", count: this.parentNotFolder.length },
+    ];
+    if (full) {
+      let structural = this._summarizeDifferences("sdiff", this.structuralDifferences);
+      result.push.apply(result, structural);
+    }
+    return result;
+  }
+}
+
+// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon.
+XPCOMUtils.defineLazyGetter(this, "SYNCED_ROOTS", () => [
+  PlacesUtils.bookmarks.menuGuid,
+  PlacesUtils.bookmarks.toolbarGuid,
+  PlacesUtils.bookmarks.unfiledGuid,
+  PlacesUtils.bookmarks.mobileGuid,
+]);
+
+class BookmarkValidator {
+
+  _followQueries(recordMap) {
+    for (let [guid, entry] of recordMap) {
+      if (entry.type !== "query" && (!entry.bmkUri || !entry.bmkUri.startsWith("place:"))) {
+        continue;
+      }
+      // Might be worth trying to parse the place: query instead so that this
+      // works "automatically" with things like aboutsync.
+      let queryNodeParent = PlacesUtils.getFolderContents(entry, false, true);
+      if (!queryNodeParent || !queryNodeParent.root.hasChildren) {
+        continue;
+      }
+      queryNodeParent = queryNodeParent.root;
+      let queryNode = null;
+      let numSiblings = 0;
+      let containerWasOpen = queryNodeParent.containerOpen;
+      queryNodeParent.containerOpen = true;
+      try {
+        try {
+          numSiblings = queryNodeParent.childCount;
+        } catch (e) {
+          // This throws when we can't actually get the children. This is the
+          // case for history containers, tag queries, ...
+          continue;
+        }
+        for (let i = 0; i < numSiblings && !queryNode; ++i) {
+          let child = queryNodeParent.getChild(i);
+          if (child && child.bookmarkGuid && child.bookmarkGuid === guid) {
+            queryNode = child;
+          }
+        }
+      } finally {
+        queryNodeParent.containerOpen = containerWasOpen;
+      }
+      if (!queryNode) {
+        continue;
+      }
+
+      let concreteId = PlacesUtils.getConcreteItemGuid(queryNode);
+      if (!concreteId) {
+        continue;
+      }
+      let concreteItem = recordMap.get(concreteId);
+      if (!concreteItem) {
+        continue;
+      }
+      entry.concrete = concreteItem;
+    }
+  }
+
+  createClientRecordsFromTree(clientTree) {
+    // Iterate over the treeNode, converting it to something more similar to what
+    // the server stores.
+    let records = [];
+    let recordsByGuid = new Map();
+    let syncedRoots = SYNCED_ROOTS;
+    function traverse(treeNode, synced) {
+      if (!synced) {
+        synced = syncedRoots.includes(treeNode.guid);
+      } else if (isNodeIgnored(treeNode)) {
+        synced = false;
+      }
+      let guid = PlacesSyncUtils.bookmarks.guidToSyncId(treeNode.guid);
+      let itemType = 'item';
+      treeNode.ignored = !synced;
+      treeNode.id = guid;
+      switch (treeNode.type) {
+        case PlacesUtils.TYPE_X_MOZ_PLACE:
+          let query = null;
+          if (treeNode.annos && treeNode.uri.startsWith("place:")) {
+            query = treeNode.annos.find(({name}) =>
+              name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO);
+          }
+          if (query && query.value) {
+            itemType = 'query';
+          } else {
+            itemType = 'bookmark';
+          }
+          break;
+        case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
+          let isLivemark = false;
+          if (treeNode.annos) {
+            for (let anno of treeNode.annos) {
+              if (anno.name === PlacesUtils.LMANNO_FEEDURI) {
+                isLivemark = true;
+                treeNode.feedUri = anno.value;
+              } else if (anno.name === PlacesUtils.LMANNO_SITEURI) {
+                isLivemark = true;
+                treeNode.siteUri = anno.value;
+              }
+            }
+          }
+          itemType = isLivemark ? "livemark" : "folder";
+          break;
+        case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
+          itemType = 'separator';
+          break;
+      }
+
+      if (treeNode.tags) {
+        treeNode.tags = treeNode.tags.split(",");
+      } else {
+        treeNode.tags = [];
+      }
+      treeNode.type = itemType;
+      treeNode.pos = treeNode.index;
+      treeNode.bmkUri = treeNode.uri;
+      records.push(treeNode);
+      // We want to use the "real" guid here.
+      recordsByGuid.set(treeNode.guid, treeNode);
+      if (treeNode.type === 'folder') {
+        treeNode.childGUIDs = [];
+        if (!treeNode.children) {
+          treeNode.children = [];
+        }
+        for (let child of treeNode.children) {
+          traverse(child, synced);
+          child.parent = treeNode;
+          child.parentid = guid;
+          treeNode.childGUIDs.push(child.guid);
+        }
+      }
+    }
+    traverse(clientTree, false);
+    clientTree.id = 'places';
+    this._followQueries(recordsByGuid);
+    return records;
+  }
+
+  /**
+   * Process the server-side list. Mainly this builds the records into a tree,
+   * but it also records information about problems, and produces arrays of the
+   * deleted and non-deleted nodes.
+   *
+   * Returns an object containing:
+   * - records:Array of non-deleted records. Each record contains the following
+   *    properties
+   *     - childGUIDs (array of strings, only present if type is 'folder'): the
+   *       list of child GUIDs stored on the server.
+   *     - children (array of records, only present if type is 'folder'):
+   *       each record has these same properties. This may differ in content
+   *       from what you may expect from the childGUIDs list, as it won't
+   *       contain any records that could not be found.
+   *     - parent (record): The parent to this record.
+   *     - Unchanged properties send down from the server: id, title, type,
+   *       parentName, parentid, bmkURI, keyword, tags, pos, queryId, loadInSidebar
+   * - root: Root of the server-side bookmark tree. Has the same properties as
+   *   above.
+   * - deletedRecords: As above, but only contains items that the server sent
+   *   where it also sent indication that the item should be deleted.
+   * - problemData: a BookmarkProblemData object, with the caveat that
+   *   the fields describing client/server relationship will not have been filled
+   *   out yet.
+   */
+  inspectServerRecords(serverRecords) {
+    let deletedItemIds = new Set();
+    let idToRecord = new Map();
+    let deletedRecords = [];
+
+    let folders = [];
+    let problems = [];
+
+    let problemData = new BookmarkProblemData();
+
+    let resultRecords = [];
+
+    for (let record of serverRecords) {
+      if (!record.id) {
+        ++problemData.missingIDs;
+        continue;
+      }
+      if (record.deleted) {
+        deletedItemIds.add(record.id);
+      } else {
+        if (idToRecord.has(record.id)) {
+          problemData.duplicates.push(record.id);
+          continue;
+        }
+      }
+      idToRecord.set(record.id, record);
+
+      if (record.children) {
+        if (record.type !== "folder") {
+          // Due to implementation details in engines/bookmarks.js, (Livemark
+          // subclassing BookmarkFolder) Livemarks will have a children array,
+          // but it should still be empty.
+          if (!record.children.length) {
+            continue;
+          }
+          // Otherwise we mark it as an error and still try to resolve the children
+          problemData.childrenOnNonFolder.push(record.id);
+        }
+        folders.push(record);
+
+        if (new Set(record.children).size !== record.children.length) {
+          problemData.duplicateChildren.push(record.id)
+        }
+
+        // The children array stores special guids as their local guid values,
+        // e.g. 'menu________' instead of 'menu', but all other parts of the
+        // serverside bookmark info stores it as the special value ('menu').
+        record.childGUIDs = record.children;
+        record.children = record.children.map(childID => {
+          return PlacesSyncUtils.bookmarks.guidToSyncId(childID);
+        });
+      }
+    }
+
+    for (let deletedId of deletedItemIds) {
+      let record = idToRecord.get(deletedId);
+      if (record && !record.isDeleted) {
+        deletedRecords.push(record);
+        record.isDeleted = true;
+      }
+    }
+
+    let root = idToRecord.get('places');
+
+    if (!root) {
+      // Fabricate a root. We want to remember that it's fake so that we can
+      // avoid complaining about stuff like it missing it's childGUIDs later.
+      root = { id: 'places', children: [], type: 'folder', title: '', fake: true };
+      resultRecords.push(root);
+      idToRecord.set('places', root);
+    } else {
+      problemData.rootOnServer = true;
+    }
+
+    // Build the tree, find orphans, and record most problems having to do with
+    // the tree structure.
+    for (let [id, record] of idToRecord) {
+      if (record === root) {
+        continue;
+      }
+
+      if (record.isDeleted) {
+        continue;
+      }
+
+      let parentID = record.parentid;
+      if (!parentID) {
+        problemData.orphans.push({id: record.id, parent: parentID});
+        continue;
+      }
+
+      let parent = idToRecord.get(parentID);
+      if (!parent) {
+        problemData.orphans.push({id: record.id, parent: parentID});
+        continue;
+      }
+
+      if (parent.type !== 'folder') {
+        problemData.parentNotFolder.push(record.id);
+        if (!parent.children) {
+          parent.children = [];
+        }
+        if (!parent.childGUIDs) {
+          parent.childGUIDs = [];
+        }
+      }
+
+      if (!record.isDeleted) {
+        resultRecords.push(record);
+      }
+
+      record.parent = parent;
+      if (parent !== root || problemData.rootOnServer) {
+        let childIndex = parent.children.indexOf(id);
+        if (childIndex < 0) {
+          problemData.parentChildMismatches.push({parent: parent.id, child: record.id});
+        } else {
+          parent.children[childIndex] = record;
+        }
+      } else {
+        parent.children.push(record);
+      }
+
+      if (parent.isDeleted && !record.isDeleted) {
+        problemData.deletedParents.push(record.id);
+      }
+
+      // We used to check if the parentName on the server matches the actual
+      // local parent name, but given this is used only for de-duping a record
+      // the first time it is seen and expensive to keep up-to-date, we decided
+      // to just stop recording it. See bug 1276969 for more.
+    }
+
+    // Check that we aren't missing any children.
+    for (let folder of folders) {
+      folder.unfilteredChildren = folder.children;
+      folder.children = [];
+      for (let ci = 0; ci < folder.unfilteredChildren.length; ++ci) {
+        let child = folder.unfilteredChildren[ci];
+        let childObject;
+        if (typeof child == "string") {
+          // This can happen the parent refers to a child that has a different
+          // parentid, or if it refers to a missing or deleted child. It shouldn't
+          // be possible with totally valid bookmarks.
+          childObject = idToRecord.get(child);
+          if (!childObject) {
+            problemData.missingChildren.push({parent: folder.id, child});
+          } else {
+            folder.unfilteredChildren[ci] = childObject;
+            if (childObject.isDeleted) {
+              problemData.deletedChildren.push({ parent: folder.id, child });
+            }
+          }
+        } else {
+          childObject = child;
+        }
+
+        if (!childObject) {
+          continue;
+        }
+
+        if (childObject.parentid === folder.id) {
+          folder.children.push(childObject);
+          continue;
+        }
+
+        // The child is very probably in multiple `children` arrays --
+        // see if we already have a problem record about it.
+        let currentProblemRecord = problemData.multipleParents.find(pr =>
+          pr.child === child);
+
+        if (currentProblemRecord) {
+          currentProblemRecord.parents.push(folder.id);
+          continue;
+        }
+
+        let otherParent = idToRecord.get(childObject.parentid);
+        // it's really an ... orphan ... sort of.
+        if (!otherParent) {
+          // if we never end up adding to this parent's list, we filter it out after this loop.
+          problemData.multipleParents.push({
+            child,
+            parents: [folder.id]
+          });
+          if (!problemData.orphans.some(r => r.id === child)) {
+            problemData.orphans.push({
+              id: child,
+              parent: childObject.parentid
+            });
+          }
+          continue;
+        }
+
+        if (otherParent.isDeleted) {
+          if (!problemData.deletedParents.includes(child)) {
+            problemData.deletedParents.push(child);
+          }
+          continue;
+        }
+
+        if (otherParent.childGUIDs && !otherParent.childGUIDs.includes(child)) {
+          if (!problemData.parentChildMismatches.some(r => r.child === child)) {
+            // Might not be possible to get here.
+            problemData.parentChildMismatches.push({ child, parent: folder.id });
+          }
+        }
+
+        problemData.multipleParents.push({
+          child,
+          parents: [childObject.parentid, folder.id]
+        });
+      }
+    }
+    problemData.multipleParents = problemData.multipleParents.filter(record =>
+      record.parents.length >= 2);
+
+    problemData.cycles = this._detectCycles(resultRecords);
+
+    return {
+      deletedRecords,
+      records: resultRecords,
+      problemData,
+      root,
+    };
+  }
+
+  // helper for inspectServerRecords
+  _detectCycles(records) {
+    // currentPath and pathLookup contain the same data. pathLookup is faster to
+    // query, but currentPath gives is the order of traversal that we need in
+    // order to report the members of the cycles.
+    let pathLookup = new Set();
+    let currentPath = [];
+    let cycles = [];
+    let seenEver = new Set();
+    const traverse = node => {
+      if (pathLookup.has(node)) {
+        let cycleStart = currentPath.lastIndexOf(node);
+        let cyclePath = currentPath.slice(cycleStart).map(n => n.id);
+        cycles.push(cyclePath);
+        return;
+      } else if (seenEver.has(node)) {
+        // If we're checking the server, this is a problem, but it should already be reported.
+        // On the client, this could happen due to including `node.concrete` in the child list.
+        return;
+      }
+      seenEver.add(node);
+      let children = node.children || [];
+      if (node.concrete) {
+        children.push(node.concrete);
+      }
+      if (children) {
+        pathLookup.add(node);
+        currentPath.push(node);
+        for (let child of children) {
+          traverse(child);
+        }
+        currentPath.pop();
+        pathLookup.delete(node);
+      }
+    };
+    for (let record of records) {
+      if (!seenEver.has(record)) {
+        traverse(record);
+      }
+    }
+
+    return cycles;
+  }
+
+  // Perform client-side sanity checking that doesn't involve server data
+  _validateClient(problemData, clientRecords) {
+    problemData.clientCycles = this._detectCycles(clientRecords);
+    for (let rootGUID of SYNCED_ROOTS) {
+      let record = clientRecords.find(record =>
+        record.guid === rootGUID);
+      if (!record || record.parentid !== "places") {
+        problemData.badClientRoots.push(rootGUID);
+      }
+    }
+  }
+
+  /**
+   * Compare the list of server records with the client tree.
+   *
+   * Returns the same data as described in the inspectServerRecords comment,
+   * with the following additional fields.
+   * - clientRecords: an array of client records in a similar format to
+   *   the .records (ie, server records) entry.
+   * - problemData is the same as for inspectServerRecords, except all properties
+   *   will be filled out.
+   */
+  compareServerWithClient(serverRecords, clientTree) {
+
+    let clientRecords = this.createClientRecordsFromTree(clientTree);
+    let inspectionInfo = this.inspectServerRecords(serverRecords);
+    inspectionInfo.clientRecords = clientRecords;
+
+    // Mainly do this to remove deleted items and normalize child guids.
+    serverRecords = inspectionInfo.records;
+    let problemData = inspectionInfo.problemData;
+
+    this._validateClient(problemData, clientRecords);
+
+    let matches = [];
+
+    let allRecords = new Map();
+    let serverDeletedLookup = new Set(inspectionInfo.deletedRecords.map(r => r.id));
+
+    for (let sr of serverRecords) {
+      if (sr.fake) {
+        continue;
+      }
+      allRecords.set(sr.id, {client: null, server: sr});
+    }
+
+    for (let cr of clientRecords) {
+      let unified = allRecords.get(cr.id);
+      if (!unified) {
+        allRecords.set(cr.id, {client: cr, server: null});
+      } else {
+        unified.client = cr;
+      }
+    }
+
+
+    for (let [id, {client, server}] of allRecords) {
+      if (!client && server) {
+        problemData.clientMissing.push(id);
+        continue;
+      }
+      if (!server && client) {
+        if (serverDeletedLookup.has(id)) {
+          problemData.serverDeleted.push(id);
+        } else if (!client.ignored && client.id != "places") {
+          problemData.serverMissing.push(id);
+        }
+        continue;
+      }
+      if (server && client && client.ignored) {
+        problemData.serverUnexpected.push(id);
+      }
+      let differences = [];
+      let structuralDifferences = [];
+
+      // Don't bother comparing titles of roots. It's okay if locally it's
+      // "Mobile Bookmarks", but the server thinks it's "mobile".
+      // TODO: We probably should be handing other localized bookmarks (e.g.
+      // default bookmarks) here as well, see bug 1316041.
+      if (!SYNCED_ROOTS.includes(client.guid)) {
+        // We want to treat undefined, null and an empty string as identical
+        if ((client.title || "") !== (server.title || "")) {
+          differences.push("title");
+        }
+      }
+
+      if (client.parentid || server.parentid) {
+        if (client.parentid !== server.parentid) {
+          structuralDifferences.push('parentid');
+        }
+      }
+
+      if (client.tags || server.tags) {
+        let cl = client.tags || [];
+        let sl = server.tags || [];
+        if (cl.length !== sl.length || !cl.every((tag, i) => sl.indexOf(tag) >= 0)) {
+          differences.push('tags');
+        }
+      }
+
+      let sameType = client.type === server.type;
+      if (!sameType) {
+        if (server.type === "query" && client.type === "bookmark" && client.bmkUri.startsWith("place:")) {
+          sameType = true;
+        }
+      }
+
+
+      if (!sameType) {
+        differences.push('type');
+      } else {
+        switch (server.type) {
+          case 'bookmark':
+          case 'query':
+            if (server.bmkUri !== client.bmkUri) {
+              differences.push('bmkUri');
+            }
+            break;
+          case "livemark":
+            if (server.feedUri != client.feedUri) {
+              differences.push("feedUri");
+            }
+            if (server.siteUri != client.siteUri) {
+              differences.push("siteUri");
+            }
+            break;
+          case 'folder':
+            if (server.id === 'places' && !problemData.rootOnServer) {
+              // It's the fabricated places root. It won't have the GUIDs, but
+              // it doesn't matter.
+              break;
+            }
+            if (client.childGUIDs || server.childGUIDs) {
+              let cl = client.childGUIDs || [];
+              let sl = server.childGUIDs || [];
+              if (cl.length !== sl.length || !cl.every((id, i) => sl[i] === id)) {
+                structuralDifferences.push('childGUIDs');
+              }
+            }
+            break;
+        }
+      }
+
+      if (differences.length) {
+        problemData.differences.push({id, differences});
+      }
+      if (structuralDifferences.length) {
+        problemData.structuralDifferences.push({ id, differences: structuralDifferences });
+      }
+    }
+    return inspectionInfo;
+  }
+
+  _getServerState(engine) {
+    let collection = engine.itemSource();
+    let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name);
+    collection.full = true;
+    let items = [];
+    collection.recordHandler = function(item) {
+      item.decrypt(collectionKey);
+      items.push(item.cleartext);
+    };
+    let resp = collection.getBatched();
+    if (!resp.success) {
+      throw resp;
+    }
+    return items;
+  }
+
+  validate(engine) {
+    let self = this;
+    return Task.spawn(function*() {
+      let start = Date.now();
+      let clientTree = yield PlacesUtils.promiseBookmarksTree("", {
+        includeItemIds: true
+      });
+      let serverState = self._getServerState(engine);
+      let serverRecordCount = serverState.length;
+      let result = self.compareServerWithClient(serverState, clientTree);
+      let end = Date.now();
+      let duration = end-start;
+      return {
+        duration,
+        version: self.version,
+        problems: result.problemData,
+        recordCount: serverRecordCount
+      };
+    });
+  }
+
+};
+
+BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION;
+

services/sync/modules/browserid_identity.js

@@ -0,0 +1,869 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["BrowserIDManager", "AuthenticationError"];
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-common/tokenserverclient.js");
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/tokenserverclient.js");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://services-sync/stages/cluster.js");
+Cu.import("resource://gre/modules/FxAccounts.jsm");
+
+// Lazy imports to prevent unnecessary load on startup.
+XPCOMUtils.defineLazyModuleGetter(this, "Weave",
+                                  "resource://services-sync/main.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "BulkKeyBundle",
+                                  "resource://services-sync/keys.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
+                                  "resource://gre/modules/FxAccounts.jsm");
+
+XPCOMUtils.defineLazyGetter(this, 'log', function() {
+  let log = Log.repository.getLogger("Sync.BrowserIDManager");
+  log.level = Log.Level[Svc.Prefs.get("log.logger.identity")] || Log.Level.Error;
+  return log;
+});
+
+// FxAccountsCommon.js doesn't use a "namespace", so create one here.
+var fxAccountsCommon = {};
+Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
+
+const OBSERVER_TOPICS = [
+  fxAccountsCommon.ONLOGIN_NOTIFICATION,
+  fxAccountsCommon.ONLOGOUT_NOTIFICATION,
+  fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION,
+];
+
+const PREF_SYNC_SHOW_CUSTOMIZATION = "services.sync-setup.ui.showCustomizationDialog";
+
+function deriveKeyBundle(kB) {
+  let out = CryptoUtils.hkdf(kB, undefined,
+                             "identity.mozilla.com/picl/v1/oldsync", 2*32);
+  let bundle = new BulkKeyBundle();
+  // [encryptionKey, hmacKey]
+  bundle.keyPair = [out.slice(0, 32), out.slice(32, 64)];
+  return bundle;
+}
+
+/*
+  General authentication error for abstracting authentication
+  errors from multiple sources (e.g., from FxAccounts, TokenServer).
+  details is additional details about the error - it might be a string, or
+  some other error object (which should do the right thing when toString() is
+  called on it)
+*/
+function AuthenticationError(details, source) {
+  this.details = details;
+  this.source = source;
+}
+
+AuthenticationError.prototype = {
+  toString: function() {
+    return "AuthenticationError(" + this.details + ")";
+  }
+}
+
+this.BrowserIDManager = function BrowserIDManager() {
+  // NOTE: _fxaService and _tokenServerClient are replaced with mocks by
+  // the test suite.
+  this._fxaService = fxAccounts;
+  this._tokenServerClient = new TokenServerClient();
+  this._tokenServerClient.observerPrefix = "weave:service";
+  // will be a promise that resolves when we are ready to authenticate
+  this.whenReadyToAuthenticate = null;
+  this._log = log;
+};
+
+this.BrowserIDManager.prototype = {
+  __proto__: IdentityManager.prototype,
+
+  _fxaService: null,
+  _tokenServerClient: null,
+  // https://docs.services.mozilla.com/token/apis.html
+  _token: null,
+  _signedInUser: null, // the signedinuser we got from FxAccounts.
+
+  // null if no error, otherwise a LOGIN_FAILED_* value that indicates why
+  // we failed to authenticate (but note it might not be an actual
+  // authentication problem, just a transient network error or similar)
+  _authFailureReason: null,
+
+  // it takes some time to fetch a sync key bundle, so until this flag is set,
+  // we don't consider the lack of a keybundle as a failure state.
+  _shouldHaveSyncKeyBundle: false,
+
+  get needsCustomization() {
+    try {
+      return Services.prefs.getBoolPref(PREF_SYNC_SHOW_CUSTOMIZATION);
+    } catch (e) {
+      return false;
+    }
+  },
+
+  hashedUID() {
+    if (!this._token) {
+      throw new Error("hashedUID: Don't have token");
+    }
+    return this._token.hashed_fxa_uid
+  },
+
+  deviceID() {
+    return this._signedInUser && this._signedInUser.deviceId;
+  },
+
+  initialize: function() {
+    for (let topic of OBSERVER_TOPICS) {
+      Services.obs.addObserver(this, topic, false);
+    }
+    // and a background fetch of account data just so we can set this.account,
+    // so we have a username available before we've actually done a login.
+    // XXX - this is actually a hack just for tests and really shouldn't be
+    // necessary. Also, you'd think it would be safe to allow this.account to
+    // be set to null when there's no user logged in, but argue with the test
+    // suite, not with me :)
+    this._fxaService.getSignedInUser().then(accountData => {
+      if (accountData) {
+        this.account = accountData.email;
+      }
+    }).catch(err => {
+      // As above, this is only for tests so it is safe to ignore.
+    });
+  },
+
+  /**
+   * Ensure the user is logged in.  Returns a promise that resolves when
+   * the user is logged in, or is rejected if the login attempt has failed.
+   */
+  ensureLoggedIn: function() {
+    if (!this._shouldHaveSyncKeyBundle && this.whenReadyToAuthenticate) {
+      // We are already in the process of logging in.
+      return this.whenReadyToAuthenticate.promise;
+    }
+
+    // If we are already happy then there is nothing more to do.
+    if (this._syncKeyBundle) {
+      return Promise.resolve();
+    }
+
+    // Similarly, if we have a previous failure that implies an explicit
+    // re-entering of credentials by the user is necessary we don't take any
+    // further action - an observer will fire when the user does that.
+    if (Weave.Status.login == LOGIN_FAILED_LOGIN_REJECTED) {
+      return Promise.reject(new Error("User needs to re-authenticate"));
+    }
+
+    // So - we've a previous auth problem and aren't currently attempting to
+    // log in - so fire that off.
+    this.initializeWithCurrentIdentity();
+    return this.whenReadyToAuthenticate.promise;
+  },
+
+  finalize: function() {
+    // After this is called, we can expect Service.identity != this.
+    for (let topic of OBSERVER_TOPICS) {
+      Services.obs.removeObserver(this, topic);
+    }
+    this.resetCredentials();
+    this._signedInUser = null;
+  },
+
+  offerSyncOptions: function () {
+    // If the user chose to "Customize sync options" when signing
+    // up with Firefox Accounts, ask them to choose what to sync.
+    const url = "chrome://browser/content/sync/customize.xul";
+    const features = "centerscreen,chrome,modal,dialog,resizable=no";
+    let win = Services.wm.getMostRecentWindow("navigator:browser");
+
+    let data = {accepted: false};
+    win.openDialog(url, "_blank", features, data);
+
+    return data;
+  },
+
+  initializeWithCurrentIdentity: function(isInitialSync=false) {
+    // While this function returns a promise that resolves once we've started
+    // the auth process, that process is complete when
+    // this.whenReadyToAuthenticate.promise resolves.
+    this._log.trace("initializeWithCurrentIdentity");
+
+    // Reset the world before we do anything async.
+    this.whenReadyToAuthenticate = Promise.defer();
+    this.whenReadyToAuthenticate.promise.catch(err => {
+      this._log.error("Could not authenticate", err);
+    });
+
+    // initializeWithCurrentIdentity() can be called after the
+    // identity module was first initialized, e.g., after the
+    // user completes a force authentication, so we should make
+    // sure all credentials are reset before proceeding.
+    this.resetCredentials();
+    this._authFailureReason = null;
+
+    return this._fxaService.getSignedInUser().then(accountData => {
+      if (!accountData) {
+        this._log.info("initializeWithCurrentIdentity has no user logged in");
+        this.account = null;
+        // and we are as ready as we can ever be for auth.
+        this._shouldHaveSyncKeyBundle = true;
+        this.whenReadyToAuthenticate.reject("no user is logged in");
+        return;
+      }
+
+      this.account = accountData.email;
+      this._updateSignedInUser(accountData);
+      // The user must be verified before we can do anything at all; we kick
+      // this and the rest of initialization off in the background (ie, we
+      // don't return the promise)
+      this._log.info("Waiting for user to be verified.");
+      this._fxaService.whenVerified(accountData).then(accountData => {
+        this._updateSignedInUser(accountData);
+        this._log.info("Starting fetch for key bundle.");
+        if (this.needsCustomization) {
+          let data = this.offerSyncOptions();
+          if (data.accepted) {
+            Services.prefs.clearUserPref(PREF_SYNC_SHOW_CUSTOMIZATION);
+
+            // Mark any non-selected engines as declined.
+            Weave.Service.engineManager.declineDisabled();
+          } else {
+            // Log out if the user canceled the dialog.
+            return this._fxaService.signOut();
+          }
+        }
+      }).then(() => {
+        return this._fetchTokenForUser();
+      }).then(token => {
+        this._token = token;
+        this._shouldHaveSyncKeyBundle = true; // and we should actually have one...
+        this.whenReadyToAuthenticate.resolve();
+        this._log.info("Background fetch for key bundle done");
+        Weave.Status.login = LOGIN_SUCCEEDED;
+        if (isInitialSync) {
+          this._log.info("Doing initial sync actions");
+          Svc.Prefs.set("firstSync", "resetClient");
+          Services.obs.notifyObservers(null, "weave:service:setup-complete", null);
+          Weave.Utils.nextTick(Weave.Service.sync, Weave.Service);
+        }
+      }).catch(authErr => {
+        // report what failed...
+        this._log.error("Background fetch for key bundle failed", authErr);
+        this._shouldHaveSyncKeyBundle = true; // but we probably don't have one...
+        this.whenReadyToAuthenticate.reject(authErr);
+      });
+      // and we are done - the fetch continues on in the background...
+    }).catch(err => {
+      this._log.error("Processing logged in account", err);
+    });
+  },
+
+  _updateSignedInUser: function(userData) {
+    // This object should only ever be used for a single user.  It is an
+    // error to update the data if the user changes (but updates are still
+    // necessary, as each call may add more attributes to the user).
+    // We start with no user, so an initial update is always ok.
+    if (this._signedInUser && this._signedInUser.email != userData.email) {
+      throw new Error("Attempting to update to a different user.")
+    }
+    this._signedInUser = userData;
+  },
+
+  logout: function() {
+    // This will be called when sync fails (or when the account is being
+    // unlinked etc).  It may have failed because we got a 401 from a sync
+    // server, so we nuke the token.  Next time sync runs and wants an
+    // authentication header, we will notice the lack of the token and fetch a
+    // new one.
+    this._token = null;
+  },
+
+  observe: function (subject, topic, data) {
+    this._log.debug("observed " + topic);
+    switch (topic) {
+    case fxAccountsCommon.ONLOGIN_NOTIFICATION:
+      // This should only happen if we've been initialized without a current
+      // user - otherwise we'd have seen the LOGOUT notification and been
+      // thrown away.
+      // The exception is when we've initialized with a user that needs to
+      // reauth with the server - in that case we will also get here, but
+      // should have the same identity.
+      // initializeWithCurrentIdentity will throw and log if these constraints
+      // aren't met (indirectly, via _updateSignedInUser()), so just go ahead
+      // and do the init.
+      this.initializeWithCurrentIdentity(true);
+      break;
+
+    case fxAccountsCommon.ONLOGOUT_NOTIFICATION:
+      Weave.Service.startOver();
+      // startOver will cause this instance to be thrown away, so there's
+      // nothing else to do.
+      break;
+
+    case fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION:
+      // throw away token and fetch a new one
+      this.resetCredentials();
+      this._ensureValidToken().catch(err =>
+        this._log.error("Error while fetching a new token", err));
+      break;
+    }
+  },
+
+  /**
+   * Compute the sha256 of the message bytes.  Return bytes.
+   */
+  _sha256: function(message) {
+    let hasher = Cc["@mozilla.org/security/hash;1"]
+                    .createInstance(Ci.nsICryptoHash);
+    hasher.init(hasher.SHA256);
+    return CryptoUtils.digestBytes(message, hasher);
+  },
+
+  /**
+   * Compute the X-Client-State header given the byte string kB.
+   *
+   * Return string: hex(first16Bytes(sha256(kBbytes)))
+   */
+  _computeXClientState: function(kBbytes) {
+    return CommonUtils.bytesAsHex(this._sha256(kBbytes).slice(0, 16), false);
+  },
+
+  /**
+   * Provide override point for testing token expiration.
+   */
+  _now: function() {
+    return this._fxaService.now()
+  },
+
+  get _localtimeOffsetMsec() {
+    return this._fxaService.localtimeOffsetMsec;
+  },
+
+  usernameFromAccount: function(val) {
+    // we don't differentiate between "username" and "account"
+    return val;
+  },
+
+  /**
+   * Obtains the HTTP Basic auth password.
+   *
+   * Returns a string if set or null if it is not set.
+   */
+  get basicPassword() {
+    this._log.error("basicPassword getter should be not used in BrowserIDManager");
+    return null;
+  },
+
+  /**
+   * Set the HTTP basic password to use.
+   *
+   * Changes will not persist unless persistSyncCredentials() is called.
+   */
+  set basicPassword(value) {
+    throw "basicPassword setter should be not used in BrowserIDManager";
+  },
+
+  /**
+   * Obtain the Sync Key.
+   *
+   * This returns a 26 character "friendly" Base32 encoded string on success or
+   * null if no Sync Key could be found.
+   *
+   * If the Sync Key hasn't been set in this session, this will look in the
+   * password manager for the sync key.
+   */
+  get syncKey() {
+    if (this.syncKeyBundle) {
+      // TODO: This is probably fine because the code shouldn't be
+      // using the sync key directly (it should use the sync key
+      // bundle), but I don't like it. We should probably refactor
+      // code that is inspecting this to not do validation on this
+      // field directly and instead call a isSyncKeyValid() function
+      // that we can override.
+      return "99999999999999999999999999";
+    }
+    else {
+      return null;
+    }
+  },
+
+  set syncKey(value) {
+    throw "syncKey setter should be not used in BrowserIDManager";
+  },
+
+  get syncKeyBundle() {
+    return this._syncKeyBundle;
+  },
+
+  /**
+   * Resets/Drops all credentials we hold for the current user.
+   */
+  resetCredentials: function() {
+    this.resetSyncKey();
+    this._token = null;
+    // The cluster URL comes from the token, so resetting it to empty will
+    // force Sync to not accidentally use a value from an earlier token.
+    Weave.Service.clusterURL = null;
+  },
+
+  /**
+   * Resets/Drops the sync key we hold for the current user.
+   */
+  resetSyncKey: function() {
+    this._syncKey = null;
+    this._syncKeyBundle = null;
+    this._syncKeyUpdated = true;
+    this._shouldHaveSyncKeyBundle = false;
+  },
+
+  /**
+   * Pre-fetches any information that might help with migration away from this
+   * identity.  Called after every sync and is really just an optimization that
+   * allows us to avoid a network request for when we actually need the
+   * migration info.
+   */
+  prefetchMigrationSentinel: function(service) {
+    // nothing to do here until we decide to migrate away from FxA.
+  },
+
+  /**
+    * Return credentials hosts for this identity only.
+    */
+  _getSyncCredentialsHosts: function() {
+    return Utils.getSyncCredentialsHostsFxA();
+  },
+
+  /**
+   * The current state of the auth credentials.
+   *
+   * This essentially validates that enough credentials are available to use
+   * Sync. It doesn't check we have all the keys we need as the master-password
+   * may have been locked when we tried to get them - we rely on
+   * unlockAndVerifyAuthState to check that for us.
+   */
+  get currentAuthState() {
+    if (this._authFailureReason) {
+      this._log.info("currentAuthState returning " + this._authFailureReason +
+                     " due to previous failure");
+      return this._authFailureReason;
+    }
+    // TODO: need to revisit this. Currently this isn't ready to go until
+    // both the username and syncKeyBundle are both configured and having no
+    // username seems to make things fail fast so that's good.
+    if (!this.username) {
+      return LOGIN_FAILED_NO_USERNAME;
+    }
+
+    return STATUS_OK;
+  },
+
+  // Do we currently have keys, or do we have enough that we should be able
+  // to successfully fetch them?
+  _canFetchKeys: function() {
+    let userData = this._signedInUser;
+    // a keyFetchToken means we can almost certainly grab them.
+    // kA and kB means we already have them.
+    return userData && (userData.keyFetchToken || (userData.kA && userData.kB));
+  },
+
+  /**
+   * Verify the current auth state, unlocking the master-password if necessary.
+   *
+   * Returns a promise that resolves with the current auth state after
+   * attempting to unlock.
+   */
+  unlockAndVerifyAuthState: function() {
+    if (this._canFetchKeys()) {
+      log.debug("unlockAndVerifyAuthState already has (or can fetch) sync keys");
+      return Promise.resolve(STATUS_OK);
+    }
+    // so no keys - ensure MP unlocked.
+    if (!Utils.ensureMPUnlocked()) {
+      // user declined to unlock, so we don't know if they are stored there.
+      log.debug("unlockAndVerifyAuthState: user declined to unlock master-password");
+      return Promise.resolve(MASTER_PASSWORD_LOCKED);
+    }
+    // now we are unlocked we must re-fetch the user data as we may now have
+    // the details that were previously locked away.
+    return this._fxaService.getSignedInUser().then(
+      accountData => {
+        this._updateSignedInUser(accountData);
+        // If we still can't get keys it probably means the user authenticated
+        // without unlocking the MP or cleared the saved logins, so we've now
+        // lost them - the user will need to reauth before continuing.
+        let result;
+        if (this._canFetchKeys()) {
+          result = STATUS_OK;
+        } else {
+          result = LOGIN_FAILED_LOGIN_REJECTED;
+        }
+        log.debug("unlockAndVerifyAuthState re-fetched credentials and is returning", result);
+        return result;
+      }
+    );
+  },
+
+  /**
+   * Do we have a non-null, not yet expired token for the user currently
+   * signed in?
+   */
+  hasValidToken: function() {
+    // If pref is set to ignore cached authentication credentials for debugging,
+    // then return false to force the fetching of a new token.
+    let ignoreCachedAuthCredentials = false;
+    try {
+      ignoreCachedAuthCredentials = Svc.Prefs.get("debug.ignoreCachedAuthCredentials");
+    } catch(e) {
+      // Pref doesn't exist
+    }
+    if (ignoreCachedAuthCredentials) {
+      return false;
+    }
+    if (!this._token) {
+      return false;
+    }
+    if (this._token.expiration < this._now()) {
+      return false;
+    }
+    return true;
+  },
+
+  // Get our tokenServerURL - a private helper. Returns a string.
+  get _tokenServerUrl() {
+    // We used to support services.sync.tokenServerURI but this was a
+    // pain-point for people using non-default servers as Sync may auto-reset
+    // all services.sync prefs. So if that still exists, it wins.
+    let url = Svc.Prefs.get("tokenServerURI"); // Svc.Prefs "root" is services.sync
+    if (!url) {
+      url = Services.prefs.getCharPref("identity.sync.tokenserver.uri");
+    }
+    while (url.endsWith("/")) { // trailing slashes cause problems...
+      url = url.slice(0, -1);
+    }
+    return url;
+  },
+
+  // Refresh the sync token for our user. Returns a promise that resolves
+  // with a token (which may be null in one sad edge-case), or rejects with an
+  // error.
+  _fetchTokenForUser: function() {
+    // tokenServerURI is mis-named - convention is uri means nsISomething...
+    let tokenServerURI = this._tokenServerUrl;
+    let log = this._log;
+    let client = this._tokenServerClient;
+    let fxa = this._fxaService;
+    let userData = this._signedInUser;
+
+    // We need kA and kB for things to work.  If we don't have them, just
+    // return null for the token - sync calling unlockAndVerifyAuthState()
+    // before actually syncing will setup the error states if necessary.
+    if (!this._canFetchKeys()) {
+      log.info("Unable to fetch keys (master-password locked?), so aborting token fetch");
+      return Promise.resolve(null);
+    }
+
+    let maybeFetchKeys = () => {
+      // This is called at login time and every time we need a new token - in
+      // the latter case we already have kA and kB, so optimise that case.
+      if (userData.kA && userData.kB) {
+        return;
+      }
+      log.info("Fetching new keys");
+      return this._fxaService.getKeys().then(
+        newUserData => {
+          userData = newUserData;
+          this._updateSignedInUser(userData); // throws if the user changed.
+        }
+      );
+    }
+
+    let getToken = assertion => {
+      log.debug("Getting a token");
+      let deferred = Promise.defer();
+      let cb = function (err, token) {
+        if (err) {
+          return deferred.reject(err);
+        }
+        log.debug("Successfully got a sync token");
+        return deferred.resolve(token);
+      };
+
+      let kBbytes = CommonUtils.hexToBytes(userData.kB);
+      let headers = {"X-Client-State": this._computeXClientState(kBbytes)};
+      client.getTokenFromBrowserIDAssertion(tokenServerURI, assertion, cb, headers);
+      return deferred.promise;
+    }
+
+    let getAssertion = () => {
+      log.info("Getting an assertion from", tokenServerURI);
+      let audience = Services.io.newURI(tokenServerURI, null, null).prePath;
+      return fxa.getAssertion(audience);
+    };
+
+    // wait until the account email is verified and we know that
+    // getAssertion() will return a real assertion (not null).
+    return fxa.whenVerified(this._signedInUser)
+      .then(() => maybeFetchKeys())
+      .then(() => getAssertion())
+      .then(assertion => getToken(assertion))
+      .catch(err => {
+        // If we get a 401 fetching the token it may be that our certificate
+        // needs to be regenerated.
+        if (!err.response || err.response.status !== 401) {
+          return Promise.reject(err);
+        }
+        log.warn("Token server returned 401, refreshing certificate and retrying token fetch");
+        return fxa.invalidateCertificate()
+          .then(() => getAssertion())
+          .then(assertion => getToken(assertion))
+      })
+      .then(token => {
+        // TODO: Make it be only 80% of the duration, so refresh the token
+        // before it actually expires. This is to avoid sync storage errors
+        // otherwise, we get a nasty notification bar briefly. Bug 966568.
+        token.expiration = this._now() + (token.duration * 1000) * 0.80;
+        if (!this._syncKeyBundle) {
+          // We are given kA/kB as hex.
+          this._syncKeyBundle = deriveKeyBundle(Utils.hexToBytes(userData.kB));
+        }
+        return token;
+      })
+      .catch(err => {
+        // TODO: unify these errors - we need to handle errors thrown by
+        // both tokenserverclient and hawkclient.
+        // A tokenserver error thrown based on a bad response.
+        if (err.response && err.response.status === 401) {
+          err = new AuthenticationError(err, "tokenserver");
+        // A hawkclient error.
+        } else if (err.code && err.code === 401) {
+          err = new AuthenticationError(err, "hawkclient");
+        // An FxAccounts.jsm error.
+        } else if (err.message == fxAccountsCommon.ERROR_AUTH_ERROR) {
+          err = new AuthenticationError(err, "fxaccounts");
+        }
+
+        // TODO: write tests to make sure that different auth error cases are handled here
+        // properly: auth error getting assertion, auth error getting token (invalid generation
+        // and client-state error)
+        if (err instanceof AuthenticationError) {
+          this._log.error("Authentication error in _fetchTokenForUser", err);
+          // set it to the "fatal" LOGIN_FAILED_LOGIN_REJECTED reason.
+          this._authFailureReason = LOGIN_FAILED_LOGIN_REJECTED;
+        } else {
+          this._log.error("Non-authentication error in _fetchTokenForUser", err);
+          // for now assume it is just a transient network related problem
+          // (although sadly, it might also be a regular unhandled exception)
+          this._authFailureReason = LOGIN_FAILED_NETWORK_ERROR;
+        }
+        // this._authFailureReason being set to be non-null in the above if clause
+        // ensures we are in the correct currentAuthState, and
+        // this._shouldHaveSyncKeyBundle being true ensures everything that cares knows
+        // that there is no authentication dance still under way.
+        this._shouldHaveSyncKeyBundle = true;
+        Weave.Status.login = this._authFailureReason;
+        throw err;
+      });
+  },
+
+  // Returns a promise that is resolved when we have a valid token for the
+  // current user stored in this._token.  When resolved, this._token is valid.
+  _ensureValidToken: function() {
+    if (this.hasValidToken()) {
+      this._log.debug("_ensureValidToken already has one");
+      return Promise.resolve();
+    }
+    const notifyStateChanged =
+      () => Services.obs.notifyObservers(null, "weave:service:login:change", null);
+    // reset this._token as a safety net to reduce the possibility of us
+    // repeatedly attempting to use an invalid token if _fetchTokenForUser throws.
+    this._token = null;
+    return this._fetchTokenForUser().then(
+      token => {
+        this._token = token;
+        notifyStateChanged();
+      },
+      error => {
+        notifyStateChanged();
+        throw error
+      }
+    );
+  },
+
+  getResourceAuthenticator: function () {
+    return this._getAuthenticationHeader.bind(this);
+  },
+
+  /**
+   * Obtain a function to be used for adding auth to RESTRequest instances.
+   */
+  getRESTRequestAuthenticator: function() {
+    return this._addAuthenticationHeader.bind(this);
+  },
+
+  /**
+   * @return a Hawk HTTP Authorization Header, lightly wrapped, for the .uri
+   * of a RESTRequest or AsyncResponse object.
+   */
+  _getAuthenticationHeader: function(httpObject, method) {
+    let cb = Async.makeSpinningCallback();
+    this._ensureValidToken().then(cb, cb);
+    // Note that in failure states we return null, causing the request to be
+    // made without authorization headers, thereby presumably causing a 401,
+    // which causes Sync to log out. If we throw, this may not happen as
+    // expected.
+    try {
+      cb.wait();
+    } catch (ex) {
+      if (Async.isShutdownException(ex)) {
+        throw ex;
+      }
+      this._log.error("Failed to fetch a token for authentication", ex);
+      return null;
+    }
+    if (!this._token) {
+      return null;
+    }
+    let credentials = {algorithm: "sha256",
+                       id: this._token.id,
+                       key: this._token.key,
+                      };
+    method = method || httpObject.method;
+
+    // Get the local clock offset from the Firefox Accounts server.  This should
+    // be close to the offset from the storage server.
+    let options = {
+      now: this._now(),
+      localtimeOffsetMsec: this._localtimeOffsetMsec,
+      credentials: credentials,
+    };
+
+    let headerValue = CryptoUtils.computeHAWK(httpObject.uri, method, options);
+    return {headers: {authorization: headerValue.field}};
+  },
+
+  _addAuthenticationHeader: function(request, method) {
+    let header = this._getAuthenticationHeader(request, method);
+    if (!header) {
+      return null;
+    }
+    request.setHeader("authorization", header.headers.authorization);
+    return request;
+  },
+
+  createClusterManager: function(service) {
+    return new BrowserIDClusterManager(service);
+  },
+
+  // Tell Sync what the login status should be if it saw a 401 fetching
+  // info/collections as part of login verification (typically immediately
+  // after login.)
+  // In our case, it almost certainly means a transient error fetching a token
+  // (and hitting this will cause us to logout, which will correctly handle an
+  // authoritative login issue.)
+  loginStatusFromVerification404() {
+    return LOGIN_FAILED_NETWORK_ERROR;
+  },
+};
+
+/* An implementation of the ClusterManager for this identity
+ */
+
+function BrowserIDClusterManager(service) {
+  ClusterManager.call(this, service);
+}
+
+BrowserIDClusterManager.prototype = {
+  __proto__: ClusterManager.prototype,
+
+  _findCluster: function() {
+    let endPointFromIdentityToken = function() {
+      // The only reason (in theory ;) that we can end up with a null token
+      // is when this.identity._canFetchKeys() returned false.  In turn, this
+      // should only happen if the master-password is locked or the credentials
+      // storage is screwed, and in those cases we shouldn't have started
+      // syncing so shouldn't get here anyway.
+      // But better safe than sorry! To keep things clearer, throw an explicit
+      // exception - the message will appear in the logs and the error will be
+      // treated as transient.
+      if (!this.identity._token) {
+        throw new Error("Can't get a cluster URL as we can't fetch keys.");
+      }
+      let endpoint = this.identity._token.endpoint;
+      // For Sync 1.5 storage endpoints, we use the base endpoint verbatim.
+      // However, it should end in "/" because we will extend it with
+      // well known path components. So we add a "/" if it's missing.
+      if (!endpoint.endsWith("/")) {
+        endpoint += "/";
+      }
+      log.debug("_findCluster returning " + endpoint);
+      return endpoint;
+    }.bind(this);
+
+    // Spinningly ensure we are ready to authenticate and have a valid token.
+    let promiseClusterURL = function() {
+      return this.identity.whenReadyToAuthenticate.promise.then(
+        () => {
+          // We need to handle node reassignment here.  If we are being asked
+          // for a clusterURL while the service already has a clusterURL, then
+          // it's likely a 401 was received using the existing token - in which
+          // case we just discard the existing token and fetch a new one.
+          if (this.service.clusterURL) {
+            log.debug("_findCluster has a pre-existing clusterURL, so discarding the current token");
+            this.identity._token = null;
+          }
+          return this.identity._ensureValidToken();
+        }
+      ).then(endPointFromIdentityToken
+      );
+    }.bind(this);
+
+    let cb = Async.makeSpinningCallback();
+    promiseClusterURL().then(function (clusterURL) {
+      cb(null, clusterURL);
+    }).then(
+      null, err => {
+      log.info("Failed to fetch the cluster URL", err);
+      // service.js's verifyLogin() method will attempt to fetch a cluster
+      // URL when it sees a 401.  If it gets null, it treats it as a "real"
+      // auth error and sets Status.login to LOGIN_FAILED_LOGIN_REJECTED, which
+      // in turn causes a notification bar to appear informing the user they
+      // need to re-authenticate.
+      // On the other hand, if fetching the cluster URL fails with an exception,
+      // verifyLogin() assumes it is a transient error, and thus doesn't show
+      // the notification bar under the assumption the issue will resolve
+      // itself.
+      // Thus:
+      // * On a real 401, we must return null.
+      // * On any other problem we must let an exception bubble up.
+      if (err instanceof AuthenticationError) {
+        // callback with no error and a null result - cb.wait() returns null.
+        cb(null, null);
+      } else {
+        // callback with an error - cb.wait() completes by raising an exception.
+        cb(err);
+      }
+    });
+    return cb.wait();
+  },
+
+  getUserBaseURL: function() {
+    // Legacy Sync and FxA Sync construct the userBaseURL differently. Legacy
+    // Sync appends path components onto an empty path, and in FxA Sync the
+    // token server constructs this for us in an opaque manner. Since the
+    // cluster manager already sets the clusterURL on Service and also has
+    // access to the current identity, we added this functionality here.
+    return this.service.clusterURL;
+  }
+}

services/sync/modules/collection_validator.js

@@ -0,0 +1,204 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const Cu = Components.utils;
+
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/main.js");
+
+this.EXPORTED_SYMBOLS = ["CollectionValidator", "CollectionProblemData"];
+
+class CollectionProblemData {
+  constructor() {
+    this.missingIDs = 0;
+    this.duplicates = [];
+    this.clientMissing = [];
+    this.serverMissing = [];
+    this.serverDeleted = [];
+    this.serverUnexpected = [];
+    this.differences = [];
+  }
+
+  /**
+   * Produce a list summarizing problems found. Each entry contains {name, count},
+   * where name is the field name for the problem, and count is the number of times
+   * the problem was encountered.
+   *
+   * Validation has failed if all counts are not 0.
+   */
+  getSummary() {
+    return [
+      { name: "clientMissing", count: this.clientMissing.length },
+      { name: "serverMissing", count: this.serverMissing.length },
+      { name: "serverDeleted", count: this.serverDeleted.length },
+      { name: "serverUnexpected", count: this.serverUnexpected.length },
+      { name: "differences", count: this.differences.length },
+      { name: "missingIDs", count: this.missingIDs },
+      { name: "duplicates", count: this.duplicates.length }
+    ];
+  }
+}
+
+class CollectionValidator {
+  // Construct a generic collection validator. This is intended to be called by
+  // subclasses.
+  // - name: Name of the engine
+  // - idProp: Property that identifies a record. That is, if a client and server
+  //   record have the same value for the idProp property, they should be
+  //   compared against eachother.
+  // - props: Array of properties that should be compared
+  constructor(name, idProp, props) {
+    this.name = name;
+    this.props = props;
+    this.idProp = idProp;
+  }
+
+  // Should a custom ProblemData type be needed, return it here.
+  emptyProblemData() {
+    return new CollectionProblemData();
+  }
+
+  getServerItems(engine) {
+    let collection = engine.itemSource();
+    let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name);
+    collection.full = true;
+    let items = [];
+    collection.recordHandler = function(item) {
+      item.decrypt(collectionKey);
+      items.push(item.cleartext);
+    };
+    let resp = collection.getBatched();
+    if (!resp.success) {
+      throw resp;
+    }
+    return items;
+  }
+
+  // Should return a promise that resolves to an array of client items.
+  getClientItems() {
+    return Promise.reject("Must implement");
+  }
+
+  // Turn the client item into something that can be compared with the server item,
+  // and is also safe to mutate.
+  normalizeClientItem(item) {
+    return Cu.cloneInto(item, {});
+  }
+
+  // Turn the server item into something that can be easily compared with the client
+  // items.
+  normalizeServerItem(item) {
+    return item;
+  }
+
+  // Return whether or not a server item should be present on the client. Expected
+  // to be overridden.
+  clientUnderstands(item) {
+    return true;
+  }
+
+  // Return whether or not a client item should be present on the server. Expected
+  // to be overridden
+  syncedByClient(item) {
+    return true;
+  }
+
+  // Compare the server item and the client item, and return a list of property
+  // names that are different. Can be overridden if needed.
+  getDifferences(client, server) {
+    let differences = [];
+    for (let prop of this.props) {
+      let clientProp = client[prop];
+      let serverProp = server[prop];
+      if ((clientProp || "") !== (serverProp || "")) {
+        differences.push(prop);
+      }
+    }
+    return differences;
+  }
+
+  // Returns an object containing
+  //   problemData: an instance of the class returned by emptyProblemData(),
+  //   clientRecords: Normalized client records
+  //   records: Normalized server records,
+  //   deletedRecords: Array of ids that were marked as deleted by the server.
+  compareClientWithServer(clientItems, serverItems) {
+    clientItems = clientItems.map(item => this.normalizeClientItem(item));
+    serverItems = serverItems.map(item => this.normalizeServerItem(item));
+    let problems = this.emptyProblemData();
+    let seenServer = new Map();
+    let serverDeleted = new Set();
+    let allRecords = new Map();
+
+    for (let record of serverItems) {
+      let id = record[this.idProp];
+      if (!id) {
+        ++problems.missingIDs;
+        continue;
+      }
+      if (record.deleted) {
+        serverDeleted.add(record);
+      } else {
+        let possibleDupe = seenServer.get(id);
+        if (possibleDupe) {
+          problems.duplicates.push(id);
+        } else {
+          seenServer.set(id, record);
+          allRecords.set(id, { server: record, client: null, });
+        }
+        record.understood = this.clientUnderstands(record);
+      }
+    }
+
+    let recordPairs = [];
+    let seenClient = new Map();
+    for (let record of clientItems) {
+      let id = record[this.idProp];
+      record.shouldSync = this.syncedByClient(record);
+      seenClient.set(id, record);
+      let combined = allRecords.get(id);
+      if (combined) {
+        combined.client = record;
+      } else {
+        allRecords.set(id,  { client: record, server: null });
+      }
+    }
+
+    for (let [id, { server, client }] of allRecords) {
+      if (!client && !server) {
+        throw new Error("Impossible: no client or server record for " + id);
+      } else if (server && !client) {
+        if (server.understood) {
+          problems.clientMissing.push(id);
+        }
+      } else if (client && !server) {
+        if (client.shouldSync) {
+          problems.serverMissing.push(id);
+        }
+      } else {
+        if (!client.shouldSync) {
+          if (!problems.serverUnexpected.includes(id)) {
+            problems.serverUnexpected.push(id);
+          }
+          continue;
+        }
+        let differences = this.getDifferences(client, server);
+        if (differences && differences.length) {
+          problems.differences.push({ id, differences });
+        }
+      }
+    }
+    return {
+      problemData: problems,
+      clientRecords: clientItems,
+      records: serverItems,
+      deletedRecords: [...serverDeleted]
+    };
+  }
+}
+
+// Default to 0, some engines may override.
+CollectionValidator.prototype.version = 0;

services/sync/modules/constants.js

@@ -0,0 +1,198 @@
+#filter substitution
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Process each item in the "constants hash" to add to "global" and give a name
+this.EXPORTED_SYMBOLS = [];
+for (let [key, val] of Object.entries({
+
+WEAVE_VERSION:                         "@weave_version@",
+
+// Sync Server API version that the client supports.
+SYNC_API_VERSION:                      "1.1",
+USER_API_VERSION:                      "1.0",
+MISC_API_VERSION:                      "1.0",
+
+// Version of the data format this client supports. The data format describes
+// how records are packaged; this is separate from the Server API version and
+// the per-engine cleartext formats.
+STORAGE_VERSION:                       5,
+PREFS_BRANCH:                          "services.sync.",
+
+// Host "key" to access Weave Identity in the password manager
+PWDMGR_HOST:                           "chrome://weave",
+PWDMGR_PASSWORD_REALM:                 "Mozilla Services Password",
+PWDMGR_PASSPHRASE_REALM:               "Mozilla Services Encryption Passphrase",
+PWDMGR_KEYBUNDLE_REALM:                "Mozilla Services Key Bundles",
+
+// Put in [] because those aren't allowed in a collection name.
+DEFAULT_KEYBUNDLE_NAME:                "[default]",
+
+// Our extra input to SHA256-HMAC in generateEntry.
+// This includes the full crypto spec; change this when our algo changes.
+HMAC_INPUT:                            "Sync-AES_256_CBC-HMAC256",
+
+// Key dimensions.
+SYNC_KEY_ENCODED_LENGTH:               26,
+SYNC_KEY_DECODED_LENGTH:               16,
+SYNC_KEY_HYPHENATED_LENGTH:            31,    // 26 chars, 5 hyphens.
+
+NO_SYNC_NODE_INTERVAL:                 10 * 60 * 1000, // 10 minutes
+
+MAX_ERROR_COUNT_BEFORE_BACKOFF:        3,
+MAX_IGNORE_ERROR_COUNT:                5,
+
+// Backoff intervals
+MINIMUM_BACKOFF_INTERVAL:              15 * 60 * 1000,      // 15 minutes
+MAXIMUM_BACKOFF_INTERVAL:              8 * 60 * 60 * 1000,  // 8 hours
+
+// HMAC event handling timeout.
+// 10 minutes: a compromise between the multi-desktop sync interval
+// and the mobile sync interval.
+HMAC_EVENT_INTERVAL:                   600000,
+
+// How long to wait between sync attempts if the Master Password is locked.
+MASTER_PASSWORD_LOCKED_RETRY_INTERVAL: 15 * 60 * 1000,   // 15 minutes
+
+// The default for how long we "block" sync from running when doing a migration.
+DEFAULT_BLOCK_PERIOD:                  2 * 24 * 60 * 60 * 1000, // 2 days
+
+// Separate from the ID fetch batch size to allow tuning for mobile.
+MOBILE_BATCH_SIZE:                     50,
+
+// 50 is hardcoded here because of URL length restrictions.
+// (GUIDs can be up to 64 chars long.)
+// Individual engines can set different values for their limit if their
+// identifiers are shorter.
+DEFAULT_GUID_FETCH_BATCH_SIZE:         50,
+DEFAULT_MOBILE_GUID_FETCH_BATCH_SIZE:  50,
+
+// Default batch size for applying incoming records.
+DEFAULT_STORE_BATCH_SIZE:              1,
+HISTORY_STORE_BATCH_SIZE:              50,      // same as MOBILE_BATCH_SIZE
+FORMS_STORE_BATCH_SIZE:                50,      // same as MOBILE_BATCH_SIZE
+PASSWORDS_STORE_BATCH_SIZE:            50,      // same as MOBILE_BATCH_SIZE
+ADDONS_STORE_BATCH_SIZE:               1000000, // process all addons at once
+APPS_STORE_BATCH_SIZE:                 50,      // same as MOBILE_BATCH_SIZE
+
+// Default batch size for download batching
+// (how many records are fetched at a time from the server when batching is used).
+DEFAULT_DOWNLOAD_BATCH_SIZE:           1000,
+
+// score thresholds for early syncs
+SINGLE_USER_THRESHOLD:                 1000,
+MULTI_DEVICE_THRESHOLD:                300,
+
+// Other score increment constants
+SCORE_INCREMENT_SMALL:                 1,
+SCORE_INCREMENT_MEDIUM:                10,
+
+// Instant sync score increment
+SCORE_INCREMENT_XLARGE:                300 + 1, //MULTI_DEVICE_THRESHOLD + 1
+
+// Delay before incrementing global score
+SCORE_UPDATE_DELAY:                    100,
+
+// Delay for the back observer debouncer. This is chosen to be longer than any
+// observed spurious idle/back events and short enough to pre-empt user activity.
+IDLE_OBSERVER_BACK_DELAY:              100,
+
+// Max number of records or bytes to upload in a single POST - we'll do multiple POSTS if either
+// MAX_UPLOAD_RECORDS or MAX_UPLOAD_BYTES is hit)
+MAX_UPLOAD_RECORDS:                    100,
+MAX_UPLOAD_BYTES:                      1024 * 1023, // just under 1MB
+MAX_HISTORY_UPLOAD:                    5000,
+MAX_HISTORY_DOWNLOAD:                  5000,
+
+// TTL of the message sent to another device when sending a tab
+NOTIFY_TAB_SENT_TTL_SECS:              1 * 3600, // 1 hour
+
+// Top-level statuses:
+STATUS_OK:                             "success.status_ok",
+SYNC_FAILED:                           "error.sync.failed",
+LOGIN_FAILED:                          "error.login.failed",
+SYNC_FAILED_PARTIAL:                   "error.sync.failed_partial",
+CLIENT_NOT_CONFIGURED:                 "service.client_not_configured",
+STATUS_DISABLED:                       "service.disabled",
+MASTER_PASSWORD_LOCKED:                "service.master_password_locked",
+
+// success states
+LOGIN_SUCCEEDED:                       "success.login",
+SYNC_SUCCEEDED:                        "success.sync",
+ENGINE_SUCCEEDED:                      "success.engine",
+
+// login failure status codes:
+LOGIN_FAILED_NO_USERNAME:              "error.login.reason.no_username",
+LOGIN_FAILED_NO_PASSWORD:              "error.login.reason.no_password2",
+LOGIN_FAILED_NO_PASSPHRASE:            "error.login.reason.no_recoverykey",
+LOGIN_FAILED_NETWORK_ERROR:            "error.login.reason.network",
+LOGIN_FAILED_SERVER_ERROR:             "error.login.reason.server",
+LOGIN_FAILED_INVALID_PASSPHRASE:       "error.login.reason.recoverykey",
+LOGIN_FAILED_LOGIN_REJECTED:           "error.login.reason.account",
+
+// sync failure status codes
+METARECORD_DOWNLOAD_FAIL:              "error.sync.reason.metarecord_download_fail",
+VERSION_OUT_OF_DATE:                   "error.sync.reason.version_out_of_date",
+DESKTOP_VERSION_OUT_OF_DATE:           "error.sync.reason.desktop_version_out_of_date",
+SETUP_FAILED_NO_PASSPHRASE:            "error.sync.reason.setup_failed_no_passphrase",
+CREDENTIALS_CHANGED:                   "error.sync.reason.credentials_changed",
+ABORT_SYNC_COMMAND:                    "aborting sync, process commands said so",
+NO_SYNC_NODE_FOUND:                    "error.sync.reason.no_node_found",
+OVER_QUOTA:                            "error.sync.reason.over_quota",
+PROLONGED_SYNC_FAILURE:                "error.sync.prolonged_failure",
+SERVER_MAINTENANCE:                    "error.sync.reason.serverMaintenance",
+
+RESPONSE_OVER_QUOTA:                   "14",
+
+// engine failure status codes
+ENGINE_UPLOAD_FAIL:                    "error.engine.reason.record_upload_fail",
+ENGINE_DOWNLOAD_FAIL:                  "error.engine.reason.record_download_fail",
+ENGINE_UNKNOWN_FAIL:                   "error.engine.reason.unknown_fail",
+ENGINE_APPLY_FAIL:                     "error.engine.reason.apply_fail",
+ENGINE_METARECORD_DOWNLOAD_FAIL:       "error.engine.reason.metarecord_download_fail",
+ENGINE_METARECORD_UPLOAD_FAIL:         "error.engine.reason.metarecord_upload_fail",
+// an upload failure where the batch was interrupted with a 412
+ENGINE_BATCH_INTERRUPTED:              "error.engine.reason.batch_interrupted",
+
+JPAKE_ERROR_CHANNEL:                   "jpake.error.channel",
+JPAKE_ERROR_NETWORK:                   "jpake.error.network",
+JPAKE_ERROR_SERVER:                    "jpake.error.server",
+JPAKE_ERROR_TIMEOUT:                   "jpake.error.timeout",
+JPAKE_ERROR_INTERNAL:                  "jpake.error.internal",
+JPAKE_ERROR_INVALID:                   "jpake.error.invalid",
+JPAKE_ERROR_NODATA:                    "jpake.error.nodata",
+JPAKE_ERROR_KEYMISMATCH:               "jpake.error.keymismatch",
+JPAKE_ERROR_WRONGMESSAGE:              "jpake.error.wrongmessage",
+JPAKE_ERROR_USERABORT:                 "jpake.error.userabort",
+JPAKE_ERROR_DELAYUNSUPPORTED:          "jpake.error.delayunsupported",
+
+// info types for Service.getStorageInfo
+INFO_COLLECTIONS:                      "collections",
+INFO_COLLECTION_USAGE:                 "collection_usage",
+INFO_COLLECTION_COUNTS:                "collection_counts",
+INFO_QUOTA:                            "quota",
+
+// Ways that a sync can be disabled (messages only to be printed in debug log)
+kSyncMasterPasswordLocked:             "User elected to leave Master Password locked",
+kSyncWeaveDisabled:                    "Weave is disabled",
+kSyncNetworkOffline:                   "Network is offline",
+kSyncBackoffNotMet:                    "Trying to sync before the server said it's okay",
+kFirstSyncChoiceNotMade:               "User has not selected an action for first sync",
+
+// Application IDs
+FIREFOX_ID:                            "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
+FENNEC_ID:                             "{a23983c0-fd0e-11dc-95ff-0800200c9a66}",
+SEAMONKEY_ID:                          "{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}",
+TEST_HARNESS_ID:                       "xuth@mozilla.org",
+
+MIN_PP_LENGTH:                         12,
+MIN_PASS_LENGTH:                       8,
+
+DEVICE_TYPE_DESKTOP:                   "desktop",
+DEVICE_TYPE_MOBILE:                    "mobile",
+
+})) {
+  this[key] = val;
+  this.EXPORTED_SYMBOLS.push(key);
+}

services/sync/modules/engines/addons.js

@@ -0,0 +1,813 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file defines the add-on sync functionality.
+ *
+ * There are currently a number of known limitations:
+ *  - We only sync XPI extensions and themes available from addons.mozilla.org.
+ *    We hope to expand support for other add-ons eventually.
+ *  - We only attempt syncing of add-ons between applications of the same type.
+ *    This means add-ons will not synchronize between Firefox desktop and
+ *    Firefox mobile, for example. This is because of significant add-on
+ *    incompatibility between application types.
+ *
+ * Add-on records exist for each known {add-on, app-id} pair in the Sync client
+ * set. Each record has a randomly chosen GUID. The records then contain
+ * basic metadata about the add-on.
+ *
+ * We currently synchronize:
+ *
+ *  - Installations
+ *  - Uninstallations
+ *  - User enabling and disabling
+ *
+ * Synchronization is influenced by the following preferences:
+ *
+ *  - services.sync.addons.ignoreUserEnabledChanges
+ *  - services.sync.addons.trustedSourceHostnames
+ *
+ *  and also influenced by whether addons have repository caching enabled and
+ *  whether they allow installation of addons from insecure options (both of
+ *  which are themselves influenced by the "extensions." pref branch)
+ *
+ * See the documentation in services-sync.js for the behavior of these prefs.
+ */
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/addonutils.js");
+Cu.import("resource://services-sync/addonsreconciler.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/collection_validator.js");
+Cu.import("resource://services-common/async.js");
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
+                                  "resource://gre/modules/AddonManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository",
+                                  "resource://gre/modules/addons/AddonRepository.jsm");
+
+this.EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"];
+
+// 7 days in milliseconds.
+const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000;
+
+/**
+ * AddonRecord represents the state of an add-on in an application.
+ *
+ * Each add-on has its own record for each application ID it is installed
+ * on.
+ *
+ * The ID of add-on records is a randomly-generated GUID. It is random instead
+ * of deterministic so the URIs of the records cannot be guessed and so
+ * compromised server credentials won't result in disclosure of the specific
+ * add-ons present in a Sync account.
+ *
+ * The record contains the following fields:
+ *
+ *  addonID
+ *    ID of the add-on. This correlates to the "id" property on an Addon type.
+ *
+ *  applicationID
+ *    The application ID this record is associated with.
+ *
+ *  enabled
+ *    Boolean stating whether add-on is enabled or disabled by the user.
+ *
+ *  source
+ *    String indicating where an add-on is from. Currently, we only support
+ *    the value "amo" which indicates that the add-on came from the official
+ *    add-ons repository, addons.mozilla.org. In the future, we may support
+ *    installing add-ons from other sources. This provides a future-compatible
+ *    mechanism for clients to only apply records they know how to handle.
+ */
+function AddonRecord(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+AddonRecord.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Record.Addon"
+};
+
+Utils.deferGetSet(AddonRecord, "cleartext", ["addonID",
+                                             "applicationID",
+                                             "enabled",
+                                             "source"]);
+
+/**
+ * The AddonsEngine handles synchronization of add-ons between clients.
+ *
+ * The engine maintains an instance of an AddonsReconciler, which is the entity
+ * maintaining state for add-ons. It provides the history and tracking APIs
+ * that AddonManager doesn't.
+ *
+ * The engine instance overrides a handful of functions on the base class. The
+ * rationale for each is documented by that function.
+ */
+this.AddonsEngine = function AddonsEngine(service) {
+  SyncEngine.call(this, "Addons", service);
+
+  this._reconciler = new AddonsReconciler();
+}
+AddonsEngine.prototype = {
+  __proto__:              SyncEngine.prototype,
+  _storeObj:              AddonsStore,
+  _trackerObj:            AddonsTracker,
+  _recordObj:             AddonRecord,
+  version:                1,
+
+  syncPriority:           5,
+
+  _reconciler:            null,
+
+  /**
+   * Override parent method to find add-ons by their public ID, not Sync GUID.
+   */
+  _findDupe: function _findDupe(item) {
+    let id = item.addonID;
+
+    // The reconciler should have been updated at the top of the sync, so we
+    // can assume it is up to date when this function is called.
+    let addons = this._reconciler.addons;
+    if (!(id in addons)) {
+      return null;
+    }
+
+    let addon = addons[id];
+    if (addon.guid != item.id) {
+      return addon.guid;
+    }
+
+    return null;
+  },
+
+  /**
+   * Override getChangedIDs to pull in tracker changes plus changes from the
+   * reconciler log.
+   */
+  getChangedIDs: function getChangedIDs() {
+    let changes = {};
+    for (let [id, modified] of Object.entries(this._tracker.changedIDs)) {
+      changes[id] = modified;
+    }
+
+    let lastSyncDate = new Date(this.lastSync * 1000);
+
+    // The reconciler should have been refreshed at the beginning of a sync and
+    // we assume this function is only called from within a sync.
+    let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate);
+    let addons = this._reconciler.addons;
+    for (let change of reconcilerChanges) {
+      let changeTime = change[0];
+      let id = change[2];
+
+      if (!(id in addons)) {
+        continue;
+      }
+
+      // Keep newest modified time.
+      if (id in changes && changeTime < changes[id]) {
+          continue;
+      }
+
+      if (!this.isAddonSyncable(addons[id])) {
+        continue;
+      }
+
+      this._log.debug("Adding changed add-on from changes log: " + id);
+      let addon = addons[id];
+      changes[addon.guid] = changeTime.getTime() / 1000;
+    }
+
+    return changes;
+  },
+
+  /**
+   * Override start of sync function to refresh reconciler.
+   *
+   * Many functions in this class assume the reconciler is refreshed at the
+   * top of a sync. If this ever changes, those functions should be revisited.
+   *
+   * Technically speaking, we don't need to refresh the reconciler on every
+   * sync since it is installed as an AddonManager listener. However, add-ons
+   * are complicated and we force a full refresh, just in case the listeners
+   * missed something.
+   */
+  _syncStartup: function _syncStartup() {
+    // We refresh state before calling parent because syncStartup in the parent
+    // looks for changed IDs, which is dependent on add-on state being up to
+    // date.
+    this._refreshReconcilerState();
+
+    SyncEngine.prototype._syncStartup.call(this);
+  },
+
+  /**
+   * Override end of sync to perform a little housekeeping on the reconciler.
+   *
+   * We prune changes to prevent the reconciler state from growing without
+   * bound. Even if it grows unbounded, there would have to be many add-on
+   * changes (thousands) for it to slow things down significantly. This is
+   * highly unlikely to occur. Still, we exercise defense just in case.
+   */
+  _syncCleanup: function _syncCleanup() {
+    let ms = 1000 * this.lastSync - PRUNE_ADDON_CHANGES_THRESHOLD;
+    this._reconciler.pruneChangesBeforeDate(new Date(ms));
+
+    SyncEngine.prototype._syncCleanup.call(this);
+  },
+
+  /**
+   * Helper function to ensure reconciler is up to date.
+   *
+   * This will synchronously load the reconciler's state from the file
+   * system (if needed) and refresh the state of the reconciler.
+   */
+  _refreshReconcilerState: function _refreshReconcilerState() {
+    this._log.debug("Refreshing reconciler state");
+    let cb = Async.makeSpinningCallback();
+    this._reconciler.refreshGlobalState(cb);
+    cb.wait();
+  },
+
+  isAddonSyncable(addon, ignoreRepoCheck) {
+    return this._store.isAddonSyncable(addon, ignoreRepoCheck);
+  }
+};
+
+/**
+ * This is the primary interface between Sync and the Addons Manager.
+ *
+ * In addition to the core store APIs, we provide convenience functions to wrap
+ * Add-on Manager APIs with Sync-specific semantics.
+ */
+function AddonsStore(name, engine) {
+  Store.call(this, name, engine);
+}
+AddonsStore.prototype = {
+  __proto__: Store.prototype,
+
+  // Define the add-on types (.type) that we support.
+  _syncableTypes: ["extension", "theme"],
+
+  _extensionsPrefs: new Preferences("extensions."),
+
+  get reconciler() {
+    return this.engine._reconciler;
+  },
+
+  /**
+   * Override applyIncoming to filter out records we can't handle.
+   */
+  applyIncoming: function applyIncoming(record) {
+    // The fields we look at aren't present when the record is deleted.
+    if (!record.deleted) {
+      // Ignore records not belonging to our application ID because that is the
+      // current policy.
+      if (record.applicationID != Services.appinfo.ID) {
+        this._log.info("Ignoring incoming record from other App ID: " +
+                        record.id);
+        return;
+      }
+
+      // Ignore records that aren't from the official add-on repository, as that
+      // is our current policy.
+      if (record.source != "amo") {
+        this._log.info("Ignoring unknown add-on source (" + record.source + ")" +
+                       " for " + record.id);
+        return;
+      }
+    }
+
+    // Ignore incoming records for which an existing non-syncable addon
+    // exists.
+    let existingMeta = this.reconciler.addons[record.addonID];
+    if (existingMeta && !this.isAddonSyncable(existingMeta)) {
+      this._log.info("Ignoring incoming record for an existing but non-syncable addon", record.addonID);
+      return;
+    }
+
+    Store.prototype.applyIncoming.call(this, record);
+  },
+
+
+  /**
+   * Provides core Store API to create/install an add-on from a record.
+   */
+  create: function create(record) {
+    let cb = Async.makeSpinningCallback();
+    AddonUtils.installAddons([{
+      id:               record.addonID,
+      syncGUID:         record.id,
+      enabled:          record.enabled,
+      requireSecureURI: this._extensionsPrefs.get("install.requireSecureOrigin", true),
+    }], cb);
+
+    // This will throw if there was an error. This will get caught by the sync
+    // engine and the record will try to be applied later.
+    let results = cb.wait();
+
+    if (results.skipped.includes(record.addonID)) {
+      this._log.info("Add-on skipped: " + record.addonID);
+      // Just early-return for skipped addons - we don't want to arrange to
+      // try again next time because the condition that caused up to skip
+      // will remain true for this addon forever.
+      return;
+    }
+
+    let addon;
+    for (let a of results.addons) {
+      if (a.id == record.addonID) {
+        addon = a;
+        break;
+      }
+    }
+
+    // This should never happen, but is present as a fail-safe.
+    if (!addon) {
+      throw new Error("Add-on not found after install: " + record.addonID);
+    }
+
+    this._log.info("Add-on installed: " + record.addonID);
+  },
+
+  /**
+   * Provides core Store API to remove/uninstall an add-on from a record.
+   */
+  remove: function remove(record) {
+    // If this is called, the payload is empty, so we have to find by GUID.
+    let addon = this.getAddonByGUID(record.id);
+    if (!addon) {
+      // We don't throw because if the add-on could not be found then we assume
+      // it has already been uninstalled and there is nothing for this function
+      // to do.
+      return;
+    }
+
+    this._log.info("Uninstalling add-on: " + addon.id);
+    let cb = Async.makeSpinningCallback();
+    AddonUtils.uninstallAddon(addon, cb);
+    cb.wait();
+  },
+
+  /**
+   * Provides core Store API to update an add-on from a record.
+   */
+  update: function update(record) {
+    let addon = this.getAddonByID(record.addonID);
+
+    // update() is called if !this.itemExists. And, since itemExists consults
+    // the reconciler only, we need to take care of some corner cases.
+    //
+    // First, the reconciler could know about an add-on that was uninstalled
+    // and no longer present in the add-ons manager.
+    if (!addon) {
+      this.create(record);
+      return;
+    }
+
+    // It's also possible that the add-on is non-restartless and has pending
+    // install/uninstall activity.
+    //
+    // We wouldn't get here if the incoming record was for a deletion. So,
+    // check for pending uninstall and cancel if necessary.
+    if (addon.pendingOperations & AddonManager.PENDING_UNINSTALL) {
+      addon.cancelUninstall();
+
+      // We continue with processing because there could be state or ID change.
+    }
+
+    let cb = Async.makeSpinningCallback();
+    this.updateUserDisabled(addon, !record.enabled, cb);
+    cb.wait();
+  },
+
+  /**
+   * Provide core Store API to determine if a record exists.
+   */
+  itemExists: function itemExists(guid) {
+    let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+    return !!addon;
+  },
+
+  /**
+   * Create an add-on record from its GUID.
+   *
+   * @param guid
+   *        Add-on GUID (from extensions DB)
+   * @param collection
+   *        Collection to add record to.
+   *
+   * @return AddonRecord instance
+   */
+  createRecord: function createRecord(guid, collection) {
+    let record = new AddonRecord(collection, guid);
+    record.applicationID = Services.appinfo.ID;
+
+    let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+    // If we don't know about this GUID or if it has been uninstalled, we mark
+    // the record as deleted.
+    if (!addon || !addon.installed) {
+      record.deleted = true;
+      return record;
+    }
+
+    record.modified = addon.modified.getTime() / 1000;
+
+    record.addonID = addon.id;
+    record.enabled = addon.enabled;
+
+    // This needs to be dynamic when add-ons don't come from AddonRepository.
+    record.source = "amo";
+
+    return record;
+  },
+
+  /**
+   * Changes the id of an add-on.
+   *
+   * This implements a core API of the store.
+   */
+  changeItemID: function changeItemID(oldID, newID) {
+    // We always update the GUID in the reconciler because it will be
+    // referenced later in the sync process.
+    let state = this.reconciler.getAddonStateFromSyncGUID(oldID);
+    if (state) {
+      state.guid = newID;
+      let cb = Async.makeSpinningCallback();
+      this.reconciler.saveState(null, cb);
+      cb.wait();
+    }
+
+    let addon = this.getAddonByGUID(oldID);
+    if (!addon) {
+      this._log.debug("Cannot change item ID (" + oldID + ") in Add-on " +
+                      "Manager because old add-on not present: " + oldID);
+      return;
+    }
+
+    addon.syncGUID = newID;
+  },
+
+  /**
+   * Obtain the set of all syncable add-on Sync GUIDs.
+   *
+   * This implements a core Store API.
+   */
+  getAllIDs: function getAllIDs() {
+    let ids = {};
+
+    let addons = this.reconciler.addons;
+    for (let id in addons) {
+      let addon = addons[id];
+      if (this.isAddonSyncable(addon)) {
+        ids[addon.guid] = true;
+      }
+    }
+
+    return ids;
+  },
+
+  /**
+   * Wipe engine data.
+   *
+   * This uninstalls all syncable addons from the application. In case of
+   * error, it logs the error and keeps trying with other add-ons.
+   */
+  wipe: function wipe() {
+    this._log.info("Processing wipe.");
+
+    this.engine._refreshReconcilerState();
+
+    // We only wipe syncable add-ons. Wipe is a Sync feature not a security
+    // feature.
+    for (let guid in this.getAllIDs()) {
+      let addon = this.getAddonByGUID(guid);
+      if (!addon) {
+        this._log.debug("Ignoring add-on because it couldn't be obtained: " +
+                        guid);
+        continue;
+      }
+
+      this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
+      Utils.catch.call(this, () => addon.uninstall())();
+    }
+  },
+
+  /***************************************************************************
+   * Functions below are unique to this store and not part of the Store API  *
+   ***************************************************************************/
+
+  /**
+   * Synchronously obtain an add-on from its public ID.
+   *
+   * @param id
+   *        Add-on ID
+   * @return Addon or undefined if not found
+   */
+  getAddonByID: function getAddonByID(id) {
+    let cb = Async.makeSyncCallback();
+    AddonManager.getAddonByID(id, cb);
+    return Async.waitForSyncCallback(cb);
+  },
+
+  /**
+   * Synchronously obtain an add-on from its Sync GUID.
+   *
+   * @param  guid
+   *         Add-on Sync GUID
+   * @return DBAddonInternal or null
+   */
+  getAddonByGUID: function getAddonByGUID(guid) {
+    let cb = Async.makeSyncCallback();
+    AddonManager.getAddonBySyncGUID(guid, cb);
+    return Async.waitForSyncCallback(cb);
+  },
+
+  /**
+   * Determines whether an add-on is suitable for Sync.
+   *
+   * @param  addon
+   *         Addon instance
+   * @param ignoreRepoCheck
+   *         Should we skip checking the Addons repository (primarially useful
+   *         for testing and validation).
+   * @return Boolean indicating whether it is appropriate for Sync
+   */
+  isAddonSyncable: function isAddonSyncable(addon, ignoreRepoCheck = false) {
+    // Currently, we limit syncable add-ons to those that are:
+    //   1) In a well-defined set of types
+    //   2) Installed in the current profile
+    //   3) Not installed by a foreign entity (i.e. installed by the app)
+    //      since they act like global extensions.
+    //   4) Is not a hotfix.
+    //   5) The addons XPIProvider doesn't veto it (i.e not being installed in
+    //      the profile directory, or any other reasons it says the addon can't
+    //      be synced)
+    //   6) Are installed from AMO
+
+    // We could represent the test as a complex boolean expression. We go the
+    // verbose route so the failure reason is logged.
+    if (!addon) {
+      this._log.debug("Null object passed to isAddonSyncable.");
+      return false;
+    }
+
+    if (this._syncableTypes.indexOf(addon.type) == -1) {
+      this._log.debug(addon.id + " not syncable: type not in whitelist: " +
+                      addon.type);
+      return false;
+    }
+
+    if (!(addon.scope & AddonManager.SCOPE_PROFILE)) {
+      this._log.debug(addon.id + " not syncable: not installed in profile.");
+      return false;
+    }
+
+    // If the addon manager says it's not syncable, we skip it.
+    if (!addon.isSyncable) {
+      this._log.debug(addon.id + " not syncable: vetoed by the addon manager.");
+      return false;
+    }
+
+    // This may be too aggressive. If an add-on is downloaded from AMO and
+    // manually placed in the profile directory, foreignInstall will be set.
+    // Arguably, that add-on should be syncable.
+    // TODO Address the edge case and come up with more robust heuristics.
+    if (addon.foreignInstall) {
+      this._log.debug(addon.id + " not syncable: is foreign install.");
+      return false;
+    }
+
+    // Ignore hotfix extensions (bug 741670). The pref may not be defined.
+    // XXX - note that addon.isSyncable will be false for hotfix addons, so
+    // this check isn't strictly necessary - except for Sync tests which aren't
+    // setup to create a "real" hotfix addon. This can be removed once those
+    // tests are fixed (but keeping it doesn't hurt either)
+    if (this._extensionsPrefs.get("hotfix.id", null) == addon.id) {
+      this._log.debug(addon.id + " not syncable: is a hotfix.");
+      return false;
+    }
+
+    // If the AddonRepository's cache isn't enabled (which it typically isn't
+    // in tests), getCachedAddonByID always returns null - so skip the check
+    // in that case. We also provide a way to specifically opt-out of the check
+    // even if the cache is enabled, which is used by the validators.
+    if (ignoreRepoCheck || !AddonRepository.cacheEnabled) {
+      return true;
+    }
+
+    let cb = Async.makeSyncCallback();
+    AddonRepository.getCachedAddonByID(addon.id, cb);
+    let result = Async.waitForSyncCallback(cb);
+
+    if (!result) {
+      this._log.debug(addon.id + " not syncable: add-on not found in add-on " +
+                      "repository.");
+      return false;
+    }
+
+    return this.isSourceURITrusted(result.sourceURI);
+  },
+
+  /**
+   * Determine whether an add-on's sourceURI field is trusted and the add-on
+   * can be installed.
+   *
+   * This function should only ever be called from isAddonSyncable(). It is
+   * exposed as a separate function to make testing easier.
+   *
+   * @param  uri
+   *         nsIURI instance to validate
+   * @return bool
+   */
+  isSourceURITrusted: function isSourceURITrusted(uri) {
+    // For security reasons, we currently limit synced add-ons to those
+    // installed from trusted hostname(s). We additionally require TLS with
+    // the add-ons site to help prevent forgeries.
+    let trustedHostnames = Svc.Prefs.get("addons.trustedSourceHostnames", "")
+                           .split(",");
+
+    if (!uri) {
+      this._log.debug("Undefined argument to isSourceURITrusted().");
+      return false;
+    }
+
+    // Scheme is validated before the hostname because uri.host may not be
+    // populated for certain schemes. It appears to always be populated for
+    // https, so we avoid the potential NS_ERROR_FAILURE on field access.
+    if (uri.scheme != "https") {
+      this._log.debug("Source URI not HTTPS: " + uri.spec);
+      return false;
+    }
+
+    if (trustedHostnames.indexOf(uri.host) == -1) {
+      this._log.debug("Source hostname not trusted: " + uri.host);
+      return false;
+    }
+
+    return true;
+  },
+
+  /**
+   * Update the userDisabled flag on an add-on.
+   *
+   * This will enable or disable an add-on and call the supplied callback when
+   * the action is complete. If no action is needed, the callback gets called
+   * immediately.
+   *
+   * @param addon
+   *        Addon instance to manipulate.
+   * @param value
+   *        Boolean to which to set userDisabled on the passed Addon.
+   * @param callback
+   *        Function to be called when action is complete. Will receive 2
+   *        arguments, a truthy value that signifies error, and the Addon
+   *        instance passed to this function.
+   */
+  updateUserDisabled: function updateUserDisabled(addon, value, callback) {
+    if (addon.userDisabled == value) {
+      callback(null, addon);
+      return;
+    }
+
+    // A pref allows changes to the enabled flag to be ignored.
+    if (Svc.Prefs.get("addons.ignoreUserEnabledChanges", false)) {
+      this._log.info("Ignoring enabled state change due to preference: " +
+                     addon.id);
+      callback(null, addon);
+      return;
+    }
+
+    AddonUtils.updateUserDisabled(addon, value, callback);
+  },
+};
+
+/**
+ * The add-ons tracker keeps track of real-time changes to add-ons.
+ *
+ * It hooks up to the reconciler and receives notifications directly from it.
+ */
+function AddonsTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+AddonsTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  get reconciler() {
+    return this.engine._reconciler;
+  },
+
+  get store() {
+    return this.engine._store;
+  },
+
+  /**
+   * This callback is executed whenever the AddonsReconciler sends out a change
+   * notification. See AddonsReconciler.addChangeListener().
+   */
+  changeListener: function changeHandler(date, change, addon) {
+    this._log.debug("changeListener invoked: " + change + " " + addon.id);
+    // Ignore changes that occur during sync.
+    if (this.ignoreAll) {
+      return;
+    }
+
+    if (!this.store.isAddonSyncable(addon)) {
+      this._log.debug("Ignoring change because add-on isn't syncable: " +
+                      addon.id);
+      return;
+    }
+
+    this.addChangedID(addon.guid, date.getTime() / 1000);
+    this.score += SCORE_INCREMENT_XLARGE;
+  },
+
+  startTracking: function() {
+    if (this.engine.enabled) {
+      this.reconciler.startListening();
+    }
+
+    this.reconciler.addChangeListener(this);
+  },
+
+  stopTracking: function() {
+    this.reconciler.removeChangeListener(this);
+    this.reconciler.stopListening();
+  },
+};
+
+class AddonValidator extends CollectionValidator {
+  constructor(engine = null) {
+    super("addons", "id", [
+      "addonID",
+      "enabled",
+      "applicationID",
+      "source"
+    ]);
+    this.engine = engine;
+  }
+
+  getClientItems() {
+    return Promise.all([
+      new Promise(resolve =>
+        AddonManager.getAllAddons(resolve)),
+      new Promise(resolve =>
+        AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"], resolve)),
+    ]).then(([installed, addonsWithPendingOperation]) => {
+      // Addons pending install won't be in the first list, but addons pending
+      // uninstall/enable/disable will be in both lists.
+      let all = new Map(installed.map(addon => [addon.id, addon]));
+      for (let addon of addonsWithPendingOperation) {
+        all.set(addon.id, addon);
+      }
+      // Convert to an array since Map.prototype.values returns an iterable
+      return [...all.values()];
+    });
+  }
+
+  normalizeClientItem(item) {
+    let enabled = !item.userDisabled;
+    if (item.pendingOperations & AddonManager.PENDING_ENABLE) {
+      enabled = true;
+    } else if (item.pendingOperations & AddonManager.PENDING_DISABLE) {
+      enabled = false;
+    }
+    return {
+      enabled,
+      id: item.syncGUID,
+      addonID: item.id,
+      applicationID: Services.appinfo.ID,
+      source: "amo", // check item.foreignInstall?
+      original: item
+    };
+  }
+
+  normalizeServerItem(item) {
+    let guid = this.engine._findDupe(item);
+    if (guid) {
+      item.id = guid;
+    }
+    return item;
+  }
+
+  clientUnderstands(item) {
+    return item.applicationID === Services.appinfo.ID;
+  }
+
+  syncedByClient(item) {
+    return !item.original.hidden &&
+           !item.original.isSystem &&
+           !(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) &&
+           this.engine.isAddonSyncable(item.original, true);
+  }
+}

services/sync/modules/engines/clients.js

@@ -0,0 +1,782 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * How does the clients engine work?
+ *
+ * - We use 2 files - commands.json and commands-syncing.json.
+ *
+ * - At sync upload time, we attempt a rename of commands.json to
+ *   commands-syncing.json, and ignore errors (helps for crash during sync!).
+ * - We load commands-syncing.json and stash the contents in
+ *   _currentlySyncingCommands which lives for the duration of the upload process.
+ * - We use _currentlySyncingCommands to build the outgoing records
+ * - Immediately after successful upload, we delete commands-syncing.json from
+ *   disk (and clear _currentlySyncingCommands). We reconcile our local records
+ *   with what we just wrote in the server, and add failed IDs commands
+ *   back in commands.json
+ * - Any time we need to "save" a command for future syncs, we load
+ *   commands.json, update it, and write it back out.
+ */
+
+this.EXPORTED_SYMBOLS = [
+  "ClientEngine",
+  "ClientsRec"
+];
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/stringbundle.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/resource.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/Services.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
+  "resource://gre/modules/FxAccounts.jsm");
+
+const CLIENTS_TTL = 1814400; // 21 days
+const CLIENTS_TTL_REFRESH = 604800; // 7 days
+const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days
+
+const SUPPORTED_PROTOCOL_VERSIONS = ["1.1", "1.5"];
+
+function hasDupeCommand(commands, action) {
+  if (!commands) {
+    return false;
+  }
+  return commands.some(other => other.command == action.command &&
+    Utils.deepEquals(other.args, action.args));
+}
+
+this.ClientsRec = function ClientsRec(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+ClientsRec.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Sync.Record.Clients",
+  ttl: CLIENTS_TTL
+};
+
+Utils.deferGetSet(ClientsRec,
+                  "cleartext",
+                  ["name", "type", "commands",
+                   "version", "protocols",
+                   "formfactor", "os", "appPackage", "application", "device",
+                   "fxaDeviceId"]);
+
+
+this.ClientEngine = function ClientEngine(service) {
+  SyncEngine.call(this, "Clients", service);
+
+  // Reset the last sync timestamp on every startup so that we fetch all clients
+  this.resetLastSync();
+}
+ClientEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _storeObj: ClientStore,
+  _recordObj: ClientsRec,
+  _trackerObj: ClientsTracker,
+  allowSkippedRecord: false,
+
+  // Always sync client data as it controls other sync behavior
+  get enabled() {
+    return true;
+  },
+
+  get lastRecordUpload() {
+    return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
+  },
+  set lastRecordUpload(value) {
+    Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
+  },
+
+  get remoteClients() {
+    // return all non-stale clients for external consumption.
+    return Object.values(this._store._remoteClients).filter(v => !v.stale);
+  },
+
+  remoteClientExists(id) {
+    let client = this._store._remoteClients[id];
+    return !!(client && !client.stale);
+  },
+
+  // Aggregate some stats on the composition of clients on this account
+  get stats() {
+    let stats = {
+      hasMobile: this.localType == DEVICE_TYPE_MOBILE,
+      names: [this.localName],
+      numClients: 1,
+    };
+
+    for (let id in this._store._remoteClients) {
+      let {name, type, stale} = this._store._remoteClients[id];
+      if (!stale) {
+        stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE;
+        stats.names.push(name);
+        stats.numClients++;
+      }
+    }
+
+    return stats;
+  },
+
+  /**
+   * Obtain information about device types.
+   *
+   * Returns a Map of device types to integer counts.
+   */
+  get deviceTypes() {
+    let counts = new Map();
+
+    counts.set(this.localType, 1);
+
+    for (let id in this._store._remoteClients) {
+      let record = this._store._remoteClients[id];
+      if (record.stale) {
+        continue; // pretend "stale" records don't exist.
+      }
+      let type = record.type;
+      if (!counts.has(type)) {
+        counts.set(type, 0);
+      }
+
+      counts.set(type, counts.get(type) + 1);
+    }
+
+    return counts;
+  },
+
+  get localID() {
+    // Generate a random GUID id we don't have one
+    let localID = Svc.Prefs.get("client.GUID", "");
+    return localID == "" ? this.localID = Utils.makeGUID() : localID;
+  },
+  set localID(value) {
+    Svc.Prefs.set("client.GUID", value);
+  },
+
+  get brandName() {
+    let brand = new StringBundle("chrome://branding/locale/brand.properties");
+    return brand.get("brandShortName");
+  },
+
+  get localName() {
+    let name = Utils.getDeviceName();
+    // If `getDeviceName` returns the default name, set the pref. FxA registers
+    // the device before syncing, so we don't need to update the registration
+    // in this case.
+    Svc.Prefs.set("client.name", name);
+    return name;
+  },
+  set localName(value) {
+    Svc.Prefs.set("client.name", value);
+    // Update the registration in the background.
+    fxAccounts.updateDeviceRegistration().catch(error => {
+      this._log.warn("failed to update fxa device registration", error);
+    });
+  },
+
+  get localType() {
+    return Utils.getDeviceType();
+  },
+  set localType(value) {
+    Svc.Prefs.set("client.type", value);
+  },
+
+  getClientName(id) {
+    if (id == this.localID) {
+      return this.localName;
+    }
+    let client = this._store._remoteClients[id];
+    return client ? client.name : "";
+  },
+
+  getClientFxaDeviceId(id) {
+    if (this._store._remoteClients[id]) {
+      return this._store._remoteClients[id].fxaDeviceId;
+    }
+    return null;
+  },
+
+  isMobile: function isMobile(id) {
+    if (this._store._remoteClients[id])
+      return this._store._remoteClients[id].type == DEVICE_TYPE_MOBILE;
+    return false;
+  },
+
+  _readCommands() {
+    let cb = Async.makeSpinningCallback();
+    Utils.jsonLoad("commands", this, commands => cb(null, commands));
+    return cb.wait() || {};
+  },
+
+  /**
+   * Low level function, do not use directly (use _addClientCommand instead).
+   */
+  _saveCommands(commands) {
+    let cb = Async.makeSpinningCallback();
+    Utils.jsonSave("commands", this, commands, error => {
+      if (error) {
+        this._log.error("Failed to save JSON outgoing commands", error);
+      }
+      cb();
+    });
+    cb.wait();
+  },
+
+  _prepareCommandsForUpload() {
+    let cb = Async.makeSpinningCallback();
+    Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors
+      .then(() => {
+        Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands));
+      });
+    return cb.wait() || {};
+  },
+
+  _deleteUploadedCommands() {
+    delete this._currentlySyncingCommands;
+    Async.promiseSpinningly(
+      Utils.jsonRemove("commands-syncing", this).catch(err => {
+        this._log.error("Failed to delete syncing-commands file", err);
+      })
+    );
+  },
+
+  _addClientCommand(clientId, command) {
+    const allCommands = this._readCommands();
+    const clientCommands = allCommands[clientId] || [];
+    if (hasDupeCommand(clientCommands, command)) {
+      return;
+    }
+    allCommands[clientId] = clientCommands.concat(command);
+    this._saveCommands(allCommands);
+  },
+
+  _syncStartup: function _syncStartup() {
+    // Reupload new client record periodically.
+    if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
+      this._tracker.addChangedID(this.localID);
+      this.lastRecordUpload = Date.now() / 1000;
+    }
+    SyncEngine.prototype._syncStartup.call(this);
+  },
+
+  _processIncoming() {
+    // Fetch all records from the server.
+    this.lastSync = 0;
+    this._incomingClients = {};
+    try {
+      SyncEngine.prototype._processIncoming.call(this);
+      // Since clients are synced unconditionally, any records in the local store
+      // that don't exist on the server must be for disconnected clients. Remove
+      // them, so that we don't upload records with commands for clients that will
+      // never see them. We also do this to filter out stale clients from the
+      // tabs collection, since showing their list of tabs is confusing.
+      for (let id in this._store._remoteClients) {
+        if (!this._incomingClients[id]) {
+          this._log.info(`Removing local state for deleted client ${id}`);
+          this._removeRemoteClient(id);
+        }
+      }
+      // Bug 1264498: Mobile clients don't remove themselves from the clients
+      // collection when the user disconnects Sync, so we mark as stale clients
+      // with the same name that haven't synced in over a week.
+      // (Note we can't simply delete them, or we re-apply them next sync - see
+      // bug 1287687)
+      delete this._incomingClients[this.localID];
+      let names = new Set([this.localName]);
+      for (let id in this._incomingClients) {
+        let record = this._store._remoteClients[id];
+        if (!names.has(record.name)) {
+          names.add(record.name);
+          continue;
+        }
+        let remoteAge = AsyncResource.serverTime - this._incomingClients[id];
+        if (remoteAge > STALE_CLIENT_REMOTE_AGE) {
+          this._log.info(`Hiding stale client ${id} with age ${remoteAge}`);
+          record.stale = true;
+        }
+      }
+    } finally {
+      this._incomingClients = null;
+    }
+  },
+
+  _uploadOutgoing() {
+    this._currentlySyncingCommands = this._prepareCommandsForUpload();
+    const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands);
+    for (let clientId of clientWithPendingCommands) {
+      if (this._store._remoteClients[clientId] || this.localID == clientId) {
+        this._modified.set(clientId, 0);
+      }
+    }
+    SyncEngine.prototype._uploadOutgoing.call(this);
+  },
+
+  _onRecordsWritten(succeeded, failed) {
+    // Reconcile the status of the local records with what we just wrote on the
+    // server
+    for (let id of succeeded) {
+      const commandChanges = this._currentlySyncingCommands[id];
+      if (id == this.localID) {
+        if (this.localCommands) {
+          this.localCommands = this.localCommands.filter(command => !hasDupeCommand(commandChanges, command));
+        }
+      } else {
+        const clientRecord = this._store._remoteClients[id];
+        if (!commandChanges || !clientRecord) {
+          // should be impossible, else we wouldn't have been writing it.
+          this._log.warn("No command/No record changes for a client we uploaded");
+          continue;
+        }
+        // fixup the client record, so our copy of _remoteClients matches what we uploaded.
+        clientRecord.commands = this._store.createRecord(id);
+        // we could do better and pass the reference to the record we just uploaded,
+        // but this will do for now
+      }
+    }
+
+    // Re-add failed commands
+    for (let id of failed) {
+      const commandChanges = this._currentlySyncingCommands[id];
+      if (!commandChanges) {
+        continue;
+      }
+      this._addClientCommand(id, commandChanges);
+    }
+
+    this._deleteUploadedCommands();
+
+    // Notify other devices that their own client collection changed
+    const idsToNotify = succeeded.reduce((acc, id) => {
+      if (id == this.localID) {
+        return acc;
+      }
+      const fxaDeviceId = this.getClientFxaDeviceId(id);
+      return fxaDeviceId ? acc.concat(fxaDeviceId) : acc;
+    }, []);
+    if (idsToNotify.length > 0) {
+      this._notifyCollectionChanged(idsToNotify);
+    }
+  },
+
+  _notifyCollectionChanged(ids) {
+    const message = {
+      version: 1,
+      command: "sync:collection_changed",
+      data: {
+        collections: ["clients"]
+      }
+    };
+    fxAccounts.notifyDevices(ids, message, NOTIFY_TAB_SENT_TTL_SECS);
+  },
+
+  _syncFinish() {
+    // Record histograms for our device types, and also write them to a pref
+    // so non-histogram telemetry (eg, UITelemetry) has easy access to them.
+    for (let [deviceType, count] of this.deviceTypes) {
+      let hid;
+      let prefName = this.name + ".devices.";
+      switch (deviceType) {
+        case "desktop":
+          hid = "WEAVE_DEVICE_COUNT_DESKTOP";
+          prefName += "desktop";
+          break;
+        case "mobile":
+          hid = "WEAVE_DEVICE_COUNT_MOBILE";
+          prefName += "mobile";
+          break;
+        default:
+          this._log.warn(`Unexpected deviceType "${deviceType}" recording device telemetry.`);
+          continue;
+      }
+      Services.telemetry.getHistogramById(hid).add(count);
+      Svc.Prefs.set(prefName, count);
+    }
+    SyncEngine.prototype._syncFinish.call(this);
+  },
+
+  _reconcile: function _reconcile(item) {
+    // Every incoming record is reconciled, so we use this to track the
+    // contents of the collection on the server.
+    this._incomingClients[item.id] = item.modified;
+
+    if (!this._store.itemExists(item.id)) {
+      return true;
+    }
+    // Clients are synced unconditionally, so we'll always have new records.
+    // Unfortunately, this will cause the scheduler to use the immediate sync
+    // interval for the multi-device case, instead of the active interval. We
+    // work around this by updating the record during reconciliation, and
+    // returning false to indicate that the record doesn't need to be applied
+    // later.
+    this._store.update(item);
+    return false;
+  },
+
+  // Treat reset the same as wiping for locally cached clients
+  _resetClient() {
+    this._wipeClient();
+  },
+
+  _wipeClient: function _wipeClient() {
+    SyncEngine.prototype._resetClient.call(this);
+    delete this.localCommands;
+    this._store.wipe();
+    const logRemoveError = err => this._log.warn("Could not delete json file", err);
+    Async.promiseSpinningly(
+      Utils.jsonRemove("commands", this).catch(logRemoveError)
+        .then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError))
+    );
+  },
+
+  removeClientData: function removeClientData() {
+    let res = this.service.resource(this.engineURL + "/" + this.localID);
+    res.delete();
+  },
+
+  // Override the default behavior to delete bad records from the server.
+  handleHMACMismatch: function handleHMACMismatch(item, mayRetry) {
+    this._log.debug("Handling HMAC mismatch for " + item.id);
+
+    let base = SyncEngine.prototype.handleHMACMismatch.call(this, item, mayRetry);
+    if (base != SyncEngine.kRecoveryStrategy.error)
+      return base;
+
+    // It's a bad client record. Save it to be deleted at the end of the sync.
+    this._log.debug("Bad client record detected. Scheduling for deletion.");
+    this._deleteId(item.id);
+
+    // Neither try again nor error; we're going to delete it.
+    return SyncEngine.kRecoveryStrategy.ignore;
+  },
+
+  /**
+   * A hash of valid commands that the client knows about. The key is a command
+   * and the value is a hash containing information about the command such as
+   * number of arguments and description.
+   */
+  _commands: {
+    resetAll:    { args: 0, desc: "Clear temporary local data for all engines" },
+    resetEngine: { args: 1, desc: "Clear temporary local data for engine" },
+    wipeAll:     { args: 0, desc: "Delete all client data for all engines" },
+    wipeEngine:  { args: 1, desc: "Delete all client data for engine" },
+    logout:      { args: 0, desc: "Log out client" },
+    displayURI:  { args: 3, desc: "Instruct a client to display a URI" },
+  },
+
+  /**
+   * Sends a command+args pair to a specific client.
+   *
+   * @param command Command string
+   * @param args Array of arguments/data for command
+   * @param clientId Client to send command to
+   */
+  _sendCommandToClient: function sendCommandToClient(command, args, clientId) {
+    this._log.trace("Sending " + command + " to " + clientId);
+
+    let client = this._store._remoteClients[clientId];
+    if (!client) {
+      throw new Error("Unknown remote client ID: '" + clientId + "'.");
+    }
+    if (client.stale) {
+      throw new Error("Stale remote client ID: '" + clientId + "'.");
+    }
+
+    let action = {
+      command: command,
+      args: args,
+    };
+
+    this._log.trace("Client " + clientId + " got a new action: " + [command, args]);
+    this._addClientCommand(clientId, action);
+    this._tracker.addChangedID(clientId);
+  },
+
+  /**
+   * Check if the local client has any remote commands and perform them.
+   *
+   * @return false to abort sync
+   */
+  processIncomingCommands: function processIncomingCommands() {
+    return this._notify("clients:process-commands", "", function() {
+      if (!this.localCommands) {
+        return true;
+      }
+
+      const clearedCommands = this._readCommands()[this.localID];
+      const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command));
+
+      let URIsToDisplay = [];
+      // Process each command in order.
+      for (let rawCommand of commands) {
+        let {command, args} = rawCommand;
+        this._log.debug("Processing command: " + command + "(" + args + ")");
+
+        let engines = [args[0]];
+        switch (command) {
+          case "resetAll":
+            engines = null;
+            // Fallthrough
+          case "resetEngine":
+            this.service.resetClient(engines);
+            break;
+          case "wipeAll":
+            engines = null;
+            // Fallthrough
+          case "wipeEngine":
+            this.service.wipeClient(engines);
+            break;
+          case "logout":
+            this.service.logout();
+            return false;
+          case "displayURI":
+            let [uri, clientId, title] = args;
+            URIsToDisplay.push({ uri, clientId, title });
+            break;
+          default:
+            this._log.debug("Received an unknown command: " + command);
+            break;
+        }
+        // Add the command to the "cleared" commands list
+        this._addClientCommand(this.localID, rawCommand)
+      }
+      this._tracker.addChangedID(this.localID);
+
+      if (URIsToDisplay.length) {
+        this._handleDisplayURIs(URIsToDisplay);
+      }
+
+      return true;
+    })();
+  },
+
+  /**
+   * Validates and sends a command to a client or all clients.
+   *
+   * Calling this does not actually sync the command data to the server. If the
+   * client already has the command/args pair, it won't receive a duplicate
+   * command.
+   *
+   * @param command
+   *        Command to invoke on remote clients
+   * @param args
+   *        Array of arguments to give to the command
+   * @param clientId
+   *        Client ID to send command to. If undefined, send to all remote
+   *        clients.
+   */
+  sendCommand: function sendCommand(command, args, clientId) {
+    let commandData = this._commands[command];
+    // Don't send commands that we don't know about.
+    if (!commandData) {
+      this._log.error("Unknown command to send: " + command);
+      return;
+    }
+    // Don't send a command with the wrong number of arguments.
+    else if (!args || args.length != commandData.args) {
+      this._log.error("Expected " + commandData.args + " args for '" +
+                      command + "', but got " + args);
+      return;
+    }
+
+    if (clientId) {
+      this._sendCommandToClient(command, args, clientId);
+    } else {
+      for (let [id, record] of Object.entries(this._store._remoteClients)) {
+        if (!record.stale) {
+          this._sendCommandToClient(command, args, id);
+        }
+      }
+    }
+  },
+
+  /**
+   * Send a URI to another client for display.
+   *
+   * A side effect is the score is increased dramatically to incur an
+   * immediate sync.
+   *
+   * If an unknown client ID is specified, sendCommand() will throw an
+   * Error object.
+   *
+   * @param uri
+   *        URI (as a string) to send and display on the remote client
+   * @param clientId
+   *        ID of client to send the command to. If not defined, will be sent
+   *        to all remote clients.
+   * @param title
+   *        Title of the page being sent.
+   */
+  sendURIToClientForDisplay: function sendURIToClientForDisplay(uri, clientId, title) {
+    this._log.info("Sending URI to client: " + uri + " -> " +
+                   clientId + " (" + title + ")");
+    this.sendCommand("displayURI", [uri, this.localID, title], clientId);
+
+    this._tracker.score += SCORE_INCREMENT_XLARGE;
+  },
+
+  /**
+   * Handle a bunch of received 'displayURI' commands.
+   *
+   * Interested parties should observe the "weave:engine:clients:display-uris"
+   * topic. The callback will receive an array as the subject parameter
+   * containing objects with the following keys:
+   *
+   *   uri       URI (string) that is requested for display.
+   *   clientId  ID of client that sent the command.
+   *   title     Title of page that loaded URI (likely) corresponds to.
+   *
+   * The 'data' parameter to the callback will not be defined.
+   *
+   * @param uris
+   *        An array containing URI objects to display
+   * @param uris[].uri
+   *        String URI that was received
+   * @param uris[].clientId
+   *        ID of client that sent URI
+   * @param uris[].title
+   *        String title of page that URI corresponds to. Older clients may not
+   *        send this.
+   */
+  _handleDisplayURIs: function _handleDisplayURIs(uris) {
+    Svc.Obs.notify("weave:engine:clients:display-uris", uris);
+  },
+
+  _removeRemoteClient(id) {
+    delete this._store._remoteClients[id];
+    this._tracker.removeChangedID(id);
+  },
+};
+
+function ClientStore(name, engine) {
+  Store.call(this, name, engine);
+}
+ClientStore.prototype = {
+  __proto__: Store.prototype,
+
+  _remoteClients: {},
+
+  create(record) {
+    this.update(record);
+  },
+
+  update: function update(record) {
+    if (record.id == this.engine.localID) {
+      // Only grab commands from the server; local name/type always wins
+      this.engine.localCommands = record.commands;
+    } else {
+      this._remoteClients[record.id] = record.cleartext;
+    }
+  },
+
+  createRecord: function createRecord(id, collection) {
+    let record = new ClientsRec(collection, id);
+
+    const commandsChanges = this.engine._currentlySyncingCommands ?
+                            this.engine._currentlySyncingCommands[id] :
+                            [];
+
+    // Package the individual components into a record for the local client
+    if (id == this.engine.localID) {
+      let cb = Async.makeSpinningCallback();
+      fxAccounts.getDeviceId().then(id => cb(null, id), cb);
+      try {
+        record.fxaDeviceId = cb.wait();
+      } catch(error) {
+        this._log.warn("failed to get fxa device id", error);
+      }
+      record.name = this.engine.localName;
+      record.type = this.engine.localType;
+      record.version = Services.appinfo.version;
+      record.protocols = SUPPORTED_PROTOCOL_VERSIONS;
+
+      // Substract the commands we recorded that we've already executed
+      if (commandsChanges && commandsChanges.length &&
+          this.engine.localCommands && this.engine.localCommands.length) {
+        record.commands = this.engine.localCommands.filter(command => !hasDupeCommand(commandsChanges, command));
+      }
+
+      // Optional fields.
+      record.os = Services.appinfo.OS;             // "Darwin"
+      record.appPackage = Services.appinfo.ID;
+      record.application = this.engine.brandName   // "Nightly"
+
+      // We can't compute these yet.
+      // record.device = "";            // Bug 1100723
+      // record.formfactor = "";        // Bug 1100722
+    } else {
+      record.cleartext = this._remoteClients[id];
+
+      // Add the commands we have to send
+      if (commandsChanges && commandsChanges.length) {
+        const recordCommands = record.cleartext.commands || [];
+        const newCommands = commandsChanges.filter(command => !hasDupeCommand(recordCommands, command));
+        record.cleartext.commands = recordCommands.concat(newCommands);
+      }
+
+      if (record.cleartext.stale) {
+        // It's almost certainly a logic error for us to upload a record we
+        // consider stale, so make log noise, but still remove the flag.
+        this._log.error(`Preparing to upload record ${id} that we consider stale`);
+        delete record.cleartext.stale;
+      }
+    }
+
+    return record;
+  },
+
+  itemExists(id) {
+    return id in this.getAllIDs();
+  },
+
+  getAllIDs: function getAllIDs() {
+    let ids = {};
+    ids[this.engine.localID] = true;
+    for (let id in this._remoteClients)
+      ids[id] = true;
+    return ids;
+  },
+
+  wipe: function wipe() {
+    this._remoteClients = {};
+  },
+};
+
+function ClientsTracker(name, engine) {
+  Tracker.call(this, name, engine);
+  Svc.Obs.add("weave:engine:start-tracking", this);
+  Svc.Obs.add("weave:engine:stop-tracking", this);
+}
+ClientsTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  _enabled: false,
+
+  observe: function observe(subject, topic, data) {
+    switch (topic) {
+      case "weave:engine:start-tracking":
+        if (!this._enabled) {
+          Svc.Prefs.observe("client.name", this);
+          this._enabled = true;
+        }
+        break;
+      case "weave:engine:stop-tracking":
+        if (this._enabled) {
+          Svc.Prefs.ignore("client.name", this);
+          this._enabled = false;
+        }
+        break;
+      case "nsPref:changed":
+        this._log.debug("client.name preference changed");
+        this.addChangedID(Svc.Prefs.get("client.GUID"));
+        this.score += SCORE_INCREMENT_XLARGE;
+        break;
+    }
+  }
+};

services/sync/modules/engines/extension-storage.js

@@ -0,0 +1,277 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine', 'EncryptionRemoteTransformer',
+                         'KeyRingEncryptionRemoteTransformer'];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/keys.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
+                                  "resource://gre/modules/ExtensionStorageSync.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
+                                  "resource://gre/modules/FxAccounts.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Task",
+                                  "resource://gre/modules/Task.jsm");
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
+  SyncEngine.call(this, "Extension-Storage", service);
+};
+ExtensionStorageEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _trackerObj: ExtensionStorageTracker,
+  // we don't need these since we implement our own sync logic
+  _storeObj: undefined,
+  _recordObj: undefined,
+
+  syncPriority: 10,
+  allowSkippedRecord: false,
+
+  _sync: function () {
+    return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
+  },
+
+  get enabled() {
+    // By default, we sync extension storage if we sync addons. This
+    // lets us simplify the UX since users probably don't consider
+    // "extension preferences" a separate category of syncing.
+    // However, we also respect engine.extension-storage.force, which
+    // can be set to true or false, if a power user wants to customize
+    // the behavior despite the lack of UI.
+    const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
+    if (forced !== undefined) {
+      return forced;
+    }
+    return Svc.Prefs.get("engine.addons", false);
+  },
+};
+
+function ExtensionStorageTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+ExtensionStorageTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function () {
+    Svc.Obs.add("ext.storage.sync-changed", this);
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("ext.storage.sync-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    if (this.ignoreAll) {
+      return;
+    }
+
+    if (topic !== "ext.storage.sync-changed") {
+      return;
+    }
+
+    // Single adds, removes and changes are not so important on their
+    // own, so let's just increment score a bit.
+    this.score += SCORE_INCREMENT_MEDIUM;
+  },
+
+  // Override a bunch of methods which don't do anything for us.
+  // This is a performance hack.
+  saveChangedIDs: function() {
+  },
+  loadChangedIDs: function() {
+  },
+  ignoreID: function() {
+  },
+  unignoreID: function() {
+  },
+  addChangedID: function() {
+  },
+  removeChangedID: function() {
+  },
+  clearChangedIDs: function() {
+  },
+};
+
+/**
+ * Utility function to enforce an order of fields when computing an HMAC.
+ */
+function ciphertextHMAC(keyBundle, id, IV, ciphertext) {
+  const hasher = keyBundle.sha256HMACHasher;
+  return Utils.bytesAsHex(Utils.digestUTF8(id + IV + ciphertext, hasher));
+}
+
+/**
+ * A "remote transformer" that the Kinto library will use to
+ * encrypt/decrypt records when syncing.
+ *
+ * This is an "abstract base class". Subclass this and override
+ * getKeys() to use it.
+ */
+class EncryptionRemoteTransformer {
+  encode(record) {
+    const self = this;
+    return Task.spawn(function* () {
+      const keyBundle = yield self.getKeys();
+      if (record.ciphertext) {
+        throw new Error("Attempt to reencrypt??");
+      }
+      let id = record.id;
+      if (!record.id) {
+        throw new Error("Record ID is missing or invalid");
+      }
+
+      let IV = Svc.Crypto.generateRandomIV();
+      let ciphertext = Svc.Crypto.encrypt(JSON.stringify(record),
+                                          keyBundle.encryptionKeyB64, IV);
+      let hmac = ciphertextHMAC(keyBundle, id, IV, ciphertext);
+      const encryptedResult = {ciphertext, IV, hmac, id};
+      if (record.hasOwnProperty("last_modified")) {
+        encryptedResult.last_modified = record.last_modified;
+      }
+      return encryptedResult;
+    });
+  }
+
+  decode(record) {
+    const self = this;
+    return Task.spawn(function* () {
+      if (!record.ciphertext) {
+        // This can happen for tombstones if a record is deleted.
+        if (record.deleted) {
+          return record;
+        }
+        throw new Error("No ciphertext: nothing to decrypt?");
+      }
+      const keyBundle = yield self.getKeys();
+      // Authenticate the encrypted blob with the expected HMAC
+      let computedHMAC = ciphertextHMAC(keyBundle, record.id, record.IV, record.ciphertext);
+
+      if (computedHMAC != record.hmac) {
+        Utils.throwHMACMismatch(record.hmac, computedHMAC);
+      }
+
+      // Handle invalid data here. Elsewhere we assume that cleartext is an object.
+      let cleartext = Svc.Crypto.decrypt(record.ciphertext,
+                                         keyBundle.encryptionKeyB64, record.IV);
+      let jsonResult = JSON.parse(cleartext);
+      if (!jsonResult || typeof jsonResult !== "object") {
+        throw new Error("Decryption failed: result is <" + jsonResult + ">, not an object.");
+      }
+
+      // Verify that the encrypted id matches the requested record's id.
+      // This should always be true, because we compute the HMAC over
+      // the original record's ID, and that was verified already (above).
+      if (jsonResult.id != record.id) {
+        throw new Error("Record id mismatch: " + jsonResult.id + " != " + record.id);
+      }
+
+      if (record.hasOwnProperty("last_modified")) {
+        jsonResult.last_modified = record.last_modified;
+      }
+
+      return jsonResult;
+    });
+  }
+
+  /**
+   * Retrieve keys to use during encryption.
+   *
+   * Returns a Promise<KeyBundle>.
+   */
+  getKeys() {
+    throw new Error("override getKeys in a subclass");
+  }
+}
+// You can inject this
+EncryptionRemoteTransformer.prototype._fxaService = fxAccounts;
+
+/**
+ * An EncryptionRemoteTransformer that provides a keybundle derived
+ * from the user's kB, suitable for encrypting a keyring.
+ */
+class KeyRingEncryptionRemoteTransformer extends EncryptionRemoteTransformer {
+  getKeys() {
+    const self = this;
+    return Task.spawn(function* () {
+      const user = yield self._fxaService.getSignedInUser();
+      // FIXME: we should permit this if the user is self-hosting
+      // their storage
+      if (!user) {
+        throw new Error("user isn't signed in to FxA; can't sync");
+      }
+
+      if (!user.kB) {
+        throw new Error("user doesn't have kB");
+      }
+
+      let kB = Utils.hexToBytes(user.kB);
+
+      let keyMaterial = CryptoUtils.hkdf(kB, undefined,
+                                       "identity.mozilla.com/picl/v1/chrome.storage.sync", 2*32);
+      let bundle = new BulkKeyBundle();
+      // [encryptionKey, hmacKey]
+      bundle.keyPair = [keyMaterial.slice(0, 32), keyMaterial.slice(32, 64)];
+      return bundle;
+    });
+  }
+  // Pass through the kbHash field from the unencrypted record. If
+  // encryption fails, we can use this to try to detect whether we are
+  // being compromised or if the record here was encoded with a
+  // different kB.
+  encode(record) {
+    const encodePromise = super.encode(record);
+    return Task.spawn(function* () {
+      const encoded = yield encodePromise;
+      encoded.kbHash = record.kbHash;
+      return encoded;
+    });
+  }
+
+  decode(record) {
+    const decodePromise = super.decode(record);
+    return Task.spawn(function* () {
+      try {
+        return yield decodePromise;
+      } catch (e) {
+        if (Utils.isHMACMismatch(e)) {
+          const currentKBHash = yield ExtensionStorageSync.getKBHash();
+          if (record.kbHash != currentKBHash) {
+            // Some other client encoded this with a kB that we don't
+            // have access to.
+            KeyRingEncryptionRemoteTransformer.throwOutdatedKB(currentKBHash, record.kbHash);
+          }
+        }
+        throw e;
+      }
+    });
+  }
+
+  // Generator and discriminator for KB-is-outdated exceptions.
+  static throwOutdatedKB(shouldBe, is) {
+    throw new Error(`kB hash on record is outdated: should be ${shouldBe}, is ${is}`);
+  }
+
+  static isOutdatedKB(exc) {
+    const kbMessage = "kB hash on record is outdated: ";
+    return exc && exc.message && exc.message.indexOf &&
+      (exc.message.indexOf(kbMessage) == 0);
+  }
+}

services/sync/modules/engines/forms.js

@@ -0,0 +1,305 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec', 'FormValidator'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/collection_validator.js");
+Cu.import("resource://gre/modules/Log.jsm");
+
+const FORMS_TTL = 3 * 365 * 24 * 60 * 60;   // Three years in seconds.
+
+this.FormRec = function FormRec(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+FormRec.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Sync.Record.Form",
+  ttl: FORMS_TTL
+};
+
+Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
+
+
+var FormWrapper = {
+  _log: Log.repository.getLogger("Sync.Engine.Forms"),
+
+  _getEntryCols: ["fieldname", "value"],
+  _guidCols:     ["guid"],
+
+  _promiseSearch: function(terms, searchData) {
+    return new Promise(resolve => {
+      let results = [];
+      let callbacks = {
+        handleResult(result) {
+          results.push(result);
+        },
+        handleCompletion(reason) {
+          resolve(results);
+        }
+      };
+      Svc.FormHistory.search(terms, searchData, callbacks);
+    })
+  },
+
+  // Do a "sync" search by spinning the event loop until it completes.
+  _searchSpinningly: function(terms, searchData) {
+    return Async.promiseSpinningly(this._promiseSearch(terms, searchData));
+  },
+
+  _updateSpinningly: function(changes) {
+    if (!Svc.FormHistory.enabled) {
+      return; // update isn't going to do anything.
+    }
+    let cb = Async.makeSpinningCallback();
+    let callbacks = {
+      handleCompletion: function(reason) {
+        cb();
+      }
+    };
+    Svc.FormHistory.update(changes, callbacks);
+    return cb.wait();
+  },
+
+  getEntry: function (guid) {
+    let results = this._searchSpinningly(this._getEntryCols, {guid: guid});
+    if (!results.length) {
+      return null;
+    }
+    return {name: results[0].fieldname, value: results[0].value};
+  },
+
+  getGUID: function (name, value) {
+    // Query for the provided entry.
+    let query = { fieldname: name, value: value };
+    let results = this._searchSpinningly(this._guidCols, query);
+    return results.length ? results[0].guid : null;
+  },
+
+  hasGUID: function (guid) {
+    // We could probably use a count function here, but searchSpinningly exists...
+    return this._searchSpinningly(this._guidCols, {guid: guid}).length != 0;
+  },
+
+  replaceGUID: function (oldGUID, newGUID) {
+    let changes = {
+      op: "update",
+      guid: oldGUID,
+      newGuid: newGUID,
+    }
+    this._updateSpinningly(changes);
+  }
+
+};
+
+this.FormEngine = function FormEngine(service) {
+  SyncEngine.call(this, "Forms", service);
+}
+FormEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _storeObj: FormStore,
+  _trackerObj: FormTracker,
+  _recordObj: FormRec,
+  applyIncomingBatchSize: FORMS_STORE_BATCH_SIZE,
+
+  syncPriority: 6,
+
+  get prefName() {
+    return "history";
+  },
+
+  _findDupe: function _findDupe(item) {
+    return FormWrapper.getGUID(item.name, item.value);
+  }
+};
+
+function FormStore(name, engine) {
+  Store.call(this, name, engine);
+}
+FormStore.prototype = {
+  __proto__: Store.prototype,
+
+  _processChange: function (change) {
+    // If this._changes is defined, then we are applying a batch, so we
+    // can defer it.
+    if (this._changes) {
+      this._changes.push(change);
+      return;
+    }
+
+    // Otherwise we must handle the change synchronously, right now.
+    FormWrapper._updateSpinningly(change);
+  },
+
+  applyIncomingBatch: function (records) {
+    // We collect all the changes to be made then apply them all at once.
+    this._changes = [];
+    let failures = Store.prototype.applyIncomingBatch.call(this, records);
+    if (this._changes.length) {
+      FormWrapper._updateSpinningly(this._changes);
+    }
+    delete this._changes;
+    return failures;
+  },
+
+  getAllIDs: function () {
+    let results = FormWrapper._searchSpinningly(["guid"], [])
+    let guids = {};
+    for (let result of results) {
+      guids[result.guid] = true;
+    }
+    return guids;
+  },
+
+  changeItemID: function (oldID, newID) {
+    FormWrapper.replaceGUID(oldID, newID);
+  },
+
+  itemExists: function (id) {
+    return FormWrapper.hasGUID(id);
+  },
+
+  createRecord: function (id, collection) {
+    let record = new FormRec(collection, id);
+    let entry = FormWrapper.getEntry(id);
+    if (entry != null) {
+      record.name = entry.name;
+      record.value = entry.value;
+    } else {
+      record.deleted = true;
+    }
+    return record;
+  },
+
+  create: function (record) {
+    this._log.trace("Adding form record for " + record.name);
+    let change = {
+      op: "add",
+      fieldname: record.name,
+      value: record.value
+    };
+    this._processChange(change);
+  },
+
+  remove: function (record) {
+    this._log.trace("Removing form record: " + record.id);
+    let change = {
+      op: "remove",
+      guid: record.id
+    };
+    this._processChange(change);
+  },
+
+  update: function (record) {
+    this._log.trace("Ignoring form record update request!");
+  },
+
+  wipe: function () {
+    let change = {
+      op: "remove"
+    };
+    FormWrapper._updateSpinningly(change);
+  }
+};
+
+function FormTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+FormTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  QueryInterface: XPCOMUtils.generateQI([
+    Ci.nsIObserver,
+    Ci.nsISupportsWeakReference]),
+
+  startTracking: function() {
+    Svc.Obs.add("satchel-storage-changed", this);
+  },
+
+  stopTracking: function() {
+    Svc.Obs.remove("satchel-storage-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+    if (this.ignoreAll) {
+      return;
+    }
+    switch (topic) {
+      case "satchel-storage-changed":
+        if (data == "formhistory-add" || data == "formhistory-remove") {
+          let guid = subject.QueryInterface(Ci.nsISupportsString).toString();
+          this.trackEntry(guid);
+        }
+        break;
+    }
+  },
+
+  trackEntry: function (guid) {
+    this.addChangedID(guid);
+    this.score += SCORE_INCREMENT_MEDIUM;
+  },
+};
+
+
+class FormsProblemData extends CollectionProblemData {
+  getSummary() {
+    // We don't support syncing deleted form data, so "clientMissing" isn't a problem
+    return super.getSummary().filter(entry =>
+      entry.name !== "clientMissing");
+  }
+}
+
+class FormValidator extends CollectionValidator {
+  constructor() {
+    super("forms", "id", ["name", "value"]);
+  }
+
+  emptyProblemData() {
+    return new FormsProblemData();
+  }
+
+  getClientItems() {
+    return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {});
+  }
+
+  normalizeClientItem(item) {
+    return {
+      id: item.guid,
+      guid: item.guid,
+      name: item.fieldname,
+      fieldname: item.fieldname,
+      value: item.value,
+      original: item,
+    };
+  }
+
+  normalizeServerItem(item) {
+    let res = Object.assign({
+      guid: item.id,
+      fieldname: item.name,
+      original: item,
+    }, item);
+    // Missing `name` or `value` causes the getGUID call to throw
+    if (item.name !== undefined && item.value !== undefined) {
+      let guid = FormWrapper.getGUID(item.name, item.value);
+      if (guid) {
+        res.guid = guid;
+        res.id = guid;
+        res.duped = true;
+      }
+    }
+
+    return res;
+  }
+}

services/sync/modules/engines/history.js

@@ -0,0 +1,442 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['HistoryEngine', 'HistoryRec'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+var Cr = Components.results;
+
+const HISTORY_TTL = 5184000; // 60 days
+
+Cu.import("resource://gre/modules/PlacesUtils.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+
+this.HistoryRec = function HistoryRec(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+HistoryRec.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Sync.Record.History",
+  ttl: HISTORY_TTL
+};
+
+Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]);
+
+
+this.HistoryEngine = function HistoryEngine(service) {
+  SyncEngine.call(this, "History", service);
+}
+HistoryEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _recordObj: HistoryRec,
+  _storeObj: HistoryStore,
+  _trackerObj: HistoryTracker,
+  downloadLimit: MAX_HISTORY_DOWNLOAD,
+  applyIncomingBatchSize: HISTORY_STORE_BATCH_SIZE,
+
+  syncPriority: 7,
+
+  _processIncoming: function (newitems) {
+    // We want to notify history observers that a batch operation is underway
+    // so they don't do lots of work for each incoming record.
+    let observers = PlacesUtils.history.getObservers();
+    function notifyHistoryObservers(notification) {
+      for (let observer of observers) {
+        try {
+          observer[notification]();
+        } catch (ex) { }
+      }
+    }
+    notifyHistoryObservers("onBeginUpdateBatch");
+    try {
+      return SyncEngine.prototype._processIncoming.call(this, newitems);
+    } finally {
+      notifyHistoryObservers("onEndUpdateBatch");
+    }
+  },
+};
+
+function HistoryStore(name, engine) {
+  Store.call(this, name, engine);
+
+  // Explicitly nullify our references to our cached services so we don't leak
+  Svc.Obs.add("places-shutdown", function() {
+    for (let query in this._stmts) {
+      let stmt = this._stmts;
+      stmt.finalize();
+    }
+    this._stmts = {};
+  }, this);
+}
+HistoryStore.prototype = {
+  __proto__: Store.prototype,
+
+  __asyncHistory: null,
+  get _asyncHistory() {
+    if (!this.__asyncHistory) {
+      this.__asyncHistory = Cc["@mozilla.org/browser/history;1"]
+                              .getService(Ci.mozIAsyncHistory);
+    }
+    return this.__asyncHistory;
+  },
+
+  _stmts: {},
+  _getStmt: function(query) {
+    if (query in this._stmts) {
+      return this._stmts[query];
+    }
+
+    this._log.trace("Creating SQL statement: " + query);
+    let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase)
+                        .DBConnection;
+    return this._stmts[query] = db.createAsyncStatement(query);
+  },
+
+  get _setGUIDStm() {
+    return this._getStmt(
+      "UPDATE moz_places " +
+      "SET guid = :guid " +
+      "WHERE url_hash = hash(:page_url) AND url = :page_url");
+  },
+
+  // Some helper functions to handle GUIDs
+  setGUID: function setGUID(uri, guid) {
+    uri = uri.spec ? uri.spec : uri;
+
+    if (!guid) {
+      guid = Utils.makeGUID();
+    }
+
+    let stmt = this._setGUIDStm;
+    stmt.params.guid = guid;
+    stmt.params.page_url = uri;
+    Async.querySpinningly(stmt);
+    return guid;
+  },
+
+  get _guidStm() {
+    return this._getStmt(
+      "SELECT guid " +
+      "FROM moz_places " +
+      "WHERE url_hash = hash(:page_url) AND url = :page_url");
+  },
+  _guidCols: ["guid"],
+
+  GUIDForUri: function GUIDForUri(uri, create) {
+    let stm = this._guidStm;
+    stm.params.page_url = uri.spec ? uri.spec : uri;
+
+    // Use the existing GUID if it exists
+    let result = Async.querySpinningly(stm, this._guidCols)[0];
+    if (result && result.guid)
+      return result.guid;
+
+    // Give the uri a GUID if it doesn't have one
+    if (create)
+      return this.setGUID(uri);
+  },
+
+  get _visitStm() {
+    return this._getStmt(`/* do not warn (bug 599936) */
+      SELECT visit_type type, visit_date date
+      FROM moz_historyvisits
+      JOIN moz_places h ON h.id = place_id
+      WHERE url_hash = hash(:url) AND url = :url
+      ORDER BY date DESC LIMIT 20`);
+  },
+  _visitCols: ["date", "type"],
+
+  get _urlStm() {
+    return this._getStmt(
+      "SELECT url, title, frecency " +
+      "FROM moz_places " +
+      "WHERE guid = :guid");
+  },
+  _urlCols: ["url", "title", "frecency"],
+
+  get _allUrlStm() {
+    return this._getStmt(
+      "SELECT url " +
+      "FROM moz_places " +
+      "WHERE last_visit_date > :cutoff_date " +
+      "ORDER BY frecency DESC " +
+      "LIMIT :max_results");
+  },
+  _allUrlCols: ["url"],
+
+  // See bug 320831 for why we use SQL here
+  _getVisits: function HistStore__getVisits(uri) {
+    this._visitStm.params.url = uri;
+    return Async.querySpinningly(this._visitStm, this._visitCols);
+  },
+
+  // See bug 468732 for why we use SQL here
+  _findURLByGUID: function HistStore__findURLByGUID(guid) {
+    this._urlStm.params.guid = guid;
+    return Async.querySpinningly(this._urlStm, this._urlCols)[0];
+  },
+
+  changeItemID: function HStore_changeItemID(oldID, newID) {
+    this.setGUID(this._findURLByGUID(oldID).url, newID);
+  },
+
+
+  getAllIDs: function HistStore_getAllIDs() {
+    // Only get places visited within the last 30 days (30*24*60*60*1000ms)
+    this._allUrlStm.params.cutoff_date = (Date.now() - 2592000000) * 1000;
+    this._allUrlStm.params.max_results = MAX_HISTORY_UPLOAD;
+
+    let urls = Async.querySpinningly(this._allUrlStm, this._allUrlCols);
+    let self = this;
+    return urls.reduce(function(ids, item) {
+      ids[self.GUIDForUri(item.url, true)] = item.url;
+      return ids;
+    }, {});
+  },
+
+  applyIncomingBatch: function applyIncomingBatch(records) {
+    let failed = [];
+
+    // Convert incoming records to mozIPlaceInfo objects. Some records can be
+    // ignored or handled directly, so we're rewriting the array in-place.
+    let i, k;
+    for (i = 0, k = 0; i < records.length; i++) {
+      let record = records[k] = records[i];
+      let shouldApply;
+
+      // This is still synchronous I/O for now.
+      try {
+        if (record.deleted) {
+          // Consider using nsIBrowserHistory::removePages() here.
+          this.remove(record);
+          // No further processing needed. Remove it from the list.
+          shouldApply = false;
+        } else {
+          shouldApply = this._recordToPlaceInfo(record);
+        }
+      } catch (ex) {
+        if (Async.isShutdownException(ex)) {
+          throw ex;
+        }
+        failed.push(record.id);
+        shouldApply = false;
+      }
+
+      if (shouldApply) {
+        k += 1;
+      }
+    }
+    records.length = k; // truncate array
+
+    // Nothing to do.
+    if (!records.length) {
+      return failed;
+    }
+
+    let updatePlacesCallback = { 
+      handleResult: function handleResult() {},
+      handleError: function handleError(resultCode, placeInfo) {
+        failed.push(placeInfo.guid);
+      },
+      handleCompletion: Async.makeSyncCallback()
+    };
+    this._asyncHistory.updatePlaces(records, updatePlacesCallback);
+    Async.waitForSyncCallback(updatePlacesCallback.handleCompletion);
+    return failed;
+  },
+
+  /**
+   * Converts a Sync history record to a mozIPlaceInfo.
+   * 
+   * Throws if an invalid record is encountered (invalid URI, etc.),
+   * returns true if the record is to be applied, false otherwise
+   * (no visits to add, etc.),
+   */
+  _recordToPlaceInfo: function _recordToPlaceInfo(record) {
+    // Sort out invalid URIs and ones Places just simply doesn't want.
+    record.uri = Utils.makeURI(record.histUri);
+    if (!record.uri) {
+      this._log.warn("Attempted to process invalid URI, skipping.");
+      throw "Invalid URI in record";
+    }
+
+    if (!Utils.checkGUID(record.id)) {
+      this._log.warn("Encountered record with invalid GUID: " + record.id);
+      return false;
+    }
+    record.guid = record.id;
+
+    if (!PlacesUtils.history.canAddURI(record.uri)) {
+      this._log.trace("Ignoring record " + record.id + " with URI "
+                      + record.uri.spec + ": can't add this URI.");
+      return false;
+    }
+
+    // We dupe visits by date and type. So an incoming visit that has
+    // the same timestamp and type as a local one won't get applied.
+    // To avoid creating new objects, we rewrite the query result so we
+    // can simply check for containment below.
+    let curVisits = this._getVisits(record.histUri);
+    let i, k;
+    for (i = 0; i < curVisits.length; i++) {
+      curVisits[i] = curVisits[i].date + "," + curVisits[i].type;
+    }
+
+    // Walk through the visits, make sure we have sound data, and eliminate
+    // dupes. The latter is done by rewriting the array in-place.
+    for (i = 0, k = 0; i < record.visits.length; i++) {
+      let visit = record.visits[k] = record.visits[i];
+
+      if (!visit.date || typeof visit.date != "number") {
+        this._log.warn("Encountered record with invalid visit date: "
+                       + visit.date);
+        continue;
+      }
+
+      if (!visit.type ||
+          !Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)) {
+        this._log.warn("Encountered record with invalid visit type: " +
+                       visit.type + "; ignoring.");
+        continue;
+      }
+
+      // Dates need to be integers.
+      visit.date = Math.round(visit.date);
+
+      if (curVisits.indexOf(visit.date + "," + visit.type) != -1) {
+        // Visit is a dupe, don't increment 'k' so the element will be
+        // overwritten.
+        continue;
+      }
+
+      visit.visitDate = visit.date;
+      visit.transitionType = visit.type;
+      k += 1;
+    }
+    record.visits.length = k; // truncate array
+
+    // No update if there aren't any visits to apply.
+    // mozIAsyncHistory::updatePlaces() wants at least one visit.
+    // In any case, the only thing we could change would be the title
+    // and that shouldn't change without a visit.
+    if (!record.visits.length) {
+      this._log.trace("Ignoring record " + record.id + " with URI "
+                      + record.uri.spec + ": no visits to add.");
+      return false;
+    }
+
+    return true;
+  },
+
+  remove: function HistStore_remove(record) {
+    let page = this._findURLByGUID(record.id);
+    if (page == null) {
+      this._log.debug("Page already removed: " + record.id);
+      return;
+    }
+
+    let uri = Utils.makeURI(page.url);
+    PlacesUtils.history.removePage(uri);
+    this._log.trace("Removed page: " + [record.id, page.url, page.title]);
+  },
+
+  itemExists: function HistStore_itemExists(id) {
+    return !!this._findURLByGUID(id);
+  },
+
+  createRecord: function createRecord(id, collection) {
+    let foo = this._findURLByGUID(id);
+    let record = new HistoryRec(collection, id);
+    if (foo) {
+      record.histUri = foo.url;
+      record.title = foo.title;
+      record.sortindex = foo.frecency;
+      record.visits = this._getVisits(record.histUri);
+    } else {
+      record.deleted = true;
+    }
+
+    return record;
+  },
+
+  wipe: function HistStore_wipe() {
+    let cb = Async.makeSyncCallback();
+    PlacesUtils.history.clear().then(result => {cb(null, result)}, err => {cb(err)});
+    return Async.waitForSyncCallback(cb);
+  }
+};
+
+function HistoryTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+HistoryTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function() {
+    this._log.info("Adding Places observer.");
+    PlacesUtils.history.addObserver(this, true);
+  },
+
+  stopTracking: function() {
+    this._log.info("Removing Places observer.");
+    PlacesUtils.history.removeObserver(this);
+  },
+
+  QueryInterface: XPCOMUtils.generateQI([
+    Ci.nsINavHistoryObserver,
+    Ci.nsISupportsWeakReference
+  ]),
+
+  onDeleteAffectsGUID: function (uri, guid, reason, source, increment) {
+    if (this.ignoreAll || reason == Ci.nsINavHistoryObserver.REASON_EXPIRED) {
+      return;
+    }
+    this._log.trace(source + ": " + uri.spec + ", reason " + reason);
+    if (this.addChangedID(guid)) {
+      this.score += increment;
+    }
+  },
+
+  onDeleteVisits: function (uri, visitTime, guid, reason) {
+    this.onDeleteAffectsGUID(uri, guid, reason, "onDeleteVisits", SCORE_INCREMENT_SMALL);
+  },
+
+  onDeleteURI: function (uri, guid, reason) {
+    this.onDeleteAffectsGUID(uri, guid, reason, "onDeleteURI", SCORE_INCREMENT_XLARGE);
+  },
+
+  onVisit: function (uri, vid, time, session, referrer, trans, guid) {
+    if (this.ignoreAll) {
+      this._log.trace("ignoreAll: ignoring visit for " + guid);
+      return;
+    }
+
+    this._log.trace("onVisit: " + uri.spec);
+    if (this.addChangedID(guid)) {
+      this.score += SCORE_INCREMENT_SMALL;
+    }
+  },
+
+  onClearHistory: function () {
+    this._log.trace("onClearHistory");
+    // Note that we're going to trigger a sync, but none of the cleared
+    // pages are tracked, so the deletions will not be propagated.
+    // See Bug 578694.
+    this.score += SCORE_INCREMENT_XLARGE;
+  },
+
+  onBeginUpdateBatch: function () {},
+  onEndUpdateBatch: function () {},
+  onPageChanged: function () {},
+  onTitleChanged: function () {},
+  onBeforeDeleteURI: function () {},
+};

services/sync/modules/engines/passwords.js

@@ -0,0 +1,371 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec', 'PasswordValidator'];
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/collection_validator.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+
+this.LoginRec = function LoginRec(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+LoginRec.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Sync.Record.Login",
+};
+
+Utils.deferGetSet(LoginRec, "cleartext", [
+    "hostname", "formSubmitURL",
+    "httpRealm", "username", "password", "usernameField", "passwordField",
+    "timeCreated", "timePasswordChanged",
+    ]);
+
+
+this.PasswordEngine = function PasswordEngine(service) {
+  SyncEngine.call(this, "Passwords", service);
+}
+PasswordEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _storeObj: PasswordStore,
+  _trackerObj: PasswordTracker,
+  _recordObj: LoginRec,
+
+  applyIncomingBatchSize: PASSWORDS_STORE_BATCH_SIZE,
+
+  syncPriority: 2,
+
+  _syncFinish: function () {
+    SyncEngine.prototype._syncFinish.call(this);
+
+    // Delete the Weave credentials from the server once.
+    if (!Svc.Prefs.get("deletePwdFxA", false)) {
+      try {
+        let ids = [];
+        for (let host of Utils.getSyncCredentialsHosts()) {
+          for (let info of Services.logins.findLogins({}, host, "", "")) {
+            ids.push(info.QueryInterface(Components.interfaces.nsILoginMetaInfo).guid);
+          }
+        }
+        if (ids.length) {
+          let coll = new Collection(this.engineURL, null, this.service);
+          coll.ids = ids;
+          let ret = coll.delete();
+          this._log.debug("Delete result: " + ret);
+          if (!ret.success && ret.status != 400) {
+            // A non-400 failure means try again next time.
+            return;
+          }
+        } else {
+          this._log.debug("Didn't find any passwords to delete");
+        }
+        // If there were no ids to delete, or we succeeded, or got a 400,
+        // record success.
+        Svc.Prefs.set("deletePwdFxA", true);
+        Svc.Prefs.reset("deletePwd"); // The old prefname we previously used.
+      } catch (ex) {
+        if (Async.isShutdownException(ex)) {
+          throw ex;
+        }
+        this._log.debug("Password deletes failed", ex);
+      }
+    }
+  },
+
+  _findDupe: function (item) {
+    let login = this._store._nsLoginInfoFromRecord(item);
+    if (!login) {
+      return;
+    }
+
+    let logins = Services.logins.findLogins({}, login.hostname, login.formSubmitURL, login.httpRealm);
+
+    this._store._sleep(0); // Yield back to main thread after synchronous operation.
+
+    // Look for existing logins that match the hostname, but ignore the password.
+    for (let local of logins) {
+      if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) {
+        return local.guid;
+      }
+    }
+  },
+};
+
+function PasswordStore(name, engine) {
+  Store.call(this, name, engine);
+  this._nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1", Ci.nsILoginInfo, "init");
+}
+PasswordStore.prototype = {
+  __proto__: Store.prototype,
+
+  _newPropertyBag: function () {
+    return Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2);
+  },
+
+  /**
+   * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo).
+   */
+  _nsLoginInfoFromRecord: function (record) {
+    function nullUndefined(x) {
+      return (x == undefined) ? null : x;
+    }
+
+    if (record.formSubmitURL && record.httpRealm) {
+      this._log.warn("Record " + record.id + " has both formSubmitURL and httpRealm. Skipping.");
+      return null;
+    }
+
+    // Passing in "undefined" results in an empty string, which later
+    // counts as a value. Explicitly `|| null` these fields according to JS
+    // truthiness. Records with empty strings or null will be unmolested.
+    let info = new this._nsLoginInfo(record.hostname,
+                                     nullUndefined(record.formSubmitURL),
+                                     nullUndefined(record.httpRealm),
+                                     record.username,
+                                     record.password,
+                                     record.usernameField,
+                                     record.passwordField);
+
+    info.QueryInterface(Ci.nsILoginMetaInfo);
+    info.guid = record.id;
+    if (record.timeCreated) {
+      info.timeCreated = record.timeCreated;
+    }
+    if (record.timePasswordChanged) {
+      info.timePasswordChanged = record.timePasswordChanged;
+    }
+
+    return info;
+  },
+
+  _getLoginFromGUID: function (id) {
+    let prop = this._newPropertyBag();
+    prop.setPropertyAsAUTF8String("guid", id);
+
+    let logins = Services.logins.searchLogins({}, prop);
+    this._sleep(0); // Yield back to main thread after synchronous operation.
+
+    if (logins.length > 0) {
+      this._log.trace(logins.length + " items matching " + id + " found.");
+      return logins[0];
+    }
+
+    this._log.trace("No items matching " + id + " found. Ignoring");
+    return null;
+  },
+
+  getAllIDs: function () {
+    let items = {};
+    let logins = Services.logins.getAllLogins({});
+
+    for (let i = 0; i < logins.length; i++) {
+      // Skip over Weave password/passphrase entries.
+      let metaInfo = logins[i].QueryInterface(Ci.nsILoginMetaInfo);
+      if (Utils.getSyncCredentialsHosts().has(metaInfo.hostname)) {
+        continue;
+      }
+
+      items[metaInfo.guid] = metaInfo;
+    }
+
+    return items;
+  },
+
+  changeItemID: function (oldID, newID) {
+    this._log.trace("Changing item ID: " + oldID + " to " + newID);
+
+    let oldLogin = this._getLoginFromGUID(oldID);
+    if (!oldLogin) {
+      this._log.trace("Can't change item ID: item doesn't exist");
+      return;
+    }
+    if (this._getLoginFromGUID(newID)) {
+      this._log.trace("Can't change item ID: new ID already in use");
+      return;
+    }
+
+    let prop = this._newPropertyBag();
+    prop.setPropertyAsAUTF8String("guid", newID);
+
+    Services.logins.modifyLogin(oldLogin, prop);
+  },
+
+  itemExists: function (id) {
+    return !!this._getLoginFromGUID(id);
+  },
+
+  createRecord: function (id, collection) {
+    let record = new LoginRec(collection, id);
+    let login = this._getLoginFromGUID(id);
+
+    if (!login) {
+      record.deleted = true;
+      return record;
+    }
+
+    record.hostname = login.hostname;
+    record.formSubmitURL = login.formSubmitURL;
+    record.httpRealm = login.httpRealm;
+    record.username = login.username;
+    record.password = login.password;
+    record.usernameField = login.usernameField;
+    record.passwordField = login.passwordField;
+
+    // Optional fields.
+    login.QueryInterface(Ci.nsILoginMetaInfo);
+    record.timeCreated = login.timeCreated;
+    record.timePasswordChanged = login.timePasswordChanged;
+
+    return record;
+  },
+
+  create: function (record) {
+    let login = this._nsLoginInfoFromRecord(record);
+    if (!login) {
+      return;
+    }
+
+    this._log.debug("Adding login for " + record.hostname);
+    this._log.trace("httpRealm: " + JSON.stringify(login.httpRealm) + "; " +
+                    "formSubmitURL: " + JSON.stringify(login.formSubmitURL));
+    try {
+      Services.logins.addLogin(login);
+    } catch(ex) {
+      this._log.debug(`Adding record ${record.id} resulted in exception`, ex);
+    }
+  },
+
+  remove: function (record) {
+    this._log.trace("Removing login " + record.id);
+
+    let loginItem = this._getLoginFromGUID(record.id);
+    if (!loginItem) {
+      this._log.trace("Asked to remove record that doesn't exist, ignoring");
+      return;
+    }
+
+    Services.logins.removeLogin(loginItem);
+  },
+
+  update: function (record) {
+    let loginItem = this._getLoginFromGUID(record.id);
+    if (!loginItem) {
+      this._log.debug("Skipping update for unknown item: " + record.hostname);
+      return;
+    }
+
+    this._log.debug("Updating " + record.hostname);
+    let newinfo = this._nsLoginInfoFromRecord(record);
+    if (!newinfo) {
+      return;
+    }
+
+    try {
+      Services.logins.modifyLogin(loginItem, newinfo);
+    } catch(ex) {
+      this._log.debug(`Modifying record ${record.id} resulted in exception; not modifying`, ex);
+    }
+  },
+
+  wipe: function () {
+    Services.logins.removeAllLogins();
+  },
+};
+
+function PasswordTracker(name, engine) {
+  Tracker.call(this, name, engine);
+  Svc.Obs.add("weave:engine:start-tracking", this);
+  Svc.Obs.add("weave:engine:stop-tracking", this);
+}
+PasswordTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function () {
+    Svc.Obs.add("passwordmgr-storage-changed", this);
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("passwordmgr-storage-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    if (this.ignoreAll) {
+      return;
+    }
+
+    // A single add, remove or change or removing all items
+    // will trigger a sync for MULTI_DEVICE.
+    switch (data) {
+      case "modifyLogin":
+        subject = subject.QueryInterface(Ci.nsIArray).queryElementAt(1, Ci.nsILoginMetaInfo);
+        // Fall through.
+      case "addLogin":
+      case "removeLogin":
+        // Skip over Weave password/passphrase changes.
+        subject.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo);
+        if (Utils.getSyncCredentialsHosts().has(subject.hostname)) {
+          break;
+        }
+
+        this.score += SCORE_INCREMENT_XLARGE;
+        this._log.trace(data + ": " + subject.guid);
+        this.addChangedID(subject.guid);
+        break;
+      case "removeAllLogins":
+        this._log.trace(data);
+        this.score += SCORE_INCREMENT_XLARGE;
+        break;
+    }
+  },
+};
+
+class PasswordValidator extends CollectionValidator {
+  constructor() {
+    super("passwords", "id", [
+      "hostname",
+      "formSubmitURL",
+      "httpRealm",
+      "password",
+      "passwordField",
+      "username",
+      "usernameField",
+    ]);
+  }
+
+  getClientItems() {
+    let logins = Services.logins.getAllLogins({});
+    let syncHosts = Utils.getSyncCredentialsHosts()
+    let result = logins.map(l => l.QueryInterface(Ci.nsILoginMetaInfo))
+                       .filter(l => !syncHosts.has(l.hostname));
+    return Promise.resolve(result);
+  }
+
+  normalizeClientItem(item) {
+    return {
+      id: item.guid,
+      guid: item.guid,
+      hostname: item.hostname,
+      formSubmitURL: item.formSubmitURL,
+      httpRealm: item.httpRealm,
+      password: item.password,
+      passwordField: item.passwordField,
+      username: item.username,
+      usernameField: item.usernameField,
+      original: item,
+    }
+  }
+
+  normalizeServerItem(item) {
+    return Object.assign({ guid: item.id }, item);
+  }
+}
+
+

services/sync/modules/engines/prefs.js

@@ -0,0 +1,273 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['PrefsEngine', 'PrefRec'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+
+const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://gre/modules/LightweightThemeManager.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+const PREFS_GUID = CommonUtils.encodeBase64URL(Services.appinfo.ID);
+
+this.PrefRec = function PrefRec(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+PrefRec.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Sync.Record.Pref",
+};
+
+Utils.deferGetSet(PrefRec, "cleartext", ["value"]);
+
+
+this.PrefsEngine = function PrefsEngine(service) {
+  SyncEngine.call(this, "Prefs", service);
+}
+PrefsEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _storeObj: PrefStore,
+  _trackerObj: PrefTracker,
+  _recordObj: PrefRec,
+  version: 2,
+
+  syncPriority: 1,
+  allowSkippedRecord: false,
+
+  getChangedIDs: function () {
+    // No need for a proper timestamp (no conflict resolution needed).
+    let changedIDs = {};
+    if (this._tracker.modified)
+      changedIDs[PREFS_GUID] = 0;
+    return changedIDs;
+  },
+
+  _wipeClient: function () {
+    SyncEngine.prototype._wipeClient.call(this);
+    this.justWiped = true;
+  },
+
+  _reconcile: function (item) {
+    // Apply the incoming item if we don't care about the local data
+    if (this.justWiped) {
+      this.justWiped = false;
+      return true;
+    }
+    return SyncEngine.prototype._reconcile.call(this, item);
+  }
+};
+
+
+function PrefStore(name, engine) {
+  Store.call(this, name, engine);
+  Svc.Obs.add("profile-before-change", function () {
+    this.__prefs = null;
+  }, this);
+}
+PrefStore.prototype = {
+  __proto__: Store.prototype,
+
+ __prefs: null,
+  get _prefs() {
+    if (!this.__prefs) {
+      this.__prefs = new Preferences();
+    }
+    return this.__prefs;
+  },
+
+  _getSyncPrefs: function () {
+    let syncPrefs = Cc["@mozilla.org/preferences-service;1"]
+                      .getService(Ci.nsIPrefService)
+                      .getBranch(PREF_SYNC_PREFS_PREFIX)
+                      .getChildList("", {});
+    // Also sync preferences that determine which prefs get synced.
+    let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref);
+    return controlPrefs.concat(syncPrefs);
+  },
+
+  _isSynced: function (pref) {
+    return pref.startsWith(PREF_SYNC_PREFS_PREFIX) ||
+           this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false);
+  },
+
+  _getAllPrefs: function () {
+    let values = {};
+    for (let pref of this._getSyncPrefs()) {
+      if (this._isSynced(pref)) {
+        // Missing and default prefs get the null value.
+        values[pref] = this._prefs.isSet(pref) ? this._prefs.get(pref, null) : null;
+      }
+    }
+    return values;
+  },
+
+  _updateLightWeightTheme (themeID) {
+    let themeObject = null;
+    if (themeID) {
+      themeObject = LightweightThemeManager.getUsedTheme(themeID);
+    }
+    LightweightThemeManager.currentTheme = themeObject;
+  },
+
+  _setAllPrefs: function (values) {
+    let selectedThemeIDPref = "lightweightThemes.selectedThemeID";
+    let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null);
+    let selectedThemeIDAfter = selectedThemeIDBefore;
+
+    // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise
+    // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device).
+    let prefs = Object.keys(values).sort(a => -a.indexOf(PREF_SYNC_PREFS_PREFIX));
+    for (let pref of prefs) {
+      if (!this._isSynced(pref)) {
+        continue;
+      }
+
+      let value = values[pref];
+
+      switch (pref) {
+        // Some special prefs we don't want to set directly.
+        case selectedThemeIDPref:
+          selectedThemeIDAfter = value;
+          break;
+
+        // default is to just set the pref
+        default:
+          if (value == null) {
+            // Pref has gone missing. The best we can do is reset it.
+            this._prefs.reset(pref);
+          } else {
+            try {
+              this._prefs.set(pref, value);
+            } catch(ex) {
+              this._log.trace("Failed to set pref: " + pref + ": " + ex);
+            }
+          }
+      }
+    }
+
+    // Notify the lightweight theme manager if the selected theme has changed.
+    if (selectedThemeIDBefore != selectedThemeIDAfter) {
+      this._updateLightWeightTheme(selectedThemeIDAfter);
+    }
+  },
+
+  getAllIDs: function () {
+    /* We store all prefs in just one WBO, with just one GUID */
+    let allprefs = {};
+    allprefs[PREFS_GUID] = true;
+    return allprefs;
+  },
+
+  changeItemID: function (oldID, newID) {
+    this._log.trace("PrefStore GUID is constant!");
+  },
+
+  itemExists: function (id) {
+    return (id === PREFS_GUID);
+  },
+
+  createRecord: function (id, collection) {
+    let record = new PrefRec(collection, id);
+
+    if (id == PREFS_GUID) {
+      record.value = this._getAllPrefs();
+    } else {
+      record.deleted = true;
+    }
+
+    return record;
+  },
+
+  create: function (record) {
+    this._log.trace("Ignoring create request");
+  },
+
+  remove: function (record) {
+    this._log.trace("Ignoring remove request");
+  },
+
+  update: function (record) {
+    // Silently ignore pref updates that are for other apps.
+    if (record.id != PREFS_GUID)
+      return;
+
+    this._log.trace("Received pref updates, applying...");
+    this._setAllPrefs(record.value);
+  },
+
+  wipe: function () {
+    this._log.trace("Ignoring wipe request");
+  }
+};
+
+function PrefTracker(name, engine) {
+  Tracker.call(this, name, engine);
+  Svc.Obs.add("profile-before-change", this);
+  Svc.Obs.add("weave:engine:start-tracking", this);
+  Svc.Obs.add("weave:engine:stop-tracking", this);
+}
+PrefTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  get modified() {
+    return Svc.Prefs.get("engine.prefs.modified", false);
+  },
+  set modified(value) {
+    Svc.Prefs.set("engine.prefs.modified", value);
+  },
+
+  loadChangedIDs: function loadChangedIDs() {
+    // Don't read changed IDs from disk at start up.
+  },
+
+  clearChangedIDs: function clearChangedIDs() {
+    this.modified = false;
+  },
+
+ __prefs: null,
+  get _prefs() {
+    if (!this.__prefs) {
+      this.__prefs = new Preferences();
+    }
+    return this.__prefs;
+  },
+
+  startTracking: function () {
+    Services.prefs.addObserver("", this, false);
+  },
+
+  stopTracking: function () {
+    this.__prefs = null;
+    Services.prefs.removeObserver("", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    switch (topic) {
+      case "profile-before-change":
+        this.stopTracking();
+        break;
+      case "nsPref:changed":
+        // Trigger a sync for MULTI-DEVICE for a change that determines
+        // which prefs are synced or a regular pref change.
+        if (data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 ||
+            this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)) {
+          this.score += SCORE_INCREMENT_XLARGE;
+          this.modified = true;
+          this._log.trace("Preference " + data + " changed");
+        }
+        break;
+    }
+  }
+};

services/sync/modules/engines/tabs.js

@@ -0,0 +1,393 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["TabEngine", "TabSetRecord"];
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+const TABS_TTL = 604800;           // 7 days.
+const TAB_ENTRIES_LIMIT = 25;      // How many URLs to include in tab history.
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/clients.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils",
+  "resource://gre/modules/PrivateBrowsingUtils.jsm");
+
+this.TabSetRecord = function TabSetRecord(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+TabSetRecord.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  _logName: "Sync.Record.Tabs",
+  ttl: TABS_TTL,
+};
+
+Utils.deferGetSet(TabSetRecord, "cleartext", ["clientName", "tabs"]);
+
+
+this.TabEngine = function TabEngine(service) {
+  SyncEngine.call(this, "Tabs", service);
+
+  // Reset the client on every startup so that we fetch recent tabs.
+  this._resetClient();
+}
+TabEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _storeObj: TabStore,
+  _trackerObj: TabTracker,
+  _recordObj: TabSetRecord,
+  // A flag to indicate if we have synced in this session. This is to help
+  // consumers of remote tabs that may want to differentiate between "I've an
+  // empty tab list as I haven't yet synced" vs "I've an empty tab list
+  // as there really are no tabs"
+  hasSyncedThisSession: false,
+
+  syncPriority: 3,
+
+  getChangedIDs: function () {
+    // No need for a proper timestamp (no conflict resolution needed).
+    let changedIDs = {};
+    if (this._tracker.modified)
+      changedIDs[this.service.clientsEngine.localID] = 0;
+    return changedIDs;
+  },
+
+  // API for use by Sync UI code to give user choices of tabs to open.
+  getAllClients: function () {
+    return this._store._remoteClients;
+  },
+
+  getClientById: function (id) {
+    return this._store._remoteClients[id];
+  },
+
+  _resetClient: function () {
+    SyncEngine.prototype._resetClient.call(this);
+    this._store.wipe();
+    this._tracker.modified = true;
+    this.hasSyncedThisSession = false;
+  },
+
+  removeClientData: function () {
+    let url = this.engineURL + "/" + this.service.clientsEngine.localID;
+    this.service.resource(url).delete();
+  },
+
+  /**
+   * Return a Set of open URLs.
+   */
+  getOpenURLs: function () {
+    let urls = new Set();
+    for (let entry of this._store.getAllTabs()) {
+      urls.add(entry.urlHistory[0]);
+    }
+    return urls;
+  },
+
+  _reconcile: function (item) {
+    // Skip our own record.
+    // TabStore.itemExists tests only against our local client ID.
+    if (this._store.itemExists(item.id)) {
+      this._log.trace("Ignoring incoming tab item because of its id: " + item.id);
+      return false;
+    }
+
+    return SyncEngine.prototype._reconcile.call(this, item);
+  },
+
+  _syncFinish() {
+    this.hasSyncedThisSession = true;
+    return SyncEngine.prototype._syncFinish.call(this);
+  },
+};
+
+
+function TabStore(name, engine) {
+  Store.call(this, name, engine);
+}
+TabStore.prototype = {
+  __proto__: Store.prototype,
+
+  itemExists: function (id) {
+    return id == this.engine.service.clientsEngine.localID;
+  },
+
+  getWindowEnumerator: function () {
+    return Services.wm.getEnumerator("navigator:browser");
+  },
+
+  shouldSkipWindow: function (win) {
+    return win.closed ||
+           PrivateBrowsingUtils.isWindowPrivate(win);
+  },
+
+  getTabState: function (tab) {
+    return JSON.parse(Svc.Session.getTabState(tab));
+  },
+
+  getAllTabs: function (filter) {
+    let filteredUrls = new RegExp(Svc.Prefs.get("engine.tabs.filteredUrls"), "i");
+
+    let allTabs = [];
+
+    let winEnum = this.getWindowEnumerator();
+    while (winEnum.hasMoreElements()) {
+      let win = winEnum.getNext();
+      if (this.shouldSkipWindow(win)) {
+        continue;
+      }
+
+      for (let tab of win.gBrowser.tabs) {
+        let tabState = this.getTabState(tab);
+
+        // Make sure there are history entries to look at.
+        if (!tabState || !tabState.entries.length) {
+          continue;
+        }
+
+        let acceptable = !filter ? (url) => url :
+                                   (url) => url && !filteredUrls.test(url);
+
+        let entries = tabState.entries;
+        let index = tabState.index;
+        let current = entries[index - 1];
+
+        // We ignore the tab completely if the current entry url is
+        // not acceptable (we need something accurate to open).
+        if (!acceptable(current.url)) {
+          continue;
+        }
+
+        if (current.url.length >= (MAX_UPLOAD_BYTES - 1000)) {
+          this._log.trace("Skipping over-long URL.");
+          continue;
+        }
+
+        // The element at `index` is the current page. Previous URLs were
+        // previously visited URLs; subsequent URLs are in the 'forward' stack,
+        // which we can't represent in Sync, so we truncate here.
+        let candidates = (entries.length == index) ?
+                         entries :
+                         entries.slice(0, index);
+
+        let urls = candidates.map((entry) => entry.url)
+                             .filter(acceptable)
+                             .reverse();                       // Because Sync puts current at index 0, and history after.
+
+        // Truncate if necessary.
+        if (urls.length > TAB_ENTRIES_LIMIT) {
+          urls.length = TAB_ENTRIES_LIMIT;
+        }
+
+        allTabs.push({
+          title: current.title || "",
+          urlHistory: urls,
+          icon: tabState.image ||
+                (tabState.attributes && tabState.attributes.image) ||
+                "",
+          lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000),
+        });
+      }
+    }
+
+    return allTabs;
+  },
+
+  createRecord: function (id, collection) {
+    let record = new TabSetRecord(collection, id);
+    record.clientName = this.engine.service.clientsEngine.localName;
+
+    // Sort tabs in descending-used order to grab the most recently used
+    let tabs = this.getAllTabs(true).sort(function (a, b) {
+      return b.lastUsed - a.lastUsed;
+    });
+
+    // Figure out how many tabs we can pack into a payload. Starting with a 28KB
+    // payload, we can estimate various overheads from encryption/JSON/WBO.
+    let size = JSON.stringify(tabs).length;
+    let origLength = tabs.length;
+    const MAX_TAB_SIZE = 20000;
+    if (size > MAX_TAB_SIZE) {
+      // Estimate a little more than the direct fraction to maximize packing
+      let cutoff = Math.ceil(tabs.length * MAX_TAB_SIZE / size);
+      tabs = tabs.slice(0, cutoff + 1);
+
+      // Keep dropping off the last entry until the data fits
+      while (JSON.stringify(tabs).length > MAX_TAB_SIZE)
+        tabs.pop();
+    }
+
+    this._log.trace("Created tabs " + tabs.length + " of " + origLength);
+    tabs.forEach(function (tab) {
+      this._log.trace("Wrapping tab: " + JSON.stringify(tab));
+    }, this);
+
+    record.tabs = tabs;
+    return record;
+  },
+
+  getAllIDs: function () {
+    // Don't report any tabs if all windows are in private browsing for
+    // first syncs.
+    let ids = {};
+    let allWindowsArePrivate = false;
+    let wins = Services.wm.getEnumerator("navigator:browser");
+    while (wins.hasMoreElements()) {
+      if (PrivateBrowsingUtils.isWindowPrivate(wins.getNext())) {
+        // Ensure that at least there is a private window.
+        allWindowsArePrivate = true;
+      } else {
+        // If there is a not private windown then finish and continue.
+        allWindowsArePrivate = false;
+        break;
+      }
+    }
+
+    if (allWindowsArePrivate &&
+        !PrivateBrowsingUtils.permanentPrivateBrowsing) {
+      return ids;
+    }
+
+    ids[this.engine.service.clientsEngine.localID] = true;
+    return ids;
+  },
+
+  wipe: function () {
+    this._remoteClients = {};
+  },
+
+  create: function (record) {
+    this._log.debug("Adding remote tabs from " + record.clientName);
+    this._remoteClients[record.id] = Object.assign({}, record.cleartext, {
+      lastModified: record.modified
+    });
+  },
+
+  update: function (record) {
+    this._log.trace("Ignoring tab updates as local ones win");
+  },
+};
+
+
+function TabTracker(name, engine) {
+  Tracker.call(this, name, engine);
+  Svc.Obs.add("weave:engine:start-tracking", this);
+  Svc.Obs.add("weave:engine:stop-tracking", this);
+
+  // Make sure "this" pointer is always set correctly for event listeners.
+  this.onTab = Utils.bind2(this, this.onTab);
+  this._unregisterListeners = Utils.bind2(this, this._unregisterListeners);
+}
+TabTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
+
+  loadChangedIDs: function () {
+    // Don't read changed IDs from disk at start up.
+  },
+
+  clearChangedIDs: function () {
+    this.modified = false;
+  },
+
+  _topics: ["pageshow", "TabOpen", "TabClose", "TabSelect"],
+
+  _registerListenersForWindow: function (window) {
+    this._log.trace("Registering tab listeners in window");
+    for (let topic of this._topics) {
+      window.addEventListener(topic, this.onTab, false);
+    }
+    window.addEventListener("unload", this._unregisterListeners, false);
+    // If it's got a tab browser we can listen for things like navigation.
+    if (window.gBrowser) {
+      window.gBrowser.addProgressListener(this);
+    }
+  },
+
+  _unregisterListeners: function (event) {
+    this._unregisterListenersForWindow(event.target);
+  },
+
+  _unregisterListenersForWindow: function (window) {
+    this._log.trace("Removing tab listeners in window");
+    window.removeEventListener("unload", this._unregisterListeners, false);
+    for (let topic of this._topics) {
+      window.removeEventListener(topic, this.onTab, false);
+    }
+    if (window.gBrowser) {
+      window.gBrowser.removeProgressListener(this);
+    }
+  },
+
+  startTracking: function () {
+    Svc.Obs.add("domwindowopened", this);
+    let wins = Services.wm.getEnumerator("navigator:browser");
+    while (wins.hasMoreElements()) {
+      this._registerListenersForWindow(wins.getNext());
+    }
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("domwindowopened", this);
+    let wins = Services.wm.getEnumerator("navigator:browser");
+    while (wins.hasMoreElements()) {
+      this._unregisterListenersForWindow(wins.getNext());
+    }
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    switch (topic) {
+      case "domwindowopened":
+        let onLoad = () => {
+          subject.removeEventListener("load", onLoad, false);
+          // Only register after the window is done loading to avoid unloads.
+          this._registerListenersForWindow(subject);
+        };
+
+        // Add tab listeners now that a window has opened.
+        subject.addEventListener("load", onLoad, false);
+        break;
+    }
+  },
+
+  onTab: function (event) {
+    if (event.originalTarget.linkedBrowser) {
+      let browser = event.originalTarget.linkedBrowser;
+      if (PrivateBrowsingUtils.isBrowserPrivate(browser) &&
+          !PrivateBrowsingUtils.permanentPrivateBrowsing) {
+        this._log.trace("Ignoring tab event from private browsing.");
+        return;
+      }
+    }
+
+    this._log.trace("onTab event: " + event.type);
+    this.modified = true;
+
+    // For page shows, bump the score 10% of the time, emulating a partial
+    // score. We don't want to sync too frequently. For all other page
+    // events, always bump the score.
+    if (event.type != "pageshow" || Math.random() < .1) {
+      this.score += SCORE_INCREMENT_SMALL;
+    }
+  },
+
+  // web progress listeners.
+  onLocationChange: function (webProgress, request, location, flags) {
+    // We only care about top-level location changes which are not in the same
+    // document.
+    if (webProgress.isTopLevel &&
+        ((flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0)) {
+      this.modified = true;
+    }
+  },
+};

services/sync/modules/identity.js

@@ -0,0 +1,605 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["IdentityManager"];
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Promise.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+
+// Lazy import to prevent unnecessary load on startup.
+for (let symbol of ["BulkKeyBundle", "SyncKeyBundle"]) {
+  XPCOMUtils.defineLazyModuleGetter(this, symbol,
+                                    "resource://services-sync/keys.js",
+                                    symbol);
+}
+
+/**
+ * Manages "legacy" identity and authentication for Sync.
+ * See browserid_identity for the Firefox Accounts based identity manager.
+ *
+ * The following entities are managed:
+ *
+ *   account - The main Sync/services account. This is typically an email
+ *     address.
+ *   username - A normalized version of your account. This is what's
+ *     transmitted to the server.
+ *   basic password - UTF-8 password used for authenticating when using HTTP
+ *     basic authentication.
+ *   sync key - The main encryption key used by Sync.
+ *   sync key bundle - A representation of your sync key.
+ *
+ * When changes are made to entities that are stored in the password manager
+ * (basic password, sync key), those changes are merely staged. To commit them
+ * to the password manager, you'll need to call persistCredentials().
+ *
+ * This type also manages authenticating Sync's network requests. Sync's
+ * network code calls into getRESTRequestAuthenticator and
+ * getResourceAuthenticator (depending on the network layer being used). Each
+ * returns a function which can be used to add authentication information to an
+ * outgoing request.
+ *
+ * In theory, this type supports arbitrary identity and authentication
+ * mechanisms. You can add support for them by monkeypatching the global
+ * instance of this type. Specifically, you'll need to redefine the
+ * aforementioned network code functions to do whatever your authentication
+ * mechanism needs them to do. In addition, you may wish to install custom
+ * functions to support your API. Although, that is certainly not required.
+ * If you do monkeypatch, please be advised that Sync expects the core
+ * attributes to have values. You will need to carry at least account and
+ * username forward. If you do not wish to support one of the built-in
+ * authentication mechanisms, you'll probably want to redefine currentAuthState
+ * and any other function that involves the built-in functionality.
+ */
+this.IdentityManager = function IdentityManager() {
+  this._log = Log.repository.getLogger("Sync.Identity");
+  this._log.Level = Log.Level[Svc.Prefs.get("log.logger.identity")];
+
+  this._basicPassword = null;
+  this._basicPasswordAllowLookup = true;
+  this._basicPasswordUpdated = false;
+  this._syncKey = null;
+  this._syncKeyAllowLookup = true;
+  this._syncKeySet = false;
+  this._syncKeyBundle = null;
+}
+IdentityManager.prototype = {
+  _log: null,
+
+  _basicPassword: null,
+  _basicPasswordAllowLookup: true,
+  _basicPasswordUpdated: false,
+
+  _syncKey: null,
+  _syncKeyAllowLookup: true,
+  _syncKeySet: false,
+
+  _syncKeyBundle: null,
+
+  /**
+   * Initialize the identity provider.
+   */
+  initialize: function() {
+    // Nothing to do for this identity provider.
+  },
+
+  finalize: function() {
+    // Nothing to do for this identity provider.
+  },
+
+  /**
+   * Called whenever Service.logout() is called.
+   */
+  logout: function() {
+    // nothing to do for this identity provider.
+  },
+
+  /**
+   * Ensure the user is logged in.  Returns a promise that resolves when
+   * the user is logged in, or is rejected if the login attempt has failed.
+   */
+  ensureLoggedIn: function() {
+    // nothing to do for this identity provider
+    return Promise.resolve();
+  },
+
+  get account() {
+    return Svc.Prefs.get("account", this.username);
+  },
+
+  /**
+   * Sets the active account name.
+   *
+   * This should almost always be called in favor of setting username, as
+   * username is derived from account.
+   *
+   * Changing the account name has the side-effect of wiping out stored
+   * credentials. Keep in mind that persistCredentials() will need to be called
+   * to flush the changes to disk.
+   *
+   * Set this value to null to clear out identity information.
+   */
+  set account(value) {
+    if (value) {
+      value = value.toLowerCase();
+      Svc.Prefs.set("account", value);
+    } else {
+      Svc.Prefs.reset("account");
+    }
+
+    this.username = this.usernameFromAccount(value);
+  },
+
+  get username() {
+    return Svc.Prefs.get("username", null);
+  },
+
+  /**
+   * Set the username value.
+   *
+   * Changing the username has the side-effect of wiping credentials.
+   */
+  set username(value) {
+    if (value) {
+      value = value.toLowerCase();
+
+      if (value == this.username) {
+        return;
+      }
+
+      Svc.Prefs.set("username", value);
+    } else {
+      Svc.Prefs.reset("username");
+    }
+
+    // If we change the username, we interpret this as a major change event
+    // and wipe out the credentials.
+    this._log.info("Username changed. Removing stored credentials.");
+    this.resetCredentials();
+  },
+
+  /**
+   * Resets/Drops all credentials we hold for the current user.
+   */
+  resetCredentials: function() {
+    this.basicPassword = null;
+    this.resetSyncKey();
+  },
+
+  /**
+   * Resets/Drops the sync key we hold for the current user.
+   */
+  resetSyncKey: function() {
+    this.syncKey = null;
+    // syncKeyBundle cleared as a result of setting syncKey.
+  },
+
+  /**
+   * Obtains the HTTP Basic auth password.
+   *
+   * Returns a string if set or null if it is not set.
+   */
+  get basicPassword() {
+    if (this._basicPasswordAllowLookup) {
+      // We need a username to find the credentials.
+      let username = this.username;
+      if (!username) {
+        return null;
+      }
+
+      for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) {
+        if (login.username.toLowerCase() == username) {
+          // It should already be UTF-8 encoded, but we don't take any chances.
+          this._basicPassword = Utils.encodeUTF8(login.password);
+        }
+      }
+
+      this._basicPasswordAllowLookup = false;
+    }
+
+    return this._basicPassword;
+  },
+
+  /**
+   * Set the HTTP basic password to use.
+   *
+   * Changes will not persist unless persistSyncCredentials() is called.
+   */
+  set basicPassword(value) {
+    // Wiping out value.
+    if (!value) {
+      this._log.info("Basic password has no value. Removing.");
+      this._basicPassword = null;
+      this._basicPasswordUpdated = true;
+      this._basicPasswordAllowLookup = false;
+      return;
+    }
+
+    let username = this.username;
+    if (!username) {
+      throw new Error("basicPassword cannot be set before username.");
+    }
+
+    this._log.info("Basic password being updated.");
+    this._basicPassword = Utils.encodeUTF8(value);
+    this._basicPasswordUpdated = true;
+  },
+
+  /**
+   * Obtain the Sync Key.
+   *
+   * This returns a 26 character "friendly" Base32 encoded string on success or
+   * null if no Sync Key could be found.
+   *
+   * If the Sync Key hasn't been set in this session, this will look in the
+   * password manager for the sync key.
+   */
+  get syncKey() {
+    if (this._syncKeyAllowLookup) {
+      let username = this.username;
+      if (!username) {
+        return null;
+      }
+
+      for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
+        if (login.username.toLowerCase() == username) {
+          this._syncKey = login.password;
+        }
+      }
+
+      this._syncKeyAllowLookup = false;
+    }
+
+    return this._syncKey;
+  },
+
+  /**
+   * Set the active Sync Key.
+   *
+   * If being set to null, the Sync Key and its derived SyncKeyBundle are
+   * removed. However, the Sync Key won't be deleted from the password manager
+   * until persistSyncCredentials() is called.
+   *
+   * If a value is provided, it should be a 26 or 32 character "friendly"
+   * Base32 string for which Utils.isPassphrase() returns true.
+   *
+   * A side-effect of setting the Sync Key is that a SyncKeyBundle is
+   * generated. For historical reasons, this will silently error out if the
+   * value is not a proper Sync Key (!Utils.isPassphrase()). This should be
+   * fixed in the future (once service.js is more sane) to throw if the passed
+   * value is not valid.
+   */
+  set syncKey(value) {
+    if (!value) {
+      this._log.info("Sync Key has no value. Deleting.");
+      this._syncKey = null;
+      this._syncKeyBundle = null;
+      this._syncKeyUpdated = true;
+      return;
+    }
+
+    if (!this.username) {
+      throw new Error("syncKey cannot be set before username.");
+    }
+
+    this._log.info("Sync Key being updated.");
+    this._syncKey = value;
+
+    // Clear any cached Sync Key Bundle and regenerate it.
+    this._syncKeyBundle = null;
+    let bundle = this.syncKeyBundle;
+
+    this._syncKeyUpdated = true;
+  },
+
+  /**
+   * Obtain the active SyncKeyBundle.
+   *
+   * This returns a SyncKeyBundle representing a key pair derived from the
+   * Sync Key on success. If no Sync Key is present or if the Sync Key is not
+   * valid, this returns null.
+   *
+   * The SyncKeyBundle should be treated as immutable.
+   */
+  get syncKeyBundle() {
+    // We can't obtain a bundle without a username set.
+    if (!this.username) {
+      this._log.warn("Attempted to obtain Sync Key Bundle with no username set!");
+      return null;
+    }
+
+    if (!this.syncKey) {
+      this._log.warn("Attempted to obtain Sync Key Bundle with no Sync Key " +
+                     "set!");
+      return null;
+    }
+
+    if (!this._syncKeyBundle) {
+      try {
+        this._syncKeyBundle = new SyncKeyBundle(this.username, this.syncKey);
+      } catch (ex) {
+        this._log.warn("Failed to create sync bundle", ex);
+        return null;
+      }
+    }
+
+    return this._syncKeyBundle;
+  },
+
+  /**
+   * The current state of the auth credentials.
+   *
+   * This essentially validates that enough credentials are available to use
+   * Sync.
+   */
+  get currentAuthState() {
+    if (!this.username) {
+      return LOGIN_FAILED_NO_USERNAME;
+    }
+
+    if (Utils.mpLocked()) {
+      return STATUS_OK;
+    }
+
+    if (!this.basicPassword) {
+      return LOGIN_FAILED_NO_PASSWORD;
+    }
+
+    if (!this.syncKey) {
+      return LOGIN_FAILED_NO_PASSPHRASE;
+    }
+
+    // If we have a Sync Key but no bundle, bundle creation failed, which
+    // implies a bad Sync Key.
+    if (!this.syncKeyBundle) {
+      return LOGIN_FAILED_INVALID_PASSPHRASE;
+    }
+
+    return STATUS_OK;
+  },
+
+  /**
+   * Verify the current auth state, unlocking the master-password if necessary.
+   *
+   * Returns a promise that resolves with the current auth state after
+   * attempting to unlock.
+   */
+  unlockAndVerifyAuthState: function() {
+    // Try to fetch the passphrase - this will prompt for MP unlock as a
+    // side-effect...
+    try {
+      this.syncKey;
+    } catch (ex) {
+      this._log.debug("Fetching passphrase threw " + ex +
+                      "; assuming master password locked.");
+      return Promise.resolve(MASTER_PASSWORD_LOCKED);
+    }
+    return Promise.resolve(STATUS_OK);
+  },
+
+  /**
+   * Persist credentials to password store.
+   *
+   * When credentials are updated, they are changed in memory only. This will
+   * need to be called to save them to the underlying password store.
+   *
+   * If the password store is locked (e.g. if the master password hasn't been
+   * entered), this could throw an exception.
+   */
+  persistCredentials: function persistCredentials(force) {
+    if (this._basicPasswordUpdated || force) {
+      if (this._basicPassword) {
+        this._setLogin(PWDMGR_PASSWORD_REALM, this.username,
+                       this._basicPassword);
+      } else {
+        for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) {
+          Services.logins.removeLogin(login);
+        }
+      }
+
+      this._basicPasswordUpdated = false;
+    }
+
+    if (this._syncKeyUpdated || force) {
+      if (this._syncKey) {
+        this._setLogin(PWDMGR_PASSPHRASE_REALM, this.username, this._syncKey);
+      } else {
+        for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
+          Services.logins.removeLogin(login);
+        }
+      }
+
+      this._syncKeyUpdated = false;
+    }
+
+  },
+
+  /**
+   * Deletes the Sync Key from the system.
+   */
+  deleteSyncKey: function deleteSyncKey() {
+    this.syncKey = null;
+    this.persistCredentials();
+  },
+
+  hasBasicCredentials: function hasBasicCredentials() {
+    // Because JavaScript.
+    return this.username && this.basicPassword && true;
+  },
+
+  /**
+   * Pre-fetches any information that might help with migration away from this
+   * identity.  Called after every sync and is really just an optimization that
+   * allows us to avoid a network request for when we actually need the
+   * migration info.
+   */
+  prefetchMigrationSentinel: function(service) {
+    // Try and fetch the migration sentinel - it will end up in the recordManager
+    // cache.
+    try {
+      service.recordManager.get(service.storageURL + "meta/fxa_credentials");
+    } catch (ex) {
+      if (Async.isShutdownException(ex)) {
+        throw ex;
+      }
+      this._log.warn("Failed to pre-fetch the migration sentinel", ex);
+    }
+  },
+
+  /**
+   * Obtains the array of basic logins from nsiPasswordManager.
+   */
+  _getLogins: function _getLogins(realm) {
+    return Services.logins.findLogins({}, PWDMGR_HOST, null, realm);
+  },
+
+  /**
+   * Set a login in the password manager.
+   *
+   * This has the side-effect of deleting any other logins for the specified
+   * realm.
+   */
+  _setLogin: function _setLogin(realm, username, password) {
+    let exists = false;
+    for (let login of this._getLogins(realm)) {
+      if (login.username == username && login.password == password) {
+        exists = true;
+      } else {
+        this._log.debug("Pruning old login for " + username + " from " + realm);
+        Services.logins.removeLogin(login);
+      }
+    }
+
+    if (exists) {
+      return;
+    }
+
+    this._log.debug("Updating saved password for " + username + " in " +
+                    realm);
+
+    let loginInfo = new Components.Constructor(
+      "@mozilla.org/login-manager/loginInfo;1", Ci.nsILoginInfo, "init");
+    let login = new loginInfo(PWDMGR_HOST, null, realm, username,
+                                password, "", "");
+    Services.logins.addLogin(login);
+  },
+
+  /**
+    * Return credentials hosts for this identity only.
+    */
+  _getSyncCredentialsHosts: function() {
+    return Utils.getSyncCredentialsHostsLegacy();
+  },
+
+  /**
+   * Deletes Sync credentials from the password manager.
+   */
+  deleteSyncCredentials: function deleteSyncCredentials() {
+    for (let host of this._getSyncCredentialsHosts()) {
+      let logins = Services.logins.findLogins({}, host, "", "");
+      for (let login of logins) {
+        Services.logins.removeLogin(login);
+      }
+    }
+
+    // Wait until after store is updated in case it fails.
+    this._basicPassword = null;
+    this._basicPasswordAllowLookup = true;
+    this._basicPasswordUpdated = false;
+
+    this._syncKey = null;
+    // this._syncKeyBundle is nullified as part of _syncKey setter.
+    this._syncKeyAllowLookup = true;
+    this._syncKeyUpdated = false;
+  },
+
+  usernameFromAccount: function usernameFromAccount(value) {
+    // If we encounter characters not allowed by the API (as found for
+    // instance in an email address), hash the value.
+    if (value && value.match(/[^A-Z0-9._-]/i)) {
+      return Utils.sha1Base32(value.toLowerCase()).toLowerCase();
+    }
+
+    return value ? value.toLowerCase() : value;
+  },
+
+  /**
+   * Obtain a function to be used for adding auth to Resource HTTP requests.
+   */
+  getResourceAuthenticator: function getResourceAuthenticator() {
+    if (this.hasBasicCredentials()) {
+      return this._onResourceRequestBasic.bind(this);
+    }
+
+    return null;
+  },
+
+  /**
+   * Helper method to return an authenticator for basic Resource requests.
+   */
+  getBasicResourceAuthenticator:
+    function getBasicResourceAuthenticator(username, password) {
+
+    return function basicAuthenticator(resource) {
+      let value = "Basic " + btoa(username + ":" + password);
+      return {headers: {authorization: value}};
+    };
+  },
+
+  _onResourceRequestBasic: function _onResourceRequestBasic(resource) {
+    let value = "Basic " + btoa(this.username + ":" + this.basicPassword);
+    return {headers: {authorization: value}};
+  },
+
+  _onResourceRequestMAC: function _onResourceRequestMAC(resource, method) {
+    // TODO Get identifier and key from somewhere.
+    let identifier;
+    let key;
+    let result = Utils.computeHTTPMACSHA1(identifier, key, method, resource.uri);
+
+    return {headers: {authorization: result.header}};
+  },
+
+  /**
+   * Obtain a function to be used for adding auth to RESTRequest instances.
+   */
+  getRESTRequestAuthenticator: function getRESTRequestAuthenticator() {
+    if (this.hasBasicCredentials()) {
+      return this.onRESTRequestBasic.bind(this);
+    }
+
+    return null;
+  },
+
+  onRESTRequestBasic: function onRESTRequestBasic(request) {
+    let up = this.username + ":" + this.basicPassword;
+    request.setHeader("authorization", "Basic " + btoa(up));
+  },
+
+  createClusterManager: function(service) {
+    Cu.import("resource://services-sync/stages/cluster.js");
+    return new ClusterManager(service);
+  },
+
+  offerSyncOptions: function () {
+    // Do nothing for Sync 1.1.
+    return {accepted: true};
+  },
+
+  // Tell Sync what the login status should be if it saw a 401 fetching
+  // info/collections as part of login verification (typically immediately
+  // after login.)
+  // In our case it means an authoritative "password is incorrect".
+  loginStatusFromVerification404() {
+    return LOGIN_FAILED_LOGIN_REJECTED;
+  }
+
+};

services/sync/modules/jpakeclient.js

@@ -0,0 +1,773 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["JPAKEClient", "SendCredentialsController"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/util.js");
+
+const REQUEST_TIMEOUT         = 60; // 1 minute
+const KEYEXCHANGE_VERSION     = 3;
+
+const JPAKE_SIGNERID_SENDER   = "sender";
+const JPAKE_SIGNERID_RECEIVER = "receiver";
+const JPAKE_LENGTH_SECRET     = 8;
+const JPAKE_LENGTH_CLIENTID   = 256;
+const JPAKE_VERIFY_VALUE      = "0123456789ABCDEF";
+
+
+/**
+ * Client to exchange encrypted data using the J-PAKE algorithm.
+ * The exchange between two clients of this type looks like this:
+ * 
+ * 
+ *  Mobile                        Server                        Desktop
+ *  ===================================================================
+ *                                   |
+ *  retrieve channel <---------------|
+ *  generate random secret           |
+ *  show PIN = secret + channel      |                 ask user for PIN
+ *  upload Mobile's message 1 ------>|
+ *                                   |----> retrieve Mobile's message 1
+ *                                   |<----- upload Desktop's message 1
+ *  retrieve Desktop's message 1 <---|
+ *  upload Mobile's message 2 ------>|
+ *                                   |----> retrieve Mobile's message 2
+ *                                   |                      compute key
+ *                                   |<----- upload Desktop's message 2
+ *  retrieve Desktop's message 2 <---|
+ *  compute key                      |
+ *  encrypt known value ------------>|
+ *                                   |-------> retrieve encrypted value
+ *                                   | verify against local known value
+ *
+ *   At this point Desktop knows whether the PIN was entered correctly.
+ *   If it wasn't, Desktop deletes the session. If it was, the account
+ *   setup can proceed. If Desktop doesn't yet have an account set up,
+ *   it will keep the channel open and let the user connect to or
+ *   create an account.
+ *
+ *                                   |              encrypt credentials
+ *                                   |<------------- upload credentials
+ *  retrieve credentials <-----------|
+ *  verify HMAC                      |
+ *  decrypt credentials              |
+ *  delete session ----------------->|
+ *  start syncing                    |
+ * 
+ * 
+ * Create a client object like so:
+ * 
+ *   let client = new JPAKEClient(controller);
+ * 
+ * The 'controller' object must implement the following methods:
+ * 
+ *   displayPIN(pin) -- Called when a PIN has been generated and is ready to
+ *     be displayed to the user. Only called on the client where the pairing
+ *     was initiated with 'receiveNoPIN()'.
+ * 
+ *   onPairingStart() -- Called when the pairing has started and messages are
+ *     being sent back and forth over the channel. Only called on the client
+ *     where the pairing was initiated with 'receiveNoPIN()'.
+ * 
+ *   onPaired() -- Called when the device pairing has been established and
+ *     we're ready to send the credentials over. To do that, the controller
+ *     must call 'sendAndComplete()' while the channel is active.
+ * 
+ *   onComplete(data) -- Called after transfer has been completed. On
+ *     the sending side this is called with no parameter and as soon as the
+ *     data has been uploaded. This does not mean the receiving side has
+ *     actually retrieved them yet.
+ *
+ *   onAbort(error) -- Called whenever an error is encountered. All errors lead
+ *     to an abort and the process has to be started again on both sides.
+ * 
+ * To start the data transfer on the receiving side, call
+ * 
+ *   client.receiveNoPIN();
+ * 
+ * This will allocate a new channel on the server, generate a PIN, have it
+ * displayed and then do the transfer once the protocol has been completed
+ * with the sending side.
+ * 
+ * To initiate the transfer from the sending side, call
+ * 
+ *   client.pairWithPIN(pin, true);
+ * 
+ * Once the pairing has been established, the controller's 'onPaired()' method
+ * will be called. To then transmit the data, call
+ * 
+ *   client.sendAndComplete(data);
+ * 
+ * To abort the process, call
+ * 
+ *   client.abort();
+ * 
+ * Note that after completion or abort, the 'client' instance may not be reused.
+ * You will have to create a new one in case you'd like to restart the process.
+ */
+this.JPAKEClient = function JPAKEClient(controller) {
+  this.controller = controller;
+
+  this._log = Log.repository.getLogger("Sync.JPAKEClient");
+  this._log.level = Log.Level[Svc.Prefs.get(
+    "log.logger.service.jpakeclient", "Debug")];
+
+  this._serverURL = Svc.Prefs.get("jpake.serverURL");
+  this._pollInterval = Svc.Prefs.get("jpake.pollInterval");
+  this._maxTries = Svc.Prefs.get("jpake.maxTries");
+  if (this._serverURL.slice(-1) != "/") {
+    this._serverURL += "/";
+  }
+
+  this._jpake = Cc["@mozilla.org/services-crypto/sync-jpake;1"]
+                  .createInstance(Ci.nsISyncJPAKE);
+
+  this._setClientID();
+}
+JPAKEClient.prototype = {
+
+  _chain: Async.chain,
+
+  /*
+   * Public API
+   */
+
+  /**
+   * Initiate pairing and receive data without providing a PIN. The PIN will
+   * be generated and passed on to the controller to be displayed to the user.
+   * 
+   * This is typically called on mobile devices where typing is tedious.
+   */
+  receiveNoPIN: function receiveNoPIN() {
+    this._my_signerid = JPAKE_SIGNERID_RECEIVER;
+    this._their_signerid = JPAKE_SIGNERID_SENDER;
+
+    this._secret = this._createSecret();
+
+    // Allow a large number of tries first while we wait for the PIN
+    // to be entered on the other device.
+    this._maxTries = Svc.Prefs.get("jpake.firstMsgMaxTries");
+    this._chain(this._getChannel,
+                this._computeStepOne,
+                this._putStep,
+                this._getStep,
+                function(callback) {
+                  // We fetched the first response from the other client.
+                  // Notify controller of the pairing starting.
+                  Utils.nextTick(this.controller.onPairingStart,
+                                 this.controller);
+
+                  // Now we can switch back to the smaller timeout.
+                  this._maxTries = Svc.Prefs.get("jpake.maxTries");
+                  callback();
+                },
+                this._computeStepTwo,
+                this._putStep,
+                this._getStep,
+                this._computeFinal,
+                this._computeKeyVerification,
+                this._putStep,
+                function(callback) {
+                  // Allow longer time-out for the last message.
+                  this._maxTries = Svc.Prefs.get("jpake.lastMsgMaxTries");
+                  callback();
+                },
+                this._getStep,
+                this._decryptData,
+                this._complete)();
+  },
+
+  /**
+   * Initiate pairing based on the PIN entered by the user.
+   * 
+   * This is typically called on desktop devices where typing is easier than
+   * on mobile.
+   * 
+   * @param pin
+   *        12 character string (in human-friendly base32) containing the PIN
+   *        entered by the user.
+   * @param expectDelay
+   *        Flag that indicates that a significant delay between the pairing
+   *        and the sending should be expected. v2 and earlier of the protocol
+   *        did not allow for this and the pairing to a v2 or earlier client
+   *        will be aborted if this flag is 'true'.
+   */
+  pairWithPIN: function pairWithPIN(pin, expectDelay) {
+    this._my_signerid = JPAKE_SIGNERID_SENDER;
+    this._their_signerid = JPAKE_SIGNERID_RECEIVER;
+
+    this._channel = pin.slice(JPAKE_LENGTH_SECRET);
+    this._channelURL = this._serverURL + this._channel;
+    this._secret = pin.slice(0, JPAKE_LENGTH_SECRET);
+
+    this._chain(this._computeStepOne,
+                this._getStep,
+                function (callback) {
+                  // Ensure that the other client can deal with a delay for
+                  // the last message if that's requested by the caller.
+                  if (!expectDelay) {
+                    return callback();
+                  }
+                  if (!this._incoming.version || this._incoming.version < 3) {
+                    return this.abort(JPAKE_ERROR_DELAYUNSUPPORTED);
+                  }
+                  return callback();
+                },
+                this._putStep,
+                this._computeStepTwo,
+                this._getStep,
+                this._putStep,
+                this._computeFinal,
+                this._getStep,
+                this._verifyPairing)();
+  },
+
+  /**
+   * Send data after a successful pairing.
+   * 
+   * @param obj
+   *        Object containing the data to send. It will be serialized as JSON.
+   */
+  sendAndComplete: function sendAndComplete(obj) {
+    if (!this._paired || this._finished) {
+      this._log.error("Can't send data, no active pairing!");
+      throw "No active pairing!";
+    }
+    this._data = JSON.stringify(obj);
+    this._chain(this._encryptData,
+                this._putStep,
+                this._complete)();
+  },
+
+  /**
+   * Abort the current pairing. The channel on the server will be deleted
+   * if the abort wasn't due to a network or server error. The controller's
+   * 'onAbort()' method is notified in all cases.
+   * 
+   * @param error [optional]
+   *        Error constant indicating the reason for the abort. Defaults to
+   *        user abort.
+   */
+  abort: function abort(error) {
+    this._log.debug("Aborting...");
+    this._finished = true;
+    let self = this;
+
+    // Default to "user aborted".
+    if (!error) {
+      error = JPAKE_ERROR_USERABORT;
+    }
+
+    if (error == JPAKE_ERROR_CHANNEL ||
+        error == JPAKE_ERROR_NETWORK ||
+        error == JPAKE_ERROR_NODATA) {
+      Utils.nextTick(function() { this.controller.onAbort(error); }, this);
+    } else {
+      this._reportFailure(error, function() { self.controller.onAbort(error); });
+    }
+  },
+
+  /*
+   * Utilities
+   */
+
+  _setClientID: function _setClientID() {
+    let rng = Cc["@mozilla.org/security/random-generator;1"]
+                .createInstance(Ci.nsIRandomGenerator);
+    let bytes = rng.generateRandomBytes(JPAKE_LENGTH_CLIENTID / 2);
+    this._clientID = bytes.map(byte => ("0" + byte.toString(16)).slice(-2)).join("");
+  },
+
+  _createSecret: function _createSecret() {
+    // 0-9a-z without 1,l,o,0
+    const key = "23456789abcdefghijkmnpqrstuvwxyz";
+    let rng = Cc["@mozilla.org/security/random-generator;1"]
+                .createInstance(Ci.nsIRandomGenerator);
+    let bytes = rng.generateRandomBytes(JPAKE_LENGTH_SECRET);
+    return bytes.map(byte => key[Math.floor(byte * key.length / 256)]).join("");
+  },
+
+  _newRequest: function _newRequest(uri) {
+    let request = new RESTRequest(uri);
+    request.setHeader("X-KeyExchange-Id", this._clientID);
+    request.timeout = REQUEST_TIMEOUT;
+    return request;
+  },
+
+  /*
+   * Steps of J-PAKE procedure
+   */
+
+  _getChannel: function _getChannel(callback) {
+    this._log.trace("Requesting channel.");
+    let request = this._newRequest(this._serverURL + "new_channel");
+    request.get(Utils.bind2(this, function handleChannel(error) {
+      if (this._finished) {
+        return;
+      }
+
+      if (error) {
+        this._log.error("Error acquiring channel ID. " + error);
+        this.abort(JPAKE_ERROR_CHANNEL);
+        return;
+      }
+      if (request.response.status != 200) {
+        this._log.error("Error acquiring channel ID. Server responded with HTTP "
+                        + request.response.status);
+        this.abort(JPAKE_ERROR_CHANNEL);
+        return;
+      }
+
+      try {
+        this._channel = JSON.parse(request.response.body);
+      } catch (ex) {
+        this._log.error("Server responded with invalid JSON.");
+        this.abort(JPAKE_ERROR_CHANNEL);
+        return;
+      }
+      this._log.debug("Using channel " + this._channel);
+      this._channelURL = this._serverURL + this._channel;
+
+      // Don't block on UI code.
+      let pin = this._secret + this._channel;
+      Utils.nextTick(function() { this.controller.displayPIN(pin); }, this);
+      callback();
+    }));
+  },
+
+  // Generic handler for uploading data.
+  _putStep: function _putStep(callback) {
+    this._log.trace("Uploading message " + this._outgoing.type);
+    let request = this._newRequest(this._channelURL);
+    if (this._their_etag) {
+      request.setHeader("If-Match", this._their_etag);
+    } else {
+      request.setHeader("If-None-Match", "*");
+    }
+    request.put(this._outgoing, Utils.bind2(this, function (error) {
+      if (this._finished) {
+        return;
+      }
+
+      if (error) {
+        this._log.error("Error uploading data. " + error);
+        this.abort(JPAKE_ERROR_NETWORK);
+        return;
+      }
+      if (request.response.status != 200) {
+        this._log.error("Could not upload data. Server responded with HTTP "
+                        + request.response.status);
+        this.abort(JPAKE_ERROR_SERVER);
+        return;
+      }
+      // There's no point in returning early here since the next step will
+      // always be a GET so let's pause for twice the poll interval.
+      this._my_etag = request.response.headers["etag"];
+      Utils.namedTimer(function () { callback(); }, this._pollInterval * 2,
+                       this, "_pollTimer");
+    }));
+  },
+
+  // Generic handler for polling for and retrieving data.
+  _pollTries: 0,
+  _getStep: function _getStep(callback) {
+    this._log.trace("Retrieving next message.");
+    let request = this._newRequest(this._channelURL);
+    if (this._my_etag) {
+      request.setHeader("If-None-Match", this._my_etag);
+    }
+
+    request.get(Utils.bind2(this, function (error) {
+      if (this._finished) {
+        return;
+      }
+
+      if (error) {
+        this._log.error("Error fetching data. " + error);
+        this.abort(JPAKE_ERROR_NETWORK);
+        return;
+      }
+
+      if (request.response.status == 304) {
+        this._log.trace("Channel hasn't been updated yet. Will try again later.");
+        if (this._pollTries >= this._maxTries) {
+          this._log.error("Tried for " + this._pollTries + " times, aborting.");
+          this.abort(JPAKE_ERROR_TIMEOUT);
+          return;
+        }
+        this._pollTries += 1;
+        Utils.namedTimer(function() { this._getStep(callback); },
+                         this._pollInterval, this, "_pollTimer");
+        return;
+      }
+      this._pollTries = 0;
+
+      if (request.response.status == 404) {
+        this._log.error("No data found in the channel.");
+        this.abort(JPAKE_ERROR_NODATA);
+        return;
+      }
+      if (request.response.status != 200) {
+        this._log.error("Could not retrieve data. Server responded with HTTP "
+                        + request.response.status);
+        this.abort(JPAKE_ERROR_SERVER);
+        return;
+      }
+
+      this._their_etag = request.response.headers["etag"];
+      if (!this._their_etag) {
+        this._log.error("Server did not supply ETag for message: "
+                        + request.response.body);
+        this.abort(JPAKE_ERROR_SERVER);
+        return;
+      }
+
+      try {
+        this._incoming = JSON.parse(request.response.body);
+      } catch (ex) {
+        this._log.error("Server responded with invalid JSON.");
+        this.abort(JPAKE_ERROR_INVALID);
+        return;
+      }
+      this._log.trace("Fetched message " + this._incoming.type);
+      callback();
+    }));
+  },
+
+  _reportFailure: function _reportFailure(reason, callback) {
+    this._log.debug("Reporting failure to server.");
+    let request = this._newRequest(this._serverURL + "report");
+    request.setHeader("X-KeyExchange-Cid", this._channel);
+    request.setHeader("X-KeyExchange-Log", reason);
+    request.post("", Utils.bind2(this, function (error) {
+      if (error) {
+        this._log.warn("Report failed: " + error);
+      } else if (request.response.status != 200) {
+        this._log.warn("Report failed. Server responded with HTTP "
+                       + request.response.status);
+      }
+
+      // Do not block on errors, we're done or aborted by now anyway.
+      callback();
+    }));
+  },
+
+  _computeStepOne: function _computeStepOne(callback) {
+    this._log.trace("Computing round 1.");
+    let gx1 = {};
+    let gv1 = {};
+    let r1 = {};
+    let gx2 = {};
+    let gv2 = {};
+    let r2 = {};
+    try {
+      this._jpake.round1(this._my_signerid, gx1, gv1, r1, gx2, gv2, r2);
+    } catch (ex) {
+      this._log.error("JPAKE round 1 threw: " + ex);
+      this.abort(JPAKE_ERROR_INTERNAL);
+      return;
+    }
+    let one = {gx1: gx1.value,
+               gx2: gx2.value,
+               zkp_x1: {gr: gv1.value, b: r1.value, id: this._my_signerid},
+               zkp_x2: {gr: gv2.value, b: r2.value, id: this._my_signerid}};
+    this._outgoing = {type: this._my_signerid + "1",
+                      version: KEYEXCHANGE_VERSION,
+                      payload: one};
+    this._log.trace("Generated message " + this._outgoing.type);
+    callback();
+  },
+
+  _computeStepTwo: function _computeStepTwo(callback) {
+    this._log.trace("Computing round 2.");
+    if (this._incoming.type != this._their_signerid + "1") {
+      this._log.error("Invalid round 1 message: "
+                      + JSON.stringify(this._incoming));
+      this.abort(JPAKE_ERROR_WRONGMESSAGE);
+      return;
+    }
+
+    let step1 = this._incoming.payload;
+    if (!step1 || !step1.zkp_x1 || step1.zkp_x1.id != this._their_signerid
+        || !step1.zkp_x2 || step1.zkp_x2.id != this._their_signerid) {
+      this._log.error("Invalid round 1 payload: " + JSON.stringify(step1));
+      this.abort(JPAKE_ERROR_WRONGMESSAGE);
+      return;
+    }
+
+    let A = {};
+    let gvA = {};
+    let rA = {};
+
+    try {
+      this._jpake.round2(this._their_signerid, this._secret,
+                         step1.gx1, step1.zkp_x1.gr, step1.zkp_x1.b,
+                         step1.gx2, step1.zkp_x2.gr, step1.zkp_x2.b,
+                         A, gvA, rA);
+    } catch (ex) {
+      this._log.error("JPAKE round 2 threw: " + ex);
+      this.abort(JPAKE_ERROR_INTERNAL);
+      return;
+    }
+    let two = {A: A.value,
+               zkp_A: {gr: gvA.value, b: rA.value, id: this._my_signerid}};
+    this._outgoing = {type: this._my_signerid + "2",
+                      version: KEYEXCHANGE_VERSION,
+                      payload: two};
+    this._log.trace("Generated message " + this._outgoing.type);
+    callback();
+  },
+
+  _computeFinal: function _computeFinal(callback) {
+    if (this._incoming.type != this._their_signerid + "2") {
+      this._log.error("Invalid round 2 message: "
+                      + JSON.stringify(this._incoming));
+      this.abort(JPAKE_ERROR_WRONGMESSAGE);
+      return;
+    }
+
+    let step2 = this._incoming.payload;
+    if (!step2 || !step2.zkp_A || step2.zkp_A.id != this._their_signerid) {
+      this._log.error("Invalid round 2 payload: " + JSON.stringify(step1));
+      this.abort(JPAKE_ERROR_WRONGMESSAGE);
+      return;
+    }
+
+    let aes256Key = {};
+    let hmac256Key = {};
+
+    try {
+      this._jpake.final(step2.A, step2.zkp_A.gr, step2.zkp_A.b, HMAC_INPUT,
+                        aes256Key, hmac256Key);
+    } catch (ex) {
+      this._log.error("JPAKE final round threw: " + ex);
+      this.abort(JPAKE_ERROR_INTERNAL);
+      return;
+    }
+
+    this._crypto_key = aes256Key.value;
+    let hmac_key = Utils.makeHMACKey(Utils.safeAtoB(hmac256Key.value));
+    this._hmac_hasher = Utils.makeHMACHasher(Ci.nsICryptoHMAC.SHA256, hmac_key);
+
+    callback();
+  },
+
+  _computeKeyVerification: function _computeKeyVerification(callback) {
+    this._log.trace("Encrypting key verification value.");
+    let iv, ciphertext;
+    try {
+      iv = Svc.Crypto.generateRandomIV();
+      ciphertext = Svc.Crypto.encrypt(JPAKE_VERIFY_VALUE,
+                                      this._crypto_key, iv);
+    } catch (ex) {
+      this._log.error("Failed to encrypt key verification value.");
+      this.abort(JPAKE_ERROR_INTERNAL);
+      return;
+    }
+    this._outgoing = {type: this._my_signerid + "3",
+                      version: KEYEXCHANGE_VERSION,
+                      payload: {ciphertext: ciphertext, IV: iv}};
+    this._log.trace("Generated message " + this._outgoing.type);
+    callback();
+  },
+
+  _verifyPairing: function _verifyPairing(callback) {
+    this._log.trace("Verifying their key.");
+    if (this._incoming.type != this._their_signerid + "3") {
+      this._log.error("Invalid round 3 data: " +
+                      JSON.stringify(this._incoming));
+      this.abort(JPAKE_ERROR_WRONGMESSAGE);
+      return;
+    }
+    let step3 = this._incoming.payload;
+    let ciphertext;
+    try {
+      ciphertext = Svc.Crypto.encrypt(JPAKE_VERIFY_VALUE,
+                                      this._crypto_key, step3.IV);
+      if (ciphertext != step3.ciphertext) {
+        throw "Key mismatch!";
+      }
+    } catch (ex) {
+      this._log.error("Keys don't match!");
+      this.abort(JPAKE_ERROR_KEYMISMATCH);
+      return;
+    }
+
+    this._log.debug("Verified pairing!");
+    this._paired = true;
+    Utils.nextTick(function () { this.controller.onPaired(); }, this);
+    callback();
+  },
+
+  _encryptData: function _encryptData(callback) {
+    this._log.trace("Encrypting data.");
+    let iv, ciphertext, hmac;
+    try {
+      iv = Svc.Crypto.generateRandomIV();
+      ciphertext = Svc.Crypto.encrypt(this._data, this._crypto_key, iv);
+      hmac = Utils.bytesAsHex(Utils.digestUTF8(ciphertext, this._hmac_hasher));
+    } catch (ex) {
+      this._log.error("Failed to encrypt data.");
+      this.abort(JPAKE_ERROR_INTERNAL);
+      return;
+    }
+    this._outgoing = {type: this._my_signerid + "3",
+                      version: KEYEXCHANGE_VERSION,
+                      payload: {ciphertext: ciphertext, IV: iv, hmac: hmac}};
+    this._log.trace("Generated message " + this._outgoing.type);
+    callback();
+  },
+
+  _decryptData: function _decryptData(callback) {
+    this._log.trace("Verifying their key.");
+    if (this._incoming.type != this._their_signerid + "3") {
+      this._log.error("Invalid round 3 data: "
+                      + JSON.stringify(this._incoming));
+      this.abort(JPAKE_ERROR_WRONGMESSAGE);
+      return;
+    }
+    let step3 = this._incoming.payload;
+    try {
+      let hmac = Utils.bytesAsHex(
+        Utils.digestUTF8(step3.ciphertext, this._hmac_hasher));
+      if (hmac != step3.hmac) {
+        throw "HMAC validation failed!";
+      }
+    } catch (ex) {
+      this._log.error("HMAC validation failed.");
+      this.abort(JPAKE_ERROR_KEYMISMATCH);
+      return;
+    }
+
+    this._log.trace("Decrypting data.");
+    let cleartext;
+    try {      
+      cleartext = Svc.Crypto.decrypt(step3.ciphertext, this._crypto_key,
+                                     step3.IV);
+    } catch (ex) {
+      this._log.error("Failed to decrypt data.");
+      this.abort(JPAKE_ERROR_INTERNAL);
+      return;
+    }
+
+    try {
+      this._newData = JSON.parse(cleartext);
+    } catch (ex) {
+      this._log.error("Invalid data data: " + JSON.stringify(cleartext));
+      this.abort(JPAKE_ERROR_INVALID);
+      return;
+    }
+
+    this._log.trace("Decrypted data.");
+    callback();
+  },
+
+  _complete: function _complete() {
+    this._log.debug("Exchange completed.");
+    this._finished = true;
+    Utils.nextTick(function () { this.controller.onComplete(this._newData); },
+                   this);
+  }
+
+};
+
+
+/**
+ * Send credentials over an active J-PAKE channel.
+ *
+ * This object is designed to take over as the JPAKEClient controller,
+ * presumably replacing one that is UI-based which would either cause
+ * DOM objects to leak or the JPAKEClient to be GC'ed when the DOM
+ * context disappears. This object stays alive for the duration of the
+ * transfer by being strong-ref'ed as an nsIObserver.
+ *
+ * Credentials are sent after the first sync has been completed
+ * (successfully or not.)
+ *
+ * Usage:
+ *
+ *   jpakeclient.controller = new SendCredentialsController(jpakeclient,
+ *                                                          service);
+ *
+ */
+this.SendCredentialsController =
+ function SendCredentialsController(jpakeclient, service) {
+  this._log = Log.repository.getLogger("Sync.SendCredentialsController");
+  this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")];
+
+  this._log.trace("Loading.");
+  this.jpakeclient = jpakeclient;
+  this.service = service;
+
+  // Register ourselves as observers the first Sync finishing (either
+  // successfully or unsuccessfully, we don't care) or for removing
+  // this device's sync configuration, in case that happens while we
+  // haven't finished the first sync yet.
+  Services.obs.addObserver(this, "weave:service:sync:finish", false);
+  Services.obs.addObserver(this, "weave:service:sync:error",  false);
+  Services.obs.addObserver(this, "weave:service:start-over",  false);
+}
+SendCredentialsController.prototype = {
+
+  unload: function unload() {
+    this._log.trace("Unloading.");
+    try {
+      Services.obs.removeObserver(this, "weave:service:sync:finish");
+      Services.obs.removeObserver(this, "weave:service:sync:error");
+      Services.obs.removeObserver(this, "weave:service:start-over");
+    } catch (ex) {
+      // Ignore.
+    }
+  },
+
+  observe: function observe(subject, topic, data) {
+    switch (topic) {
+      case "weave:service:sync:finish":
+      case "weave:service:sync:error":
+        Utils.nextTick(this.sendCredentials, this);
+        break;
+      case "weave:service:start-over":
+        // This will call onAbort which will call unload().
+        this.jpakeclient.abort();
+        break;
+    }
+  },
+
+  sendCredentials: function sendCredentials() {
+    this._log.trace("Sending credentials.");
+    let credentials = {account:   this.service.identity.account,
+                       password:  this.service.identity.basicPassword,
+                       synckey:   this.service.identity.syncKey,
+                       serverURL: this.service.serverURL};
+    this.jpakeclient.sendAndComplete(credentials);
+  },
+
+  // JPAKEClient controller API
+
+  onComplete: function onComplete() {
+    this._log.debug("Exchange was completed successfully!");
+    this.unload();
+
+    // Schedule a Sync for soonish to fetch the data uploaded by the
+    // device with which we just paired.
+    this.service.scheduler.scheduleNextSync(this.service.scheduler.activeInterval);
+  },
+
+  onAbort: function onAbort(error) {
+    // It doesn't really matter why we aborted, but the channel is closed
+    // for sure, so we won't be able to do anything with it.
+    this._log.debug("Exchange was aborted with error: " + error);
+    this.unload();
+  },
+
+  // Irrelevant methods for this controller:
+  displayPIN: function displayPIN() {},
+  onPairingStart: function onPairingStart() {},
+  onPaired: function onPaired() {},
+};

services/sync/modules/keys.js

@@ -0,0 +1,214 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "BulkKeyBundle",
+  "SyncKeyBundle"
+];
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/util.js");
+
+/**
+ * Represents a pair of keys.
+ *
+ * Each key stored in a key bundle is 256 bits. One key is used for symmetric
+ * encryption. The other is used for HMAC.
+ *
+ * A KeyBundle by itself is just an anonymous pair of keys. Other types
+ * deriving from this one add semantics, such as associated collections or
+ * generating a key bundle via HKDF from another key.
+ */
+function KeyBundle() {
+  this._encrypt = null;
+  this._encryptB64 = null;
+  this._hmac = null;
+  this._hmacB64 = null;
+  this._hmacObj = null;
+  this._sha256HMACHasher = null;
+}
+KeyBundle.prototype = {
+  _encrypt: null,
+  _encryptB64: null,
+  _hmac: null,
+  _hmacB64: null,
+  _hmacObj: null,
+  _sha256HMACHasher: null,
+
+  equals: function equals(bundle) {
+    return bundle &&
+           (bundle.hmacKey == this.hmacKey) &&
+           (bundle.encryptionKey == this.encryptionKey);
+  },
+
+  /*
+   * Accessors for the two keys.
+   */
+  get encryptionKey() {
+    return this._encrypt;
+  },
+
+  set encryptionKey(value) {
+    if (!value || typeof value != "string") {
+      throw new Error("Encryption key can only be set to string values.");
+    }
+
+    if (value.length < 16) {
+      throw new Error("Encryption key must be at least 128 bits long.");
+    }
+
+    this._encrypt = value;
+    this._encryptB64 = btoa(value);
+  },
+
+  get encryptionKeyB64() {
+    return this._encryptB64;
+  },
+
+  get hmacKey() {
+    return this._hmac;
+  },
+
+  set hmacKey(value) {
+    if (!value || typeof value != "string") {
+      throw new Error("HMAC key can only be set to string values.");
+    }
+
+    if (value.length < 16) {
+      throw new Error("HMAC key must be at least 128 bits long.");
+    }
+
+    this._hmac = value;
+    this._hmacB64 = btoa(value);
+    this._hmacObj = value ? Utils.makeHMACKey(value) : null;
+    this._sha256HMACHasher = value ? Utils.makeHMACHasher(
+      Ci.nsICryptoHMAC.SHA256, this._hmacObj) : null;
+  },
+
+  get hmacKeyB64() {
+    return this._hmacB64;
+  },
+
+  get hmacKeyObject() {
+    return this._hmacObj;
+  },
+
+  get sha256HMACHasher() {
+    return this._sha256HMACHasher;
+  },
+
+  /**
+   * Populate this key pair with 2 new, randomly generated keys.
+   */
+  generateRandom: function generateRandom() {
+    let generatedHMAC = Svc.Crypto.generateRandomKey();
+    let generatedEncr = Svc.Crypto.generateRandomKey();
+    this.keyPairB64 = [generatedEncr, generatedHMAC];
+  },
+
+};
+
+/**
+ * Represents a KeyBundle associated with a collection.
+ *
+ * This is just a KeyBundle with a collection attached.
+ */
+this.BulkKeyBundle = function BulkKeyBundle(collection) {
+  let log = Log.repository.getLogger("Sync.BulkKeyBundle");
+  log.info("BulkKeyBundle being created for " + collection);
+  KeyBundle.call(this);
+
+  this._collection = collection;
+}
+
+BulkKeyBundle.prototype = {
+  __proto__: KeyBundle.prototype,
+
+  get collection() {
+    return this._collection;
+  },
+
+  /**
+   * Obtain the key pair in this key bundle.
+   *
+   * The returned keys are represented as raw byte strings.
+   */
+  get keyPair() {
+    return [this.encryptionKey, this.hmacKey];
+  },
+
+  set keyPair(value) {
+    if (!Array.isArray(value) || value.length != 2) {
+      throw new Error("BulkKeyBundle.keyPair value must be array of 2 keys.");
+    }
+
+    this.encryptionKey = value[0];
+    this.hmacKey       = value[1];
+  },
+
+  get keyPairB64() {
+    return [this.encryptionKeyB64, this.hmacKeyB64];
+  },
+
+  set keyPairB64(value) {
+    if (!Array.isArray(value) || value.length != 2) {
+      throw new Error("BulkKeyBundle.keyPairB64 value must be an array of 2 " +
+                      "keys.");
+    }
+
+    this.encryptionKey  = Utils.safeAtoB(value[0]);
+    this.hmacKey        = Utils.safeAtoB(value[1]);
+  },
+};
+
+/**
+ * Represents a key pair derived from a Sync Key via HKDF.
+ *
+ * Instances of this type should be considered immutable. You create an
+ * instance by specifying the username and 26 character "friendly" Base32
+ * encoded Sync Key. The Sync Key is derived at instance creation time.
+ *
+ * If the username or Sync Key is invalid, an Error will be thrown.
+ */
+this.SyncKeyBundle = function SyncKeyBundle(username, syncKey) {
+  let log = Log.repository.getLogger("Sync.SyncKeyBundle");
+  log.info("SyncKeyBundle being created.");
+  KeyBundle.call(this);
+
+  this.generateFromKey(username, syncKey);
+}
+SyncKeyBundle.prototype = {
+  __proto__: KeyBundle.prototype,
+
+  /*
+   * If we've got a string, hash it into keys and store them.
+   */
+  generateFromKey: function generateFromKey(username, syncKey) {
+    if (!username || (typeof username != "string")) {
+      throw new Error("Sync Key cannot be generated from non-string username.");
+    }
+
+    if (!syncKey || (typeof syncKey != "string")) {
+      throw new Error("Sync Key cannot be generated from non-string key.");
+    }
+
+    if (!Utils.isPassphrase(syncKey)) {
+      throw new Error("Provided key is not a passphrase, cannot derive Sync " +
+                      "Key Bundle.");
+    }
+
+    // Expand the base32 Sync Key to an AES 256 and 256 bit HMAC key.
+    let prk = Utils.decodeKeyBase32(syncKey);
+    let info = HMAC_INPUT + username;
+    let okm = Utils.hkdfExpand(prk, info, 32 * 2);
+    this.encryptionKey = okm.slice(0, 32);
+    this.hmacKey = okm.slice(32, 64);
+  },
+};
+

services/sync/modules/main.js

@@ -0,0 +1,30 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['Weave'];
+
+this.Weave = {};
+Components.utils.import("resource://services-sync/constants.js", Weave);
+var lazies = {
+  "jpakeclient.js":       ["JPAKEClient", "SendCredentialsController"],
+  "service.js":           ["Service"],
+  "status.js":            ["Status"],
+  "util.js":              ['Utils', 'Svc']
+};
+
+function lazyImport(module, dest, props) {
+  function getter(prop) {
+    return function() {
+      let ns = {};
+      Components.utils.import(module, ns);
+      delete dest[prop];
+      return dest[prop] = ns[prop];
+    };
+  }
+  props.forEach(function (prop) { dest.__defineGetter__(prop, getter(prop)); });
+}
+
+for (let mod in lazies) {
+  lazyImport("resource://services-sync/" + mod, Weave, lazies[mod]);
+}

services/sync/modules/policies.js

@@ -0,0 +1,983 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = [
+  "ErrorHandler",
+  "SyncScheduler",
+];
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/logmanager.js");
+Cu.import("resource://services-common/async.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "Status",
+                                  "resource://services-sync/status.js");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
+                                  "resource://gre/modules/AddonManager.jsm");
+
+// Get the value for an interval that's stored in preferences. To save users
+// from themselves (and us from them!) the minimum time they can specify
+// is 60s.
+function getThrottledIntervalPreference(prefName) {
+  return Math.max(Svc.Prefs.get(prefName), 60) * 1000;
+}
+
+this.SyncScheduler = function SyncScheduler(service) {
+  this.service = service;
+  this.init();
+}
+SyncScheduler.prototype = {
+  _log: Log.repository.getLogger("Sync.SyncScheduler"),
+
+  _fatalLoginStatus: [LOGIN_FAILED_NO_USERNAME,
+                      LOGIN_FAILED_NO_PASSWORD,
+                      LOGIN_FAILED_NO_PASSPHRASE,
+                      LOGIN_FAILED_INVALID_PASSPHRASE,
+                      LOGIN_FAILED_LOGIN_REJECTED],
+
+  /**
+   * The nsITimer object that schedules the next sync. See scheduleNextSync().
+   */
+  syncTimer: null,
+
+  setDefaults: function setDefaults() {
+    this._log.trace("Setting SyncScheduler policy values to defaults.");
+
+    let service = Cc["@mozilla.org/weave/service;1"]
+                    .getService(Ci.nsISupports)
+                    .wrappedJSObject;
+
+    let part = service.fxAccountsEnabled ? "fxa" : "sync11";
+    let prefSDInterval = "scheduler." + part + ".singleDeviceInterval";
+    this.singleDeviceInterval = getThrottledIntervalPreference(prefSDInterval);
+
+    this.idleInterval         = getThrottledIntervalPreference("scheduler.idleInterval");
+    this.activeInterval       = getThrottledIntervalPreference("scheduler.activeInterval");
+    this.immediateInterval    = getThrottledIntervalPreference("scheduler.immediateInterval");
+    this.eolInterval          = getThrottledIntervalPreference("scheduler.eolInterval");
+
+    // A user is non-idle on startup by default.
+    this.idle = false;
+
+    this.hasIncomingItems = false;
+
+    this.clearSyncTriggers();
+  },
+
+  // nextSync is in milliseconds, but prefs can't hold that much
+  get nextSync() {
+    return Svc.Prefs.get("nextSync", 0) * 1000;
+  },
+  set nextSync(value) {
+    Svc.Prefs.set("nextSync", Math.floor(value / 1000));
+  },
+
+  get syncInterval() {
+    return Svc.Prefs.get("syncInterval", this.singleDeviceInterval);
+  },
+  set syncInterval(value) {
+    Svc.Prefs.set("syncInterval", value);
+  },
+
+  get syncThreshold() {
+    return Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD);
+  },
+  set syncThreshold(value) {
+    Svc.Prefs.set("syncThreshold", value);
+  },
+
+  get globalScore() {
+    return Svc.Prefs.get("globalScore", 0);
+  },
+  set globalScore(value) {
+    Svc.Prefs.set("globalScore", value);
+  },
+
+  get numClients() {
+    return Svc.Prefs.get("numClients", 0);
+  },
+  set numClients(value) {
+    Svc.Prefs.set("numClients", value);
+  },
+
+  init: function init() {
+    this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")];
+    this.setDefaults();
+    Svc.Obs.add("weave:engine:score:updated", this);
+    Svc.Obs.add("network:offline-status-changed", this);
+    Svc.Obs.add("weave:service:sync:start", this);
+    Svc.Obs.add("weave:service:sync:finish", this);
+    Svc.Obs.add("weave:engine:sync:finish", this);
+    Svc.Obs.add("weave:engine:sync:error", this);
+    Svc.Obs.add("weave:service:login:error", this);
+    Svc.Obs.add("weave:service:logout:finish", this);
+    Svc.Obs.add("weave:service:sync:error", this);
+    Svc.Obs.add("weave:service:backoff:interval", this);
+    Svc.Obs.add("weave:service:ready", this);
+    Svc.Obs.add("weave:engine:sync:applied", this);
+    Svc.Obs.add("weave:service:setup-complete", this);
+    Svc.Obs.add("weave:service:start-over", this);
+    Svc.Obs.add("FxA:hawk:backoff:interval", this);
+
+    if (Status.checkSetup() == STATUS_OK) {
+      Svc.Obs.add("wake_notification", this);
+      Svc.Idle.addIdleObserver(this, Svc.Prefs.get("scheduler.idleTime"));
+    }
+  },
+
+  observe: function observe(subject, topic, data) {
+    this._log.trace("Handling " + topic);
+    switch(topic) {
+      case "weave:engine:score:updated":
+        if (Status.login == LOGIN_SUCCEEDED) {
+          Utils.namedTimer(this.calculateScore, SCORE_UPDATE_DELAY, this,
+                           "_scoreTimer");
+        }
+        break;
+      case "network:offline-status-changed":
+        // Whether online or offline, we'll reschedule syncs
+        this._log.trace("Network offline status change: " + data);
+        this.checkSyncStatus();
+        break;
+      case "weave:service:sync:start":
+        // Clear out any potentially pending syncs now that we're syncing
+        this.clearSyncTriggers();
+
+        // reset backoff info, if the server tells us to continue backing off,
+        // we'll handle that later
+        Status.resetBackoff();
+
+        this.globalScore = 0;
+        break;
+      case "weave:service:sync:finish":
+        this.nextSync = 0;
+        this.adjustSyncInterval();
+
+        if (Status.service == SYNC_FAILED_PARTIAL && this.requiresBackoff) {
+          this.requiresBackoff = false;
+          this.handleSyncError();
+          return;
+        }
+
+        let sync_interval;
+        this._syncErrors = 0;
+        if (Status.sync == NO_SYNC_NODE_FOUND) {
+          this._log.trace("Scheduling a sync at interval NO_SYNC_NODE_FOUND.");
+          sync_interval = NO_SYNC_NODE_INTERVAL;
+        }
+        this.scheduleNextSync(sync_interval);
+        break;
+      case "weave:engine:sync:finish":
+        if (data == "clients") {
+          // Update the client mode because it might change what we sync.
+          this.updateClientMode();
+        }
+        break;
+      case "weave:engine:sync:error":
+        // `subject` is the exception thrown by an engine's sync() method.
+        let exception = subject;
+        if (exception.status >= 500 && exception.status <= 504) {
+          this.requiresBackoff = true;
+        }
+        break;
+      case "weave:service:login:error":
+        this.clearSyncTriggers();
+
+        if (Status.login == MASTER_PASSWORD_LOCKED) {
+          // Try again later, just as if we threw an error... only without the
+          // error count.
+          this._log.debug("Couldn't log in: master password is locked.");
+          this._log.trace("Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL");
+          this.scheduleAtInterval(MASTER_PASSWORD_LOCKED_RETRY_INTERVAL);
+        } else if (this._fatalLoginStatus.indexOf(Status.login) == -1) {
+          // Not a fatal login error, just an intermittent network or server
+          // issue. Keep on syncin'.
+          this.checkSyncStatus();
+        }
+        break;
+      case "weave:service:logout:finish":
+        // Start or cancel the sync timer depending on if
+        // logged in or logged out
+        this.checkSyncStatus();
+        break;
+      case "weave:service:sync:error":
+        // There may be multiple clients but if the sync fails, client mode
+        // should still be updated so that the next sync has a correct interval.
+        this.updateClientMode();
+        this.adjustSyncInterval();
+        this.nextSync = 0;
+        this.handleSyncError();
+        break;
+      case "FxA:hawk:backoff:interval":
+      case "weave:service:backoff:interval":
+        let requested_interval = subject * 1000;
+        this._log.debug("Got backoff notification: " + requested_interval + "ms");
+        // Leave up to 25% more time for the back off.
+        let interval = requested_interval * (1 + Math.random() * 0.25);
+        Status.backoffInterval = interval;
+        Status.minimumNextSync = Date.now() + requested_interval;
+        this._log.debug("Fuzzed minimum next sync: " + Status.minimumNextSync);
+        break;
+      case "weave:service:ready":
+        // Applications can specify this preference if they want autoconnect
+        // to happen after a fixed delay.
+        let delay = Svc.Prefs.get("autoconnectDelay");
+        if (delay) {
+          this.delayedAutoConnect(delay);
+        }
+        break;
+      case "weave:engine:sync:applied":
+        let numItems = subject.succeeded;
+        this._log.trace("Engine " + data + " successfully applied " + numItems +
+                        " items.");
+        if (numItems) {
+          this.hasIncomingItems = true;
+        }
+        break;
+      case "weave:service:setup-complete":
+         Services.prefs.savePrefFile(null);
+         Svc.Idle.addIdleObserver(this, Svc.Prefs.get("scheduler.idleTime"));
+         Svc.Obs.add("wake_notification", this);
+         break;
+      case "weave:service:start-over":
+         this.setDefaults();
+         try {
+           Svc.Idle.removeIdleObserver(this, Svc.Prefs.get("scheduler.idleTime"));
+         } catch (ex) {
+           if (ex.result != Cr.NS_ERROR_FAILURE) {
+             throw ex;
+           }
+           // In all likelihood we didn't have an idle observer registered yet.
+           // It's all good.
+         }
+         break;
+      case "idle":
+        this._log.trace("We're idle.");
+        this.idle = true;
+        // Adjust the interval for future syncs. This won't actually have any
+        // effect until the next pending sync (which will happen soon since we
+        // were just active.)
+        this.adjustSyncInterval();
+        break;
+      case "active":
+        this._log.trace("Received notification that we're back from idle.");
+        this.idle = false;
+        Utils.namedTimer(function onBack() {
+          if (this.idle) {
+            this._log.trace("... and we're idle again. " +
+                            "Ignoring spurious back notification.");
+            return;
+          }
+
+          this._log.trace("Genuine return from idle. Syncing.");
+          // Trigger a sync if we have multiple clients.
+          if (this.numClients > 1) {
+            this.scheduleNextSync(0);
+          }
+        }, IDLE_OBSERVER_BACK_DELAY, this, "idleDebouncerTimer");
+        break;
+      case "wake_notification":
+        this._log.debug("Woke from sleep.");
+        Utils.nextTick(() => {
+          // Trigger a sync if we have multiple clients. We give it 5 seconds
+          // incase the network is still in the process of coming back up.
+          if (this.numClients > 1) {
+            this._log.debug("More than 1 client. Will sync in 5s.");
+            this.scheduleNextSync(5000);
+          }
+        });
+        break;
+    }
+  },
+
+  adjustSyncInterval: function adjustSyncInterval() {
+    if (Status.eol) {
+      this._log.debug("Server status is EOL; using eolInterval.");
+      this.syncInterval = this.eolInterval;
+      return;
+    }
+
+    if (this.numClients <= 1) {
+      this._log.trace("Adjusting syncInterval to singleDeviceInterval.");
+      this.syncInterval = this.singleDeviceInterval;
+      return;
+    }
+
+    // Only MULTI_DEVICE clients will enter this if statement
+    // since SINGLE_USER clients will be handled above.
+    if (this.idle) {
+      this._log.trace("Adjusting syncInterval to idleInterval.");
+      this.syncInterval = this.idleInterval;
+      return;
+    }
+
+    if (this.hasIncomingItems) {
+      this._log.trace("Adjusting syncInterval to immediateInterval.");
+      this.hasIncomingItems = false;
+      this.syncInterval = this.immediateInterval;
+    } else {
+      this._log.trace("Adjusting syncInterval to activeInterval.");
+      this.syncInterval = this.activeInterval;
+    }
+  },
+
+  calculateScore: function calculateScore() {
+    let engines = [this.service.clientsEngine].concat(this.service.engineManager.getEnabled());
+    for (let i = 0;i < engines.length;i++) {
+      this._log.trace(engines[i].name + ": score: " + engines[i].score);
+      this.globalScore += engines[i].score;
+      engines[i]._tracker.resetScore();
+    }
+
+    this._log.trace("Global score updated: " + this.globalScore);
+    this.checkSyncStatus();
+  },
+
+  /**
+   * Process the locally stored clients list to figure out what mode to be in
+   */
+  updateClientMode: function updateClientMode() {
+    // Nothing to do if it's the same amount
+    let numClients = this.service.clientsEngine.stats.numClients;
+    if (this.numClients == numClients)
+      return;
+
+    this._log.debug("Client count: " + this.numClients + " -> " + numClients);
+    this.numClients = numClients;
+
+    if (numClients <= 1) {
+      this._log.trace("Adjusting syncThreshold to SINGLE_USER_THRESHOLD");
+      this.syncThreshold = SINGLE_USER_THRESHOLD;
+    } else {
+      this._log.trace("Adjusting syncThreshold to MULTI_DEVICE_THRESHOLD");
+      this.syncThreshold = MULTI_DEVICE_THRESHOLD;
+    }
+    this.adjustSyncInterval();
+  },
+
+  /**
+   * Check if we should be syncing and schedule the next sync, if it's not scheduled
+   */
+  checkSyncStatus: function checkSyncStatus() {
+    // Should we be syncing now, if not, cancel any sync timers and return
+    // if we're in backoff, we'll schedule the next sync.
+    let ignore = [kSyncBackoffNotMet, kSyncMasterPasswordLocked];
+    let skip = this.service._checkSync(ignore);
+    this._log.trace("_checkSync returned \"" + skip + "\".");
+    if (skip) {
+      this.clearSyncTriggers();
+      return;
+    }
+
+    // Only set the wait time to 0 if we need to sync right away
+    let wait;
+    if (this.globalScore > this.syncThreshold) {
+      this._log.debug("Global Score threshold hit, triggering sync.");
+      wait = 0;
+    }
+    this.scheduleNextSync(wait);
+  },
+
+  /**
+   * Call sync() if Master Password is not locked.
+   *
+   * Otherwise, reschedule a sync for later.
+   */
+  syncIfMPUnlocked: function syncIfMPUnlocked() {
+    // No point if we got kicked out by the master password dialog.
+    if (Status.login == MASTER_PASSWORD_LOCKED &&
+        Utils.mpLocked()) {
+      this._log.debug("Not initiating sync: Login status is " + Status.login);
+
+      // If we're not syncing now, we need to schedule the next one.
+      this._log.trace("Scheduling a sync at MASTER_PASSWORD_LOCKED_RETRY_INTERVAL");
+      this.scheduleAtInterval(MASTER_PASSWORD_LOCKED_RETRY_INTERVAL);
+      return;
+    }
+
+    Utils.nextTick(this.service.sync, this.service);
+  },
+
+  /**
+   * Set a timer for the next sync
+   */
+  scheduleNextSync: function scheduleNextSync(interval) {
+    // If no interval was specified, use the current sync interval.
+    if (interval == null) {
+      interval = this.syncInterval;
+    }
+
+    // Ensure the interval is set to no less than the backoff.
+    if (Status.backoffInterval && interval < Status.backoffInterval) {
+      this._log.trace("Requested interval " + interval +
+                      " ms is smaller than the backoff interval. " +
+                      "Using backoff interval " +
+                      Status.backoffInterval + " ms instead.");
+      interval = Status.backoffInterval;
+    }
+
+    if (this.nextSync != 0) {
+      // There's already a sync scheduled. Don't reschedule if there's already
+      // a timer scheduled for sooner than requested.
+      let currentInterval = this.nextSync - Date.now();
+      this._log.trace("There's already a sync scheduled in " +
+                      currentInterval + " ms.");
+      if (currentInterval < interval && this.syncTimer) {
+        this._log.trace("Ignoring scheduling request for next sync in " +
+                        interval + " ms.");
+        return;
+      }
+    }
+
+    // Start the sync right away if we're already late.
+    if (interval <= 0) {
+      this._log.trace("Requested sync should happen right away.");
+      this.syncIfMPUnlocked();
+      return;
+    }
+
+    this._log.debug("Next sync in " + interval + " ms.");
+    Utils.namedTimer(this.syncIfMPUnlocked, interval, this, "syncTimer");
+
+    // Save the next sync time in-case sync is disabled (logout/offline/etc.)
+    this.nextSync = Date.now() + interval;
+  },
+
+
+  /**
+   * Incorporates the backoff/retry logic used in error handling and elective
+   * non-syncing.
+   */
+  scheduleAtInterval: function scheduleAtInterval(minimumInterval) {
+    let interval = Utils.calculateBackoff(this._syncErrors,
+                                          MINIMUM_BACKOFF_INTERVAL,
+                                          Status.backoffInterval);
+    if (minimumInterval) {
+      interval = Math.max(minimumInterval, interval);
+    }
+
+    this._log.debug("Starting client-initiated backoff. Next sync in " +
+                    interval + " ms.");
+    this.scheduleNextSync(interval);
+  },
+
+ /**
+  * Automatically start syncing after the given delay (in seconds).
+  *
+  * Applications can define the `services.sync.autoconnectDelay` preference
+  * to have this called automatically during start-up with the pref value as
+  * the argument. Alternatively, they can call it themselves to control when
+  * Sync should first start to sync.
+  */
+  delayedAutoConnect: function delayedAutoConnect(delay) {
+    if (this.service._checkSetup() == STATUS_OK) {
+      Utils.namedTimer(this.autoConnect, delay * 1000, this, "_autoTimer");
+    }
+  },
+
+  autoConnect: function autoConnect() {
+    if (this.service._checkSetup() == STATUS_OK && !this.service._checkSync()) {
+      // Schedule a sync based on when a previous sync was scheduled.
+      // scheduleNextSync() will do the right thing if that time lies in
+      // the past.
+      this.scheduleNextSync(this.nextSync - Date.now());
+    }
+
+    // Once autoConnect is called we no longer need _autoTimer.
+    if (this._autoTimer) {
+      this._autoTimer.clear();
+    }
+  },
+
+  _syncErrors: 0,
+  /**
+   * Deal with sync errors appropriately
+   */
+  handleSyncError: function handleSyncError() {
+    this._log.trace("In handleSyncError. Error count: " + this._syncErrors);
+    this._syncErrors++;
+
+    // Do nothing on the first couple of failures, if we're not in
+    // backoff due to 5xx errors.
+    if (!Status.enforceBackoff) {
+      if (this._syncErrors < MAX_ERROR_COUNT_BEFORE_BACKOFF) {
+        this.scheduleNextSync();
+        return;
+      }
+      this._log.debug("Sync error count has exceeded " +
+                      MAX_ERROR_COUNT_BEFORE_BACKOFF + "; enforcing backoff.");
+      Status.enforceBackoff = true;
+    }
+
+    this.scheduleAtInterval();
+  },
+
+
+  /**
+   * Remove any timers/observers that might trigger a sync
+   */
+  clearSyncTriggers: function clearSyncTriggers() {
+    this._log.debug("Clearing sync triggers and the global score.");
+    this.globalScore = this.nextSync = 0;
+
+    // Clear out any scheduled syncs
+    if (this.syncTimer)
+      this.syncTimer.clear();
+  },
+
+};
+
+this.ErrorHandler = function ErrorHandler(service) {
+  this.service = service;
+  this.init();
+}
+ErrorHandler.prototype = {
+  MINIMUM_ALERT_INTERVAL_MSEC: 604800000,   // One week.
+
+  /**
+   * Flag that turns on error reporting for all errors, incl. network errors.
+   */
+  dontIgnoreErrors: false,
+
+  /**
+   * Flag that indicates if we have already reported a prolonged failure.
+   * Once set, we don't report it again, meaning this error is only reported
+   * one per run.
+   */
+  didReportProlongedError: false,
+
+  init: function init() {
+    Svc.Obs.add("weave:engine:sync:applied", this);
+    Svc.Obs.add("weave:engine:sync:error", this);
+    Svc.Obs.add("weave:service:login:error", this);
+    Svc.Obs.add("weave:service:sync:error", this);
+    Svc.Obs.add("weave:service:sync:finish", this);
+
+    this.initLogs();
+  },
+
+  initLogs: function initLogs() {
+    this._log = Log.repository.getLogger("Sync.ErrorHandler");
+    this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")];
+
+    let root = Log.repository.getLogger("Sync");
+    root.level = Log.Level[Svc.Prefs.get("log.rootLogger")];
+
+    let logs = ["Sync", "FirefoxAccounts", "Hawk", "Common.TokenServerClient",
+                "Sync.SyncMigration", "browserwindow.syncui",
+                "Services.Common.RESTRequest", "Services.Common.RESTRequest",
+                "BookmarkSyncUtils"
+               ];
+
+    this._logManager = new LogManager(Svc.Prefs, logs, "sync");
+  },
+
+  observe: function observe(subject, topic, data) {
+    this._log.trace("Handling " + topic);
+    switch(topic) {
+      case "weave:engine:sync:applied":
+        if (subject.newFailed) {
+          // An engine isn't able to apply one or more incoming records.
+          // We don't fail hard on this, but it usually indicates a bug,
+          // so for now treat it as sync error (c.f. Service._syncEngine())
+          Status.engines = [data, ENGINE_APPLY_FAIL];
+          this._log.debug(data + " failed to apply some records.");
+        }
+        break;
+      case "weave:engine:sync:error": {
+        let exception = subject;  // exception thrown by engine's sync() method
+        let engine_name = data;   // engine name that threw the exception
+
+        this.checkServerError(exception);
+
+        Status.engines = [engine_name, exception.failureCode || ENGINE_UNKNOWN_FAIL];
+        if (Async.isShutdownException(exception)) {
+          this._log.debug(engine_name + " was interrupted due to the application shutting down");
+        } else {
+          this._log.debug(engine_name + " failed", exception);
+          Services.telemetry.getKeyedHistogramById("WEAVE_ENGINE_SYNC_ERRORS")
+                            .add(engine_name);
+        }
+        break;
+      }
+      case "weave:service:login:error":
+        this._log.error("Sync encountered a login error");
+        this.resetFileLog();
+
+        if (this.shouldReportError()) {
+          this.notifyOnNextTick("weave:ui:login:error");
+        } else {
+          this.notifyOnNextTick("weave:ui:clear-error");
+        }
+
+        this.dontIgnoreErrors = false;
+        break;
+      case "weave:service:sync:error": {
+        if (Status.sync == CREDENTIALS_CHANGED) {
+          this.service.logout();
+        }
+
+        let exception = subject;
+        if (Async.isShutdownException(exception)) {
+          // If we are shutting down we just log the fact, attempt to flush
+          // the log file and get out of here!
+          this._log.error("Sync was interrupted due to the application shutting down");
+          this.resetFileLog();
+          break;
+        }
+
+        // Not a shutdown related exception...
+        this._log.error("Sync encountered an error", exception);
+        this.resetFileLog();
+
+        if (this.shouldReportError()) {
+          this.notifyOnNextTick("weave:ui:sync:error");
+        } else {
+          this.notifyOnNextTick("weave:ui:sync:finish");
+        }
+
+        this.dontIgnoreErrors = false;
+        break;
+      }
+      case "weave:service:sync:finish":
+        this._log.trace("Status.service is " + Status.service);
+
+        // Check both of these status codes: in the event of a failure in one
+        // engine, Status.service will be SYNC_FAILED_PARTIAL despite
+        // Status.sync being SYNC_SUCCEEDED.
+        // *facepalm*
+        if (Status.sync    == SYNC_SUCCEEDED &&
+            Status.service == STATUS_OK) {
+          // Great. Let's clear our mid-sync 401 note.
+          this._log.trace("Clearing lastSyncReassigned.");
+          Svc.Prefs.reset("lastSyncReassigned");
+        }
+
+        if (Status.service == SYNC_FAILED_PARTIAL) {
+          this._log.error("Some engines did not sync correctly.");
+          this.resetFileLog();
+
+          if (this.shouldReportError()) {
+            this.dontIgnoreErrors = false;
+            this.notifyOnNextTick("weave:ui:sync:error");
+            break;
+          }
+        } else {
+          this.resetFileLog();
+        }
+        this.dontIgnoreErrors = false;
+        this.notifyOnNextTick("weave:ui:sync:finish");
+        break;
+    }
+  },
+
+  notifyOnNextTick: function notifyOnNextTick(topic) {
+    Utils.nextTick(function() {
+      this._log.trace("Notifying " + topic +
+                      ". Status.login is " + Status.login +
+                      ". Status.sync is " + Status.sync);
+      Svc.Obs.notify(topic);
+    }, this);
+  },
+
+  /**
+   * Trigger a sync and don't muffle any errors, particularly network errors.
+   */
+  syncAndReportErrors: function syncAndReportErrors() {
+    this._log.debug("Beginning user-triggered sync.");
+
+    this.dontIgnoreErrors = true;
+    Utils.nextTick(this.service.sync, this.service);
+  },
+
+  _dumpAddons: function _dumpAddons() {
+    // Just dump the items that sync may be concerned with. Specifically,
+    // active extensions that are not hidden.
+    let addonPromise = new Promise(resolve => {
+      try {
+        AddonManager.getAddonsByTypes(["extension"], resolve);
+      } catch (e) {
+        this._log.warn("Failed to dump addons", e)
+        resolve([])
+      }
+    });
+
+    return addonPromise.then(addons => {
+      let relevantAddons = addons.filter(x => x.isActive && !x.hidden);
+      this._log.debug("Addons installed", relevantAddons.length);
+      for (let addon of relevantAddons) {
+        this._log.debug(" - ${name}, version ${version}, id ${id}", addon);
+      }
+    });
+  },
+
+  /**
+   * Generate a log file for the sync that just completed
+   * and refresh the input & output streams.
+   */
+  resetFileLog: function resetFileLog() {
+    let onComplete = logType => {
+      Svc.Obs.notify("weave:service:reset-file-log");
+      this._log.trace("Notified: " + Date.now());
+      if (logType == this._logManager.ERROR_LOG_WRITTEN) {
+        Cu.reportError("Sync encountered an error - see about:sync-log for the log file.");
+      }
+    };
+
+    // If we're writing an error log, dump extensions that may be causing problems.
+    let beforeResetLog;
+    if (this._logManager.sawError) {
+      beforeResetLog = this._dumpAddons();
+    } else {
+      beforeResetLog = Promise.resolve();
+    }
+    // Note we do not return the promise here - the caller doesn't need to wait
+    // for this to complete.
+    beforeResetLog
+      .then(() => this._logManager.resetFileLog())
+      .then(onComplete, onComplete);
+  },
+
+  /**
+   * Translates server error codes to meaningful strings.
+   *
+   * @param code
+   *        server error code as an integer
+   */
+  errorStr: function errorStr(code) {
+    switch (code.toString()) {
+    case "1":
+      return "illegal-method";
+    case "2":
+      return "invalid-captcha";
+    case "3":
+      return "invalid-username";
+    case "4":
+      return "cannot-overwrite-resource";
+    case "5":
+      return "userid-mismatch";
+    case "6":
+      return "json-parse-failure";
+    case "7":
+      return "invalid-password";
+    case "8":
+      return "invalid-record";
+    case "9":
+      return "weak-password";
+    default:
+      return "generic-server-error";
+    }
+  },
+
+  // A function to indicate if Sync errors should be "reported" - which in this
+  // context really means "should be notify observers of an error" - but note
+  // that since bug 1180587, no one is going to surface an error to the user.
+  shouldReportError: function shouldReportError() {
+    if (Status.login == MASTER_PASSWORD_LOCKED) {
+      this._log.trace("shouldReportError: false (master password locked).");
+      return false;
+    }
+
+    if (this.dontIgnoreErrors) {
+      return true;
+    }
+
+    if (Status.login == LOGIN_FAILED_LOGIN_REJECTED) {
+      // An explicit LOGIN_REJECTED state is always reported (bug 1081158)
+      this._log.trace("shouldReportError: true (login was rejected)");
+      return true;
+    }
+
+    let lastSync = Svc.Prefs.get("lastSync");
+    if (lastSync && ((Date.now() - Date.parse(lastSync)) >
+        Svc.Prefs.get("errorhandler.networkFailureReportTimeout") * 1000)) {
+      Status.sync = PROLONGED_SYNC_FAILURE;
+      if (this.didReportProlongedError) {
+        this._log.trace("shouldReportError: false (prolonged sync failure, but" +
+                        " we've already reported it).");
+        return false;
+      }
+      this._log.trace("shouldReportError: true (first prolonged sync failure).");
+      this.didReportProlongedError = true;
+      return true;
+    }
+
+    // We got a 401 mid-sync. Wait for the next sync before actually handling
+    // an error. This assumes that we'll get a 401 again on a login fetch in
+    // order to report the error.
+    if (!this.service.clusterURL) {
+      this._log.trace("shouldReportError: false (no cluster URL; " +
+                      "possible node reassignment).");
+      return false;
+    }
+
+
+    let result = ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 &&
+                  [Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1);
+    this._log.trace("shouldReportError: ${result} due to login=${login}, sync=${sync}",
+                    {result, login: Status.login, sync: Status.sync});
+    return result;
+  },
+
+  get currentAlertMode() {
+    return Svc.Prefs.get("errorhandler.alert.mode");
+  },
+
+  set currentAlertMode(str) {
+    return Svc.Prefs.set("errorhandler.alert.mode", str);
+  },
+
+  get earliestNextAlert() {
+    return Svc.Prefs.get("errorhandler.alert.earliestNext", 0) * 1000;
+  },
+
+  set earliestNextAlert(msec) {
+    return Svc.Prefs.set("errorhandler.alert.earliestNext", msec / 1000);
+  },
+
+  clearServerAlerts: function () {
+    // If we have any outstanding alerts, apparently they're no longer relevant.
+    Svc.Prefs.resetBranch("errorhandler.alert");
+  },
+
+  /**
+   * X-Weave-Alert headers can include a JSON object:
+   *
+   *   {
+   *    "code":    // One of "hard-eol", "soft-eol".
+   *    "url":     // For "Learn more" link.
+   *    "message": // Logged in Sync logs.
+   *   }
+   */
+  handleServerAlert: function (xwa) {
+    if (!xwa.code) {
+      this._log.warn("Got structured X-Weave-Alert, but no alert code.");
+      return;
+    }
+
+    switch (xwa.code) {
+      // Gently and occasionally notify the user that this service will be
+      // shutting down.
+      case "soft-eol":
+        // Fall through.
+
+      // Tell the user that this service has shut down, and drop our syncing
+      // frequency dramatically.
+      case "hard-eol":
+        // Note that both of these alerts should be subservient to future "sign
+        // in with your Firefox Account" storage alerts.
+        if ((this.currentAlertMode != xwa.code) ||
+            (this.earliestNextAlert < Date.now())) {
+          Utils.nextTick(function() {
+            Svc.Obs.notify("weave:eol", xwa);
+          }, this);
+          this._log.error("X-Weave-Alert: " + xwa.code + ": " + xwa.message);
+          this.earliestNextAlert = Date.now() + this.MINIMUM_ALERT_INTERVAL_MSEC;
+          this.currentAlertMode = xwa.code;
+        }
+        break;
+      default:
+        this._log.debug("Got unexpected X-Weave-Alert code: " + xwa.code);
+    }
+  },
+
+  /**
+   * Handle HTTP response results or exceptions and set the appropriate
+   * Status.* bits.
+   *
+   * This method also looks for "side-channel" warnings.
+   */
+  checkServerError: function (resp) {
+    switch (resp.status) {
+      case 200:
+      case 404:
+      case 513:
+        let xwa = resp.headers['x-weave-alert'];
+
+        // Only process machine-readable alerts.
+        if (!xwa || !xwa.startsWith("{")) {
+          this.clearServerAlerts();
+          return;
+        }
+
+        try {
+          xwa = JSON.parse(xwa);
+        } catch (ex) {
+          this._log.warn("Malformed X-Weave-Alert from server: " + xwa);
+          return;
+        }
+
+        this.handleServerAlert(xwa);
+        break;
+
+      case 400:
+        if (resp == RESPONSE_OVER_QUOTA) {
+          Status.sync = OVER_QUOTA;
+        }
+        break;
+
+      case 401:
+        this.service.logout();
+        this._log.info("Got 401 response; resetting clusterURL.");
+        this.service.clusterURL = null;
+
+        let delay = 0;
+        if (Svc.Prefs.get("lastSyncReassigned")) {
+          // We got a 401 in the middle of the previous sync, and we just got
+          // another. Login must have succeeded in order for us to get here, so
+          // the password should be correct.
+          // This is likely to be an intermittent server issue, so back off and
+          // give it time to recover.
+          this._log.warn("Last sync also failed for 401. Delaying next sync.");
+          delay = MINIMUM_BACKOFF_INTERVAL;
+        } else {
+          this._log.debug("New mid-sync 401 failure. Making a note.");
+          Svc.Prefs.set("lastSyncReassigned", true);
+        }
+        this._log.info("Attempting to schedule another sync.");
+        this.service.scheduler.scheduleNextSync(delay);
+        break;
+
+      case 500:
+      case 502:
+      case 503:
+      case 504:
+        Status.enforceBackoff = true;
+        if (resp.status == 503 && resp.headers["retry-after"]) {
+          let retryAfter = resp.headers["retry-after"];
+          this._log.debug("Got Retry-After: " + retryAfter);
+          if (this.service.isLoggedIn) {
+            Status.sync = SERVER_MAINTENANCE;
+          } else {
+            Status.login = SERVER_MAINTENANCE;
+          }
+          Svc.Obs.notify("weave:service:backoff:interval",
+                         parseInt(retryAfter, 10));
+        }
+        break;
+    }
+
+    switch (resp.result) {
+      case Cr.NS_ERROR_UNKNOWN_HOST:
+      case Cr.NS_ERROR_CONNECTION_REFUSED:
+      case Cr.NS_ERROR_NET_TIMEOUT:
+      case Cr.NS_ERROR_NET_RESET:
+      case Cr.NS_ERROR_NET_INTERRUPT:
+      case Cr.NS_ERROR_PROXY_CONNECTION_REFUSED:
+        // The constant says it's about login, but in fact it just
+        // indicates general network error.
+        if (this.service.isLoggedIn) {
+          Status.sync = LOGIN_FAILED_NETWORK_ERROR;
+        } else {
+          Status.login = LOGIN_FAILED_NETWORK_ERROR;
+        }
+        break;
+    }
+  },
+};

services/sync/modules/resource.js

@@ -0,0 +1,669 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = [
+  "AsyncResource",
+  "Resource"
+];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/NetUtil.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/util.js");
+
+const DEFAULT_LOAD_FLAGS =
+  // Always validate the cache:
+  Ci.nsIRequest.LOAD_BYPASS_CACHE |
+  Ci.nsIRequest.INHIBIT_CACHING |
+  // Don't send user cookies over the wire (Bug 644734).
+  Ci.nsIRequest.LOAD_ANONYMOUS;
+
+/*
+ * AsyncResource represents a remote network resource, identified by a URI.
+ * Create an instance like so:
+ *
+ *   let resource = new AsyncResource("http://foobar.com/path/to/resource");
+ *
+ * The 'resource' object has the following methods to issue HTTP requests
+ * of the corresponding HTTP methods:
+ *
+ *   get(callback)
+ *   put(data, callback)
+ *   post(data, callback)
+ *   delete(callback)
+ *
+ * 'callback' is a function with the following signature:
+ *
+ *   function callback(error, result) {...}
+ *
+ * 'error' will be null on successful requests. Likewise, result will not be
+ * passed (=undefined) when an error occurs. Note that this is independent of
+ * the status of the HTTP response.
+ */
+this.AsyncResource = function AsyncResource(uri) {
+  this._log = Log.repository.getLogger(this._logName);
+  this._log.level =
+    Log.Level[Svc.Prefs.get("log.logger.network.resources")];
+  this.uri = uri;
+  this._headers = {};
+  this._onComplete = Utils.bind2(this, this._onComplete);
+}
+AsyncResource.prototype = {
+  _logName: "Sync.AsyncResource",
+
+  // ** {{{ AsyncResource.serverTime }}} **
+  //
+  // Caches the latest server timestamp (X-Weave-Timestamp header).
+  serverTime: null,
+
+  /**
+   * Callback to be invoked at request time to add authentication details.
+   *
+   * By default, a global authenticator is provided. If this is set, it will
+   * be used instead of the global one.
+   */
+  authenticator: null,
+
+  // Wait 5 minutes before killing a request.
+  ABORT_TIMEOUT: 300000,
+
+  // ** {{{ AsyncResource.headers }}} **
+  //
+  // Headers to be included when making a request for the resource.
+  // Note: Header names should be all lower case, there's no explicit
+  // check for duplicates due to case!
+  get headers() {
+    return this._headers;
+  },
+  set headers(value) {
+    this._headers = value;
+  },
+  setHeader: function Res_setHeader(header, value) {
+    this._headers[header.toLowerCase()] = value;
+  },
+  get headerNames() {
+    return Object.keys(this.headers);
+  },
+
+  // ** {{{ AsyncResource.uri }}} **
+  //
+  // URI representing this resource.
+  get uri() {
+    return this._uri;
+  },
+  set uri(value) {
+    if (typeof value == 'string')
+      this._uri = CommonUtils.makeURI(value);
+    else
+      this._uri = value;
+  },
+
+  // ** {{{ AsyncResource.spec }}} **
+  //
+  // Get the string representation of the URI.
+  get spec() {
+    if (this._uri)
+      return this._uri.spec;
+    return null;
+  },
+
+  // ** {{{ AsyncResource.data }}} **
+  //
+  // Get and set the data encapulated in the resource.
+  _data: null,
+  get data() {
+    return this._data;
+  },
+  set data(value) {
+    this._data = value;
+  },
+
+  // ** {{{ AsyncResource._createRequest }}} **
+  //
+  // This method returns a new IO Channel for requests to be made
+  // through. It is never called directly, only {{{_doRequest}}} uses it
+  // to obtain a request channel.
+  //
+  _createRequest: function Res__createRequest(method) {
+    let channel = NetUtil.newChannel({uri: this.spec, loadUsingSystemPrincipal: true})
+                         .QueryInterface(Ci.nsIRequest)
+                         .QueryInterface(Ci.nsIHttpChannel);
+
+    channel.loadFlags |= DEFAULT_LOAD_FLAGS;
+
+    // Setup a callback to handle channel notifications.
+    let listener = new ChannelNotificationListener(this.headerNames);
+    channel.notificationCallbacks = listener;
+
+    // Compose a UA string fragment from the various available identifiers.
+    if (Svc.Prefs.get("sendVersionInfo", true)) {
+      channel.setRequestHeader("user-agent", Utils.userAgent, false);
+    }
+
+    let headers = this.headers;
+
+    if (this.authenticator) {
+      let result = this.authenticator(this, method);
+      if (result && result.headers) {
+        for (let [k, v] of Object.entries(result.headers)) {
+          headers[k.toLowerCase()] = v;
+        }
+      }
+    } else {
+      this._log.debug("No authenticator found.");
+    }
+
+    for (let [key, value] of Object.entries(headers)) {
+      if (key == 'authorization')
+        this._log.trace("HTTP Header " + key + ": ***** (suppressed)");
+      else
+        this._log.trace("HTTP Header " + key + ": " + headers[key]);
+      channel.setRequestHeader(key, headers[key], false);
+    }
+    return channel;
+  },
+
+  _onProgress: function Res__onProgress(channel) {},
+
+  _doRequest: function _doRequest(action, data, callback) {
+    this._log.trace("In _doRequest.");
+    this._callback = callback;
+    let channel = this._createRequest(action);
+
+    if ("undefined" != typeof(data))
+      this._data = data;
+
+    // PUT and POST are treated differently because they have payload data.
+    if ("PUT" == action || "POST" == action) {
+      // Convert non-string bodies into JSON
+      if (this._data.constructor.toString() != String)
+        this._data = JSON.stringify(this._data);
+
+      this._log.debug(action + " Length: " + this._data.length);
+      this._log.trace(action + " Body: " + this._data);
+
+      let type = ('content-type' in this._headers) ?
+        this._headers['content-type'] : 'text/plain';
+
+      let stream = Cc["@mozilla.org/io/string-input-stream;1"].
+        createInstance(Ci.nsIStringInputStream);
+      stream.setData(this._data, this._data.length);
+
+      channel.QueryInterface(Ci.nsIUploadChannel);
+      channel.setUploadStream(stream, type, this._data.length);
+    }
+
+    // Setup a channel listener so that the actual network operation
+    // is performed asynchronously.
+    let listener = new ChannelListener(this._onComplete, this._onProgress,
+                                       this._log, this.ABORT_TIMEOUT);
+    channel.requestMethod = action;
+    try {
+      channel.asyncOpen2(listener);
+    } catch (ex) {
+      // asyncOpen2 can throw in a bunch of cases -- e.g., a forbidden port.
+      this._log.warn("Caught an error in asyncOpen2", ex);
+      CommonUtils.nextTick(callback.bind(this, ex));
+    }
+  },
+
+  _onComplete: function _onComplete(error, data, channel) {
+    this._log.trace("In _onComplete. Error is " + error + ".");
+
+    if (error) {
+      this._callback(error);
+      return;
+    }
+
+    this._data = data;
+    let action = channel.requestMethod;
+
+    this._log.trace("Channel: " + channel);
+    this._log.trace("Action: "  + action);
+
+    // Process status and success first. This way a problem with headers
+    // doesn't fail to include accurate status information.
+    let status = 0;
+    let success = false;
+
+    try {
+      status  = channel.responseStatus;
+      success = channel.requestSucceeded;    // HTTP status.
+
+      this._log.trace("Status: " + status);
+      this._log.trace("Success: " + success);
+
+      // Log the status of the request.
+      let mesg = [action, success ? "success" : "fail", status,
+                  channel.URI.spec].join(" ");
+      this._log.debug("mesg: " + mesg);
+
+      if (mesg.length > 200)
+        mesg = mesg.substr(0, 200) + "…";
+      this._log.debug(mesg);
+
+      // Additionally give the full response body when Trace logging.
+      if (this._log.level <= Log.Level.Trace)
+        this._log.trace(action + " body: " + data);
+
+    } catch(ex) {
+      // Got a response, but an exception occurred during processing.
+      // This shouldn't occur.
+      this._log.warn("Caught unexpected exception in _oncomplete", ex);
+    }
+
+    // Process headers. They can be empty, or the call can otherwise fail, so
+    // put this in its own try block.
+    let headers = {};
+    try {
+      this._log.trace("Processing response headers.");
+
+      // Read out the response headers if available.
+      channel.visitResponseHeaders({
+        visitHeader: function visitHeader(header, value) {
+          headers[header.toLowerCase()] = value;
+        }
+      });
+
+      // This is a server-side safety valve to allow slowing down
+      // clients without hurting performance.
+      if (headers["x-weave-backoff"]) {
+        let backoff = headers["x-weave-backoff"];
+        this._log.debug("Got X-Weave-Backoff: " + backoff);
+        Observers.notify("weave:service:backoff:interval",
+                         parseInt(backoff, 10));
+      }
+
+      if (success && headers["x-weave-quota-remaining"]) {
+        Observers.notify("weave:service:quota:remaining",
+                         parseInt(headers["x-weave-quota-remaining"], 10));
+      }
+
+      let contentLength = headers["content-length"];
+      if (success && contentLength && data &&
+          contentLength != data.length) {
+        this._log.warn("The response body's length of: " + data.length +
+                       " doesn't match the header's content-length of: " +
+                       contentLength + ".");
+      }
+    } catch (ex) {
+      this._log.debug("Caught exception visiting headers in _onComplete", ex);
+    }
+
+    let ret     = new String(data);
+    ret.url     = channel.URI.spec;
+    ret.status  = status;
+    ret.success = success;
+    ret.headers = headers;
+
+    if (!success) {
+      this._log.warn(`${action} request to ${ret.url} failed with status ${status}`);
+    }
+    // Make a lazy getter to convert the json response into an object.
+    // Note that this can cause a parse error to be thrown far away from the
+    // actual fetch, so be warned!
+    XPCOMUtils.defineLazyGetter(ret, "obj", function() {
+      try {
+        return JSON.parse(ret);
+      } catch (ex) {
+        this._log.warn("Got exception parsing response body", ex);
+        // Stringify to avoid possibly printing non-printable characters.
+        this._log.debug("Parse fail: Response body starts: \"" +
+                        JSON.stringify((ret + "").slice(0, 100)) +
+                        "\".");
+        throw ex;
+      }
+    }.bind(this));
+
+    this._callback(null, ret);
+  },
+
+  get: function get(callback) {
+    this._doRequest("GET", undefined, callback);
+  },
+
+  put: function put(data, callback) {
+    if (typeof data == "function")
+      [data, callback] = [undefined, data];
+    this._doRequest("PUT", data, callback);
+  },
+
+  post: function post(data, callback) {
+    if (typeof data == "function")
+      [data, callback] = [undefined, data];
+    this._doRequest("POST", data, callback);
+  },
+
+  delete: function delete_(callback) {
+    this._doRequest("DELETE", undefined, callback);
+  }
+};
+
+
+/*
+ * Represent a remote network resource, identified by a URI, with a
+ * synchronous API.
+ *
+ * 'Resource' is not recommended for new code. Use the asynchronous API of
+ * 'AsyncResource' instead.
+ */
+this.Resource = function Resource(uri) {
+  AsyncResource.call(this, uri);
+}
+Resource.prototype = {
+
+  __proto__: AsyncResource.prototype,
+
+  _logName: "Sync.Resource",
+
+  // ** {{{ Resource._request }}} **
+  //
+  // Perform a particular HTTP request on the resource. This method
+  // is never called directly, but is used by the high-level
+  // {{{get}}}, {{{put}}}, {{{post}}} and {{delete}} methods.
+  _request: function Res__request(action, data) {
+    let cb = Async.makeSyncCallback();
+    function callback(error, ret) {
+      if (error)
+        cb.throw(error);
+      else
+        cb(ret);
+    }
+
+    // The channel listener might get a failure code
+    try {
+      this._doRequest(action, data, callback);
+      return Async.waitForSyncCallback(cb);
+    } catch (ex) {
+      if (Async.isShutdownException(ex)) {
+        throw ex;
+      }
+      this._log.warn("${action} request to ${url} failed: ${ex}",
+                     { action, url: this.uri.spec, ex });
+      // Combine the channel stack with this request stack.  Need to create
+      // a new error object for that.
+      let error = Error(ex.message);
+      error.result = ex.result;
+      let chanStack = [];
+      if (ex.stack)
+        chanStack = ex.stack.trim().split(/\n/).slice(1);
+      let requestStack = error.stack.split(/\n/).slice(1);
+
+      // Strip out the args for the last 2 frames because they're usually HUGE!
+      for (let i = 0; i <= 1; i++)
+        requestStack[i] = requestStack[i].replace(/\(".*"\)@/, "(...)@");
+
+      error.stack = chanStack.concat(requestStack).join("\n");
+      throw error;
+    }
+  },
+
+  // ** {{{ Resource.get }}} **
+  //
+  // Perform an asynchronous HTTP GET for this resource.
+  get: function Res_get() {
+    return this._request("GET");
+  },
+
+  // ** {{{ Resource.put }}} **
+  //
+  // Perform a HTTP PUT for this resource.
+  put: function Res_put(data) {
+    return this._request("PUT", data);
+  },
+
+  // ** {{{ Resource.post }}} **
+  //
+  // Perform a HTTP POST for this resource.
+  post: function Res_post(data) {
+    return this._request("POST", data);
+  },
+
+  // ** {{{ Resource.delete }}} **
+  //
+  // Perform a HTTP DELETE for this resource.
+  delete: function Res_delete() {
+    return this._request("DELETE");
+  }
+};
+
+// = ChannelListener =
+//
+// This object implements the {{{nsIStreamListener}}} interface
+// and is called as the network operation proceeds.
+function ChannelListener(onComplete, onProgress, logger, timeout) {
+  this._onComplete = onComplete;
+  this._onProgress = onProgress;
+  this._log = logger;
+  this._timeout = timeout;
+  this.delayAbort();
+}
+ChannelListener.prototype = {
+
+  onStartRequest: function Channel_onStartRequest(channel) {
+    this._log.trace("onStartRequest called for channel " + channel + ".");
+
+    try {
+      channel.QueryInterface(Ci.nsIHttpChannel);
+    } catch (ex) {
+      this._log.error("Unexpected error: channel is not a nsIHttpChannel!");
+      channel.cancel(Cr.NS_BINDING_ABORTED);
+      return;
+    }
+
+    // Save the latest server timestamp when possible.
+    try {
+      AsyncResource.serverTime = channel.getResponseHeader("X-Weave-Timestamp") - 0;
+    }
+    catch(ex) {}
+
+    this._log.trace("onStartRequest: " + channel.requestMethod + " " +
+                    channel.URI.spec);
+    this._data = '';
+    this.delayAbort();
+  },
+
+  onStopRequest: function Channel_onStopRequest(channel, context, status) {
+    // Clear the abort timer now that the channel is done.
+    this.abortTimer.clear();
+
+    if (!this._onComplete) {
+      this._log.error("Unexpected error: _onComplete not defined in onStopRequest.");
+      this._onProgress = null;
+      return;
+    }
+
+    try {
+      channel.QueryInterface(Ci.nsIHttpChannel);
+    } catch (ex) {
+      this._log.error("Unexpected error: channel is not a nsIHttpChannel!");
+
+      this._onComplete(ex, this._data, channel);
+      this._onComplete = this._onProgress = null;
+      return;
+    }
+
+    let statusSuccess = Components.isSuccessCode(status);
+    let uri = channel && channel.URI && channel.URI.spec || "<unknown>";
+    this._log.trace("Channel for " + channel.requestMethod + " " + uri + ": " +
+                    "isSuccessCode(" + status + ")? " + statusSuccess);
+
+    if (this._data == '') {
+      this._data = null;
+    }
+
+    // Pass back the failure code and stop execution. Use Components.Exception()
+    // instead of Error() so the exception is QI-able and can be passed across
+    // XPCOM borders while preserving the status code.
+    if (!statusSuccess) {
+      let message = Components.Exception("", status).name;
+      let error   = Components.Exception(message, status);
+
+      this._onComplete(error, undefined, channel);
+      this._onComplete = this._onProgress = null;
+      return;
+    }
+
+    this._log.trace("Channel: flags = " + channel.loadFlags +
+                    ", URI = " + uri +
+                    ", HTTP success? " + channel.requestSucceeded);
+    this._onComplete(null, this._data, channel);
+    this._onComplete = this._onProgress = null;
+  },
+
+  onDataAvailable: function Channel_onDataAvail(req, cb, stream, off, count) {
+    let siStream;
+    try {
+      siStream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance(Ci.nsIScriptableInputStream);
+      siStream.init(stream);
+    } catch (ex) {
+      this._log.warn("Exception creating nsIScriptableInputStream", ex);
+      this._log.debug("Parameters: " + req.URI.spec + ", " + stream + ", " + off + ", " + count);
+      // Cannot proceed, so rethrow and allow the channel to cancel itself.
+      throw ex;
+    }
+
+    try {
+      this._data += siStream.read(count);
+    } catch (ex) {
+      this._log.warn("Exception thrown reading " + count + " bytes from " + siStream + ".");
+      throw ex;
+    }
+
+    try {
+      this._onProgress();
+    } catch (ex) {
+      if (Async.isShutdownException(ex)) {
+        throw ex;
+      }
+      this._log.warn("Got exception calling onProgress handler during fetch of "
+                     + req.URI.spec, ex);
+      this._log.trace("Rethrowing; expect a failure code from the HTTP channel.");
+      throw ex;
+    }
+
+    this.delayAbort();
+  },
+
+  /**
+   * Create or push back the abort timer that kills this request.
+   */
+  delayAbort: function delayAbort() {
+    try {
+      CommonUtils.namedTimer(this.abortRequest, this._timeout, this, "abortTimer");
+    } catch (ex) {
+      this._log.warn("Got exception extending abort timer", ex);
+    }
+  },
+
+  abortRequest: function abortRequest() {
+    // Ignore any callbacks if we happen to get any now
+    this.onStopRequest = function() {};
+    let error = Components.Exception("Aborting due to channel inactivity.",
+                                     Cr.NS_ERROR_NET_TIMEOUT);
+    if (!this._onComplete) {
+      this._log.error("Unexpected error: _onComplete not defined in " +
+                      "abortRequest.");
+      return;
+    }
+    this._onComplete(error);
+  }
+};
+
+/**
+ * This class handles channel notification events.
+ *
+ * An instance of this class is bound to each created channel.
+ *
+ * Optionally pass an array of header names. Each header named
+ * in this array will be copied between the channels in the
+ * event of a redirect.
+ */
+function ChannelNotificationListener(headersToCopy) {
+  this._headersToCopy = headersToCopy;
+
+  this._log = Log.repository.getLogger(this._logName);
+  this._log.level = Log.Level[Svc.Prefs.get("log.logger.network.resources")];
+}
+ChannelNotificationListener.prototype = {
+  _logName: "Sync.Resource",
+
+  getInterface: function(aIID) {
+    return this.QueryInterface(aIID);
+  },
+
+  QueryInterface: function(aIID) {
+    if (aIID.equals(Ci.nsIBadCertListener2) ||
+        aIID.equals(Ci.nsIInterfaceRequestor) ||
+        aIID.equals(Ci.nsISupports) ||
+        aIID.equals(Ci.nsIChannelEventSink))
+      return this;
+
+    throw Cr.NS_ERROR_NO_INTERFACE;
+  },
+
+  notifyCertProblem: function certProblem(socketInfo, sslStatus, targetHost) {
+    let log = Log.repository.getLogger("Sync.CertListener");
+    log.warn("Invalid HTTPS certificate encountered!");
+
+    // This suppresses the UI warning only. The request is still cancelled.
+    return true;
+  },
+
+  asyncOnChannelRedirect:
+    function asyncOnChannelRedirect(oldChannel, newChannel, flags, callback) {
+
+    let oldSpec = (oldChannel && oldChannel.URI) ? oldChannel.URI.spec : "<undefined>";
+    let newSpec = (newChannel && newChannel.URI) ? newChannel.URI.spec : "<undefined>";
+    this._log.debug("Channel redirect: " + oldSpec + ", " + newSpec + ", " + flags);
+
+    this._log.debug("Ensuring load flags are set.");
+    newChannel.loadFlags |= DEFAULT_LOAD_FLAGS;
+
+    // For internal redirects, copy the headers that our caller set.
+    try {
+      if ((flags & Ci.nsIChannelEventSink.REDIRECT_INTERNAL) &&
+          newChannel.URI.equals(oldChannel.URI)) {
+        this._log.debug("Copying headers for safe internal redirect.");
+
+        // QI the channel so we can set headers on it.
+        try {
+          newChannel.QueryInterface(Ci.nsIHttpChannel);
+        } catch (ex) {
+          this._log.error("Unexpected error: channel is not a nsIHttpChannel!");
+          throw ex;
+        }
+
+        for (let header of this._headersToCopy) {
+          let value = oldChannel.getRequestHeader(header);
+          if (value) {
+            let printed = (header == "authorization") ? "****" : value;
+            this._log.debug("Header: " + header + " = " + printed);
+            newChannel.setRequestHeader(header, value, false);
+          } else {
+            this._log.warn("No value for header " + header);
+          }
+        }
+      }
+    } catch (ex) {
+      this._log.error("Error copying headers", ex);
+    }
+
+    // We let all redirects proceed.
+    try {
+      callback.onRedirectVerifyCallback(Cr.NS_OK);
+    } catch (ex) {
+      this._log.error("onRedirectVerifyCallback threw!", ex);
+    }
+  }
+};

services/sync/modules/rest.js

@@ -0,0 +1,90 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+
+this.EXPORTED_SYMBOLS = ["SyncStorageRequest"];
+
+const STORAGE_REQUEST_TIMEOUT = 5 * 60; // 5 minutes
+
+/**
+ * RESTRequest variant for use against a Sync storage server.
+ */
+this.SyncStorageRequest = function SyncStorageRequest(uri) {
+  RESTRequest.call(this, uri);
+
+  this.authenticator = null;
+}
+SyncStorageRequest.prototype = {
+
+  __proto__: RESTRequest.prototype,
+
+  _logName: "Sync.StorageRequest",
+
+  /**
+   * Wait 5 minutes before killing a request.
+   */
+  timeout: STORAGE_REQUEST_TIMEOUT,
+
+  dispatch: function dispatch(method, data, onComplete, onProgress) {
+    // Compose a UA string fragment from the various available identifiers.
+    if (Svc.Prefs.get("sendVersionInfo", true)) {
+      this.setHeader("user-agent", Utils.userAgent);
+    }
+
+    if (this.authenticator) {
+      this.authenticator(this);
+    } else {
+      this._log.debug("No authenticator found.");
+    }
+
+    return RESTRequest.prototype.dispatch.apply(this, arguments);
+  },
+
+  onStartRequest: function onStartRequest(channel) {
+    RESTRequest.prototype.onStartRequest.call(this, channel);
+    if (this.status == this.ABORTED) {
+      return;
+    }
+
+    let headers = this.response.headers;
+    // Save the latest server timestamp when possible.
+    if (headers["x-weave-timestamp"]) {
+      SyncStorageRequest.serverTime = parseFloat(headers["x-weave-timestamp"]);
+    }
+
+    // This is a server-side safety valve to allow slowing down
+    // clients without hurting performance.
+    if (headers["x-weave-backoff"]) {
+      Svc.Obs.notify("weave:service:backoff:interval",
+                     parseInt(headers["x-weave-backoff"], 10));
+    }
+
+    if (this.response.success && headers["x-weave-quota-remaining"]) {
+      Svc.Obs.notify("weave:service:quota:remaining",
+                     parseInt(headers["x-weave-quota-remaining"], 10));
+    }
+  },
+
+  onStopRequest: function onStopRequest(channel, context, statusCode) {
+    if (this.status != this.ABORTED) {
+      let resp = this.response;
+      let contentLength = resp.headers ? resp.headers["content-length"] : "";
+
+      if (resp.success && contentLength &&
+          contentLength != resp.body.length) {
+        this._log.warn("The response body's length of: " + resp.body.length +
+                       " doesn't match the header's content-length of: " +
+                       contentLength + ".");
+      }
+    }
+
+    RESTRequest.prototype.onStopRequest.apply(this, arguments);
+  }
+};

services/sync/modules/stages/cluster.js

@@ -0,0 +1,113 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["ClusterManager"];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/policies.js");
+Cu.import("resource://services-sync/util.js");
+
+/**
+ * Contains code for managing the Sync cluster we are in.
+ */
+this.ClusterManager = function ClusterManager(service) {
+  this._log = Log.repository.getLogger("Sync.Service");
+  this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")];
+
+  this.service = service;
+}
+ClusterManager.prototype = {
+  get identity() {
+    return this.service.identity;
+  },
+
+  /**
+   * Obtain the cluster for the current user.
+   *
+   * Returns the string URL of the cluster or null on error.
+   */
+  _findCluster: function _findCluster() {
+    this._log.debug("Finding cluster for user " + this.identity.username);
+
+    // This should ideally use UserAPI10Client but the legacy hackiness is
+    // strong with this code.
+    let fail;
+    let url = this.service.userAPIURI + this.identity.username + "/node/weave";
+    let res = this.service.resource(url);
+    try {
+      let node = res.get();
+      switch (node.status) {
+        case 400:
+          this.service.status.login = LOGIN_FAILED_LOGIN_REJECTED;
+          fail = "Find cluster denied: " + this.service.errorHandler.errorStr(node);
+          break;
+        case 404:
+          this._log.debug("Using serverURL as data cluster (multi-cluster support disabled)");
+          return this.service.serverURL;
+        case 0:
+        case 200:
+          if (node == "null") {
+            node = null;
+          }
+          this._log.trace("_findCluster successfully returning " + node);
+          return node;
+        default:
+          this.service.errorHandler.checkServerError(node);
+          fail = "Unexpected response code: " + node.status;
+          break;
+      }
+    } catch (e) {
+      this._log.debug("Network error on findCluster");
+      this.service.status.login = LOGIN_FAILED_NETWORK_ERROR;
+      this.service.errorHandler.checkServerError(e);
+      fail = e;
+    }
+    throw fail;
+  },
+
+  /**
+   * Determine the cluster for the current user and update state.
+   */
+  setCluster: function setCluster() {
+    // Make sure we didn't get some unexpected response for the cluster.
+    let cluster = this._findCluster();
+    this._log.debug("Cluster value = " + cluster);
+    if (cluster == null) {
+      return false;
+    }
+
+    // Convert from the funky "String object with additional properties" that
+    // resource.js returns to a plain-old string.
+    cluster = cluster.toString();
+    // Don't update stuff if we already have the right cluster
+    if (cluster == this.service.clusterURL) {
+      return false;
+    }
+
+    this._log.debug("Setting cluster to " + cluster);
+    this.service.clusterURL = cluster;
+
+    return true;
+  },
+
+  getUserBaseURL: function getUserBaseURL() {
+    // Legacy Sync and FxA Sync construct the userBaseURL differently. Legacy
+    // Sync appends path components onto an empty path, and in FxA Sync, the
+    // token server constructs this for us in an opaque manner. Since the
+    // cluster manager already sets the clusterURL on Service and also has
+    // access to the current identity, we added this functionality here.
+
+    // If the clusterURL hasn't been set, the userBaseURL shouldn't be set
+    // either. Some tests expect "undefined" to be returned here.
+    if (!this.service.clusterURL) {
+      return undefined;
+    }
+    let storageAPI = this.service.clusterURL + SYNC_API_VERSION + "/";
+    return storageAPI + this.identity.username + "/";
+  }
+};
+Object.freeze(ClusterManager.prototype);

services/sync/modules/stages/declined.js

@@ -0,0 +1,76 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains code for maintaining the set of declined engines,
+ * in conjunction with EngineManager.
+ */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["DeclinedEngines"];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+
+
+this.DeclinedEngines = function (service) {
+  this._log = Log.repository.getLogger("Sync.Declined");
+  this._log.level = Log.Level[new Preferences(PREFS_BRANCH).get("log.logger.declined")];
+
+  this.service = service;
+}
+this.DeclinedEngines.prototype = {
+  updateDeclined: function (meta, engineManager=this.service.engineManager) {
+    let enabled = new Set(engineManager.getEnabled().map(e => e.name));
+    let known = new Set(engineManager.getAll().map(e => e.name));
+    let remoteDeclined = new Set(meta.payload.declined || []);
+    let localDeclined = new Set(engineManager.getDeclined());
+
+    this._log.debug("Handling remote declined: " + JSON.stringify([...remoteDeclined]));
+    this._log.debug("Handling local declined: " + JSON.stringify([...localDeclined]));
+
+    // Any engines that are locally enabled should be removed from the remote
+    // declined list.
+    //
+    // Any engines that are locally declined should be added to the remote
+    // declined list.
+    let newDeclined = CommonUtils.union(localDeclined, CommonUtils.difference(remoteDeclined, enabled));
+
+    // If our declined set has changed, put it into the meta object and mark
+    // it as changed.
+    let declinedChanged = !CommonUtils.setEqual(newDeclined, remoteDeclined);
+    this._log.debug("Declined changed? " + declinedChanged);
+    if (declinedChanged) {
+      meta.changed = true;
+      meta.payload.declined = [...newDeclined];
+    }
+
+    // Update the engine manager regardless.
+    engineManager.setDeclined(newDeclined);
+
+    // Any engines that are locally known, locally disabled, and not remotely
+    // or locally declined, are candidates for enablement.
+    let undecided = CommonUtils.difference(CommonUtils.difference(known, enabled), newDeclined);
+    if (undecided.size) {
+      let subject = {
+        declined: newDeclined,
+        enabled: enabled,
+        known: known,
+        undecided: undecided,
+      };
+      CommonUtils.nextTick(() => {
+        Observers.notify("weave:engines:notdeclined", subject);
+      });
+    }
+
+    return declinedChanged;
+  },
+};

services/sync/modules/stages/enginesync.js

@@ -0,0 +1,449 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains code for synchronizing engines.
+ */
+
+this.EXPORTED_SYMBOLS = ["EngineSynchronizer"];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/policies.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://gre/modules/Task.jsm");
+
+/**
+ * Perform synchronization of engines.
+ *
+ * This was originally split out of service.js. The API needs lots of love.
+ */
+this.EngineSynchronizer = function EngineSynchronizer(service) {
+  this._log = Log.repository.getLogger("Sync.Synchronizer");
+  this._log.level = Log.Level[Svc.Prefs.get("log.logger.synchronizer")];
+
+  this.service = service;
+
+  this.onComplete = null;
+}
+
+EngineSynchronizer.prototype = {
+  sync: function sync(engineNamesToSync) {
+    if (!this.onComplete) {
+      throw new Error("onComplete handler not installed.");
+    }
+
+    let startTime = Date.now();
+
+    this.service.status.resetSync();
+
+    // Make sure we should sync or record why we shouldn't.
+    let reason = this.service._checkSync();
+    if (reason) {
+      if (reason == kSyncNetworkOffline) {
+        this.service.status.sync = LOGIN_FAILED_NETWORK_ERROR;
+      }
+
+      // this is a purposeful abort rather than a failure, so don't set
+      // any status bits
+      reason = "Can't sync: " + reason;
+      this.onComplete(new Error("Can't sync: " + reason));
+      return;
+    }
+
+    // If we don't have a node, get one. If that fails, retry in 10 minutes.
+    if (!this.service.clusterURL && !this.service._clusterManager.setCluster()) {
+      this.service.status.sync = NO_SYNC_NODE_FOUND;
+      this._log.info("No cluster URL found. Cannot sync.");
+      this.onComplete(null);
+      return;
+    }
+
+    // Ping the server with a special info request once a day.
+    let infoURL = this.service.infoURL;
+    let now = Math.floor(Date.now() / 1000);
+    let lastPing = Svc.Prefs.get("lastPing", 0);
+    if (now - lastPing > 86400) { // 60 * 60 * 24
+      infoURL += "?v=" + WEAVE_VERSION;
+      Svc.Prefs.set("lastPing", now);
+    }
+
+    let engineManager = this.service.engineManager;
+
+    // Figure out what the last modified time is for each collection
+    let info = this.service._fetchInfo(infoURL);
+
+    // Convert the response to an object and read out the modified times
+    for (let engine of [this.service.clientsEngine].concat(engineManager.getAll())) {
+      engine.lastModified = info.obj[engine.name] || 0;
+    }
+
+    if (!(this.service._remoteSetup(info))) {
+      this.onComplete(new Error("Aborting sync, remote setup failed"));
+      return;
+    }
+
+    // Make sure we have an up-to-date list of clients before sending commands
+    this._log.debug("Refreshing client list.");
+    if (!this._syncEngine(this.service.clientsEngine)) {
+      // Clients is an engine like any other; it can fail with a 401,
+      // and we can elect to abort the sync.
+      this._log.warn("Client engine sync failed. Aborting.");
+      this.onComplete(null);
+      return;
+    }
+
+    // We only honor the "hint" of what engines to Sync if this isn't
+    // a first sync.
+    let allowEnginesHint = false;
+    // Wipe data in the desired direction if necessary
+    switch (Svc.Prefs.get("firstSync")) {
+      case "resetClient":
+        this.service.resetClient(engineManager.enabledEngineNames);
+        break;
+      case "wipeClient":
+        this.service.wipeClient(engineManager.enabledEngineNames);
+        break;
+      case "wipeRemote":
+        this.service.wipeRemote(engineManager.enabledEngineNames);
+        break;
+      default:
+        allowEnginesHint = true;
+        break;
+    }
+
+    if (this.service.clientsEngine.localCommands) {
+      try {
+        if (!(this.service.clientsEngine.processIncomingCommands())) {
+          this.service.status.sync = ABORT_SYNC_COMMAND;
+          this.onComplete(new Error("Processed command aborted sync."));
+          return;
+        }
+
+        // Repeat remoteSetup in-case the commands forced us to reset
+        if (!(this.service._remoteSetup(info))) {
+          this.onComplete(new Error("Remote setup failed after processing commands."));
+          return;
+        }
+      }
+      finally {
+        // Always immediately attempt to push back the local client (now
+        // without commands).
+        // Note that we don't abort here; if there's a 401 because we've
+        // been reassigned, we'll handle it around another engine.
+        this._syncEngine(this.service.clientsEngine);
+      }
+    }
+
+    // Update engines because it might change what we sync.
+    try {
+      this._updateEnabledEngines();
+    } catch (ex) {
+      this._log.debug("Updating enabled engines failed", ex);
+      this.service.errorHandler.checkServerError(ex);
+      this.onComplete(ex);
+      return;
+    }
+
+    // If the engines to sync has been specified, we sync in the order specified.
+    let enginesToSync;
+    if (allowEnginesHint && engineNamesToSync) {
+      this._log.info("Syncing specified engines", engineNamesToSync);
+      enginesToSync = engineManager.get(engineNamesToSync).filter(e => e.enabled);
+    } else {
+      this._log.info("Syncing all enabled engines.");
+      enginesToSync = engineManager.getEnabled();
+    }
+    try {
+      // We don't bother validating engines that failed to sync.
+      let enginesToValidate = [];
+      for (let engine of enginesToSync) {
+        // If there's any problems with syncing the engine, report the failure
+        if (!(this._syncEngine(engine)) || this.service.status.enforceBackoff) {
+          this._log.info("Aborting sync for failure in " + engine.name);
+          break;
+        }
+        enginesToValidate.push(engine);
+      }
+
+      // If _syncEngine fails for a 401, we might not have a cluster URL here.
+      // If that's the case, break out of this immediately, rather than
+      // throwing an exception when trying to fetch metaURL.
+      if (!this.service.clusterURL) {
+        this._log.debug("Aborting sync, no cluster URL: " +
+                        "not uploading new meta/global.");
+        this.onComplete(null);
+        return;
+      }
+
+      // Upload meta/global if any engines changed anything.
+      let meta = this.service.recordManager.get(this.service.metaURL);
+      if (meta.isNew || meta.changed) {
+        this._log.info("meta/global changed locally: reuploading.");
+        try {
+          this.service.uploadMetaGlobal(meta);
+          delete meta.isNew;
+          delete meta.changed;
+        } catch (error) {
+          this._log.error("Unable to upload meta/global. Leaving marked as new.");
+        }
+      }
+
+      Async.promiseSpinningly(this._tryValidateEngines(enginesToValidate));
+
+      // If there were no sync engine failures
+      if (this.service.status.service != SYNC_FAILED_PARTIAL) {
+        Svc.Prefs.set("lastSync", new Date().toString());
+        this.service.status.sync = SYNC_SUCCEEDED;
+      }
+    } finally {
+      Svc.Prefs.reset("firstSync");
+
+      let syncTime = ((Date.now() - startTime) / 1000).toFixed(2);
+      let dateStr = Utils.formatTimestamp(new Date());
+      this._log.info("Sync completed at " + dateStr
+                     + " after " + syncTime + " secs.");
+    }
+
+    this.onComplete(null);
+  },
+
+  _tryValidateEngines: Task.async(function* (recentlySyncedEngines) {
+    if (!Services.telemetry.canRecordBase || !Svc.Prefs.get("validation.enabled", false)) {
+      this._log.info("Skipping validation: validation or telemetry reporting is disabled");
+      return;
+    }
+
+    let lastValidation = Svc.Prefs.get("validation.lastTime", 0);
+    let validationInterval = Svc.Prefs.get("validation.interval");
+    let nowSeconds = Math.floor(Date.now() / 1000);
+
+    if (nowSeconds - lastValidation < validationInterval) {
+      this._log.info("Skipping validation: too recent since last validation attempt");
+      return;
+    }
+    // Update the time now, even if we may return false still. We don't want to
+    // check the rest of these more frequently than once a day.
+    Svc.Prefs.set("validation.lastTime", nowSeconds);
+
+    // Validation only occurs a certain percentage of the time.
+    let validationProbability = Svc.Prefs.get("validation.percentageChance", 0) / 100.0;
+    if (validationProbability < Math.random()) {
+      this._log.info("Skipping validation: Probability threshold not met");
+      return;
+    }
+    let maxRecords = Svc.Prefs.get("validation.maxRecords");
+    if (!maxRecords) {
+      // Don't bother asking the server for the counts if we know validation
+      // won't happen anyway.
+      return;
+    }
+
+    // maxRecords of -1 means "any number", so we can skip asking the server.
+    // Used for tests.
+    let info;
+    if (maxRecords < 0) {
+      info = {};
+      for (let e of recentlySyncedEngines) {
+        info[e.name] = 1; // needs to be < maxRecords
+      }
+      maxRecords = 2;
+    } else {
+
+      let collectionCountsURL = this.service.userBaseURL + "info/collection_counts";
+      try {
+        let infoResp = this.service._fetchInfo(collectionCountsURL);
+        if (!infoResp.success) {
+          this._log.error("Can't run validation: request to info/collection_counts responded with "
+                          + resp.status);
+          return;
+        }
+        info = infoResp.obj; // might throw because obj is a getter which parses json.
+      } catch (e) {
+        // Not running validation is totally fine, so we just write an error log and return.
+        this._log.error("Can't run validation: Caught error when fetching counts", e);
+        return;
+      }
+    }
+
+    if (!info) {
+      return;
+    }
+
+    let engineLookup = new Map(recentlySyncedEngines.map(e => [e.name, e]));
+    let toRun = [];
+    for (let [engineName, recordCount] of Object.entries(info)) {
+      let engine = engineLookup.get(engineName);
+      if (recordCount > maxRecords || !engine) {
+        this._log.debug(`Skipping validation for ${engineName} because it's not an engine or ` +
+                        `the number of records (${recordCount}) is greater than the maximum allowed (${maxRecords}).`);
+        continue;
+      }
+      let validator = engine.getValidator();
+      if (!validator) {
+        continue;
+      }
+      // Put this in an array so that we know how many we're going to do, so we
+      // don't tell users we're going to run some validators when we aren't.
+      toRun.push({ engine, validator });
+    }
+
+    if (!toRun.length) {
+      return;
+    }
+    Services.console.logStringMessage(
+      "Sync is about to run a consistency check. This may be slow, and " +
+      "can be controlled using the pref \"services.sync.validation.enabled\".\n" +
+      "If you encounter any problems because of this, please file a bug.");
+    for (let { validator, engine } of toRun) {
+      try {
+        let result = yield validator.validate(engine);
+        Observers.notify("weave:engine:validate:finish", result, engine.name);
+      } catch (e) {
+        this._log.error(`Failed to run validation on ${engine.name}!`, e);
+        Observers.notify("weave:engine:validate:error", e, engine.name)
+        // Keep validating -- there's no reason to think that a failure for one
+        // validator would mean the others will fail.
+      }
+    }
+  }),
+
+  // Returns true if sync should proceed.
+  // false / no return value means sync should be aborted.
+  _syncEngine: function _syncEngine(engine) {
+    try {
+      engine.sync();
+    }
+    catch(e) {
+      if (e.status == 401) {
+        // Maybe a 401, cluster update perhaps needed?
+        // We rely on ErrorHandler observing the sync failure notification to
+        // schedule another sync and clear node assignment values.
+        // Here we simply want to muffle the exception and return an
+        // appropriate value.
+        return false;
+      }
+    }
+
+    return true;
+  },
+
+  _updateEnabledFromMeta: function (meta, numClients, engineManager=this.service.engineManager) {
+    this._log.info("Updating enabled engines: " +
+                    numClients + " clients.");
+
+    if (meta.isNew || !meta.payload.engines) {
+      this._log.debug("meta/global isn't new, or is missing engines. Not updating enabled state.");
+      return;
+    }
+
+    // If we're the only client, and no engines are marked as enabled,
+    // thumb our noses at the server data: it can't be right.
+    // Belt-and-suspenders approach to Bug 615926.
+    let hasEnabledEngines = false;
+    for (let e in meta.payload.engines) {
+      if (e != "clients") {
+        hasEnabledEngines = true;
+        break;
+      }
+    }
+
+    if ((numClients <= 1) && !hasEnabledEngines) {
+      this._log.info("One client and no enabled engines: not touching local engine status.");
+      return;
+    }
+
+    this.service._ignorePrefObserver = true;
+
+    let enabled = engineManager.enabledEngineNames;
+
+    let toDecline = new Set();
+    let toUndecline = new Set();
+
+    for (let engineName in meta.payload.engines) {
+      if (engineName == "clients") {
+        // Clients is special.
+        continue;
+      }
+      let index = enabled.indexOf(engineName);
+      if (index != -1) {
+        // The engine is enabled locally. Nothing to do.
+        enabled.splice(index, 1);
+        continue;
+      }
+      let engine = engineManager.get(engineName);
+      if (!engine) {
+        // The engine doesn't exist locally. Nothing to do.
+        continue;
+      }
+
+      let attemptedEnable = false;
+      // If the engine was enabled remotely, enable it locally.
+      if (!Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) {
+        this._log.trace("Engine " + engineName + " was enabled. Marking as non-declined.");
+        toUndecline.add(engineName);
+        this._log.trace(engineName + " engine was enabled remotely.");
+        engine.enabled = true;
+        // Note that setting engine.enabled to true might not have worked for
+        // the password engine if a master-password is enabled.  However, it's
+        // still OK that we added it to undeclined - the user *tried* to enable
+        // it remotely - so it still winds up as not being flagged as declined
+        // even though it's disabled remotely.
+        attemptedEnable = true;
+      }
+
+      // If either the engine was disabled locally or enabling the engine
+      // failed (see above re master-password) then wipe server data and
+      // disable it everywhere.
+      if (!engine.enabled) {
+        this._log.trace("Wiping data for " + engineName + " engine.");
+        engine.wipeServer();
+        delete meta.payload.engines[engineName];
+        meta.changed = true; // the new enabled state must propagate
+        // We also here mark the engine as declined, because the pref
+        // was explicitly changed to false - unless we tried, and failed,
+        // to enable it - in which case we leave the declined state alone.
+        if (!attemptedEnable) {
+          // This will be reflected in meta/global in the next stage.
+          this._log.trace("Engine " + engineName + " was disabled locally. Marking as declined.");
+          toDecline.add(engineName);
+        }
+      }
+    }
+
+    // Any remaining engines were either enabled locally or disabled remotely.
+    for (let engineName of enabled) {
+      let engine = engineManager.get(engineName);
+      if (Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) {
+        this._log.trace("The " + engineName + " engine was enabled locally.");
+        toUndecline.add(engineName);
+      } else {
+        this._log.trace("The " + engineName + " engine was disabled remotely.");
+
+        // Don't automatically mark it as declined!
+        engine.enabled = false;
+      }
+    }
+
+    engineManager.decline(toDecline);
+    engineManager.undecline(toUndecline);
+
+    Svc.Prefs.resetBranch("engineStatusChanged.");
+    this.service._ignorePrefObserver = false;
+  },
+
+  _updateEnabledEngines: function () {
+    let meta = this.service.recordManager.get(this.service.metaURL);
+    let numClients = this.service.scheduler.numClients;
+    let engineManager = this.service.engineManager;
+
+    this._updateEnabledFromMeta(meta, numClients, engineManager);
+  },
+};
+Object.freeze(EngineSynchronizer.prototype);

services/sync/modules/status.js

@@ -0,0 +1,145 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+* License, v. 2.0. If a copy of the MPL was not distributed with this
+* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["Status"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-sync/browserid_identity.js");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://services-common/async.js");
+
+this.Status = {
+  _log: Log.repository.getLogger("Sync.Status"),
+  __authManager: null,
+  ready: false,
+
+  get _authManager() {
+    if (this.__authManager) {
+      return this.__authManager;
+    }
+    let service = Components.classes["@mozilla.org/weave/service;1"]
+                    .getService(Components.interfaces.nsISupports)
+                    .wrappedJSObject;
+    let idClass = service.fxAccountsEnabled ? BrowserIDManager : IdentityManager;
+    this.__authManager = new idClass();
+    this.__authManager.initialize();
+    return this.__authManager;
+  },
+
+  get service() {
+    return this._service;
+  },
+
+  set service(code) {
+    this._log.debug("Status.service: " + (this._service || undefined) + " => " + code);
+    this._service = code;
+  },
+
+  get login() {
+    return this._login;
+  },
+
+  set login(code) {
+    this._log.debug("Status.login: " + this._login + " => " + code);
+    this._login = code;
+
+    if (code == LOGIN_FAILED_NO_USERNAME ||
+        code == LOGIN_FAILED_NO_PASSWORD ||
+        code == LOGIN_FAILED_NO_PASSPHRASE) {
+      this.service = CLIENT_NOT_CONFIGURED;
+    } else if (code != LOGIN_SUCCEEDED) {
+      this.service = LOGIN_FAILED;
+    } else {
+      this.service = STATUS_OK;
+    }
+  },
+
+  get sync() {
+    return this._sync;
+  },
+
+  set sync(code) {
+    this._log.debug("Status.sync: " + this._sync + " => " + code);
+    this._sync = code;
+    this.service = code == SYNC_SUCCEEDED ? STATUS_OK : SYNC_FAILED;
+  },
+
+  get eol() {
+    let modePref = PREFS_BRANCH + "errorhandler.alert.mode";
+    try {
+      return Services.prefs.getCharPref(modePref) == "hard-eol";
+    } catch (ex) {
+      return false;
+    }
+  },
+
+  get engines() {
+    return this._engines;
+  },
+
+  set engines([name, code]) {
+    this._log.debug("Status for engine " + name + ": " + code);
+    this._engines[name] = code;
+
+    if (code != ENGINE_SUCCEEDED) {
+      this.service = SYNC_FAILED_PARTIAL;
+    }
+  },
+
+  // Implement toString because adding a logger introduces a cyclic object
+  // value, so we can't trivially debug-print Status as JSON.
+  toString: function toString() {
+    return "<Status" +
+           ": login: "   + Status.login +
+           ", service: " + Status.service +
+           ", sync: "    + Status.sync + ">";
+  },
+
+  checkSetup: function checkSetup() {
+    let result = this._authManager.currentAuthState;
+    if (result == STATUS_OK) {
+      Status.service = result;
+      return result;
+    }
+
+    Status.login = result;
+    return Status.service;
+  },
+
+  resetBackoff: function resetBackoff() {
+    this.enforceBackoff = false;
+    this.backoffInterval = 0;
+    this.minimumNextSync = 0;
+  },
+
+  resetSync: function resetSync() {
+    // Logger setup.
+    let logPref = PREFS_BRANCH + "log.logger.status";
+    let logLevel = "Trace";
+    try {
+      logLevel = Services.prefs.getCharPref(logPref);
+    } catch (ex) {
+      // Use default.
+    }
+    this._log.level = Log.Level[logLevel];
+
+    this._log.info("Resetting Status.");
+    this.service = STATUS_OK;
+    this._login = LOGIN_SUCCEEDED;
+    this._sync = SYNC_SUCCEEDED;
+    this._engines = {};
+    this.partial = false;
+  }
+};
+
+// Initialize various status values.
+Status.resetBackoff();
+Status.resetSync();

services/sync/modules/telemetry.js

@@ -0,0 +1,578 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+this.EXPORTED_SYMBOLS = ["SyncTelemetry"];
+
+Cu.import("resource://services-sync/browserid_identity.js");
+Cu.import("resource://services-sync/main.js");
+Cu.import("resource://services-sync/status.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/TelemetryController.jsm");
+Cu.import("resource://gre/modules/FxAccounts.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/osfile.jsm", this);
+
+let constants = {};
+Cu.import("resource://services-sync/constants.js", constants);
+
+var fxAccountsCommon = {};
+Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
+
+XPCOMUtils.defineLazyServiceGetter(this, "Telemetry",
+                                   "@mozilla.org/base/telemetry;1",
+                                   "nsITelemetry");
+
+const log = Log.repository.getLogger("Sync.Telemetry");
+
+const TOPICS = [
+  "profile-before-change",
+  "weave:service:sync:start",
+  "weave:service:sync:finish",
+  "weave:service:sync:error",
+
+  "weave:engine:sync:start",
+  "weave:engine:sync:finish",
+  "weave:engine:sync:error",
+  "weave:engine:sync:applied",
+  "weave:engine:sync:uploaded",
+  "weave:engine:validate:finish",
+  "weave:engine:validate:error",
+];
+
+const PING_FORMAT_VERSION = 1;
+
+// The set of engines we record telemetry for - any other engines are ignored.
+const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
+                         "passwords", "prefs", "tabs", "extension-storage"]);
+
+// A regex we can use to replace the profile dir in error messages. We use a
+// regexp so we can simply replace all case-insensitive occurences.
+// This escaping function is from:
+// https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
+const reProfileDir = new RegExp(
+        OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
+        "gi");
+
+function transformError(error, engineName) {
+  if (Async.isShutdownException(error)) {
+    return { name: "shutdownerror" };
+  }
+
+  if (typeof error === "string") {
+    if (error.startsWith("error.")) {
+      // This is hacky, but I can't imagine that it's not also accurate.
+      return { name: "othererror", error };
+    }
+    // There's a chance the profiledir is in the error string which is PII we
+    // want to avoid including in the ping.
+    error = error.replace(reProfileDir, "[profileDir]");
+    return { name: "unexpectederror", error };
+  }
+
+  if (error.failureCode) {
+    return { name: "othererror", error: error.failureCode };
+  }
+
+  if (error instanceof AuthenticationError) {
+    return { name: "autherror", from: error.source };
+  }
+
+  if (error instanceof Ci.mozIStorageError) {
+    return { name: "sqlerror", code: error.result };
+  }
+
+  let httpCode = error.status ||
+    (error.response && error.response.status) ||
+    error.code;
+
+  if (httpCode) {
+    return { name: "httperror", code: httpCode };
+  }
+
+  if (error.result) {
+    return { name: "nserror", code: error.result };
+  }
+
+  return {
+    name: "unexpectederror",
+    // as above, remove the profile dir value.
+    error: String(error).replace(reProfileDir, "[profileDir]")
+  }
+}
+
+function tryGetMonotonicTimestamp() {
+  try {
+    return Telemetry.msSinceProcessStart();
+  } catch (e) {
+    log.warn("Unable to get a monotonic timestamp!");
+    return -1;
+  }
+}
+
+function timeDeltaFrom(monotonicStartTime) {
+  let now = tryGetMonotonicTimestamp();
+  if (monotonicStartTime !== -1 && now !== -1) {
+    return Math.round(now - monotonicStartTime);
+  }
+  return -1;
+}
+
+class EngineRecord {
+  constructor(name) {
+    // startTime is in ms from process start, but is monotonic (unlike Date.now())
+    // so we need to keep both it and when.
+    this.startTime = tryGetMonotonicTimestamp();
+    this.name = name;
+  }
+
+  toJSON() {
+    let result = Object.assign({}, this);
+    delete result.startTime;
+    return result;
+  }
+
+  finished(error) {
+    let took = timeDeltaFrom(this.startTime);
+    if (took > 0) {
+      this.took = took;
+    }
+    if (error) {
+      this.failureReason = transformError(error, this.name);
+    }
+  }
+
+  recordApplied(counts) {
+    if (this.incoming) {
+      log.error(`Incoming records applied multiple times for engine ${this.name}!`);
+      return;
+    }
+    if (this.name === "clients" && !counts.failed) {
+      // ignore successful application of client records
+      // since otherwise they show up every time and are meaningless.
+      return;
+    }
+
+    let incomingData = {};
+    let properties = ["applied", "failed", "newFailed", "reconciled"];
+    // Only record non-zero properties and only record incoming at all if
+    // there's at least one property we care about.
+    for (let property of properties) {
+      if (counts[property]) {
+        incomingData[property] = counts[property];
+        this.incoming = incomingData;
+      }
+    }
+  }
+
+  recordValidation(validationResult) {
+    if (this.validation) {
+      log.error(`Multiple validations occurred for engine ${this.name}!`);
+      return;
+    }
+    let { problems, version, duration, recordCount } = validationResult;
+    let validation = {
+      version: version || 0,
+      checked: recordCount || 0,
+    };
+    if (duration > 0) {
+      validation.took = Math.round(duration);
+    }
+    let summarized = problems.getSummary(true).filter(({count}) => count > 0);
+    if (summarized.length) {
+      validation.problems = summarized;
+    }
+    this.validation = validation;
+  }
+
+  recordValidationError(e) {
+    if (this.validation) {
+      log.error(`Multiple validations occurred for engine ${this.name}!`);
+      return;
+    }
+
+    this.validation = {
+      failureReason: transformError(e)
+    };
+  }
+
+  recordUploaded(counts) {
+    if (counts.sent || counts.failed) {
+      if (!this.outgoing) {
+        this.outgoing = [];
+      }
+      this.outgoing.push({
+        sent: counts.sent || undefined,
+        failed: counts.failed || undefined,
+      });
+    }
+  }
+}
+
+class TelemetryRecord {
+  constructor(allowedEngines) {
+    this.allowedEngines = allowedEngines;
+    // Our failure reason. This property only exists in the generated ping if an
+    // error actually occurred.
+    this.failureReason = undefined;
+    this.uid = "";
+    this.when = Date.now();
+    this.startTime = tryGetMonotonicTimestamp();
+    this.took = 0; // will be set later.
+
+    // All engines that have finished (ie, does not include the "current" one)
+    // We omit this from the ping if it's empty.
+    this.engines = [];
+    // The engine that has started but not yet stopped.
+    this.currentEngine = null;
+  }
+
+  toJSON() {
+    let result = {
+      when: this.when,
+      uid: this.uid,
+      took: this.took,
+      failureReason: this.failureReason,
+      status: this.status,
+      deviceID: this.deviceID,
+      devices: this.devices,
+    };
+    let engines = [];
+    for (let engine of this.engines) {
+      engines.push(engine.toJSON());
+    }
+    if (engines.length > 0) {
+      result.engines = engines;
+    }
+    return result;
+  }
+
+  finished(error) {
+    this.took = timeDeltaFrom(this.startTime);
+    if (this.currentEngine != null) {
+      log.error("Finished called for the sync before the current engine finished");
+      this.currentEngine.finished(null);
+      this.onEngineStop(this.currentEngine.name);
+    }
+    if (error) {
+      this.failureReason = transformError(error);
+    }
+
+    // We don't bother including the "devices" field if we can't come up with a
+    // UID or device ID for *this* device -- If that's the case, any data we'd
+    // put there would be likely to be full of garbage anyway.
+    let includeDeviceInfo = false;
+    try {
+      this.uid = Weave.Service.identity.hashedUID();
+      let deviceID = Weave.Service.identity.deviceID();
+      if (deviceID) {
+        // Combine the raw device id with the metrics uid to create a stable
+        // unique identifier that can't be mapped back to the user's FxA
+        // identity without knowing the metrics HMAC key.
+        this.deviceID = Utils.sha256(deviceID + this.uid);
+        includeDeviceInfo = true;
+      }
+    } catch (e) {
+      this.uid = "0".repeat(32);
+      this.deviceID = undefined;
+    }
+
+    if (includeDeviceInfo) {
+      let remoteDevices = Weave.Service.clientsEngine.remoteClients;
+      this.devices = remoteDevices.map(device => {
+        return {
+          os: device.os,
+          version: device.version,
+          id: Utils.sha256(device.id + this.uid)
+        };
+      });
+    }
+
+    // Check for engine statuses. -- We do this now, and not in engine.finished
+    // to make sure any statuses that get set "late" are recorded
+    for (let engine of this.engines) {
+      let status = Status.engines[engine.name];
+      if (status && status !== constants.ENGINE_SUCCEEDED) {
+        engine.status = status;
+      }
+    }
+
+    let statusObject = {};
+
+    let serviceStatus = Status.service;
+    if (serviceStatus && serviceStatus !== constants.STATUS_OK) {
+      statusObject.service = serviceStatus;
+      this.status = statusObject;
+    }
+    let syncStatus = Status.sync;
+    if (syncStatus && syncStatus !== constants.SYNC_SUCCEEDED) {
+      statusObject.sync = syncStatus;
+      this.status = statusObject;
+    }
+  }
+
+  onEngineStart(engineName) {
+    if (this._shouldIgnoreEngine(engineName, false)) {
+      return;
+    }
+
+    if (this.currentEngine) {
+      log.error(`Being told that engine ${engineName} has started, but current engine ${
+        this.currentEngine.name} hasn't stopped`);
+      // Just discard the current engine rather than making up data for it.
+    }
+    this.currentEngine = new EngineRecord(engineName);
+  }
+
+  onEngineStop(engineName, error) {
+    // We only care if it's the current engine if we have a current engine.
+    if (this._shouldIgnoreEngine(engineName, !!this.currentEngine)) {
+      return;
+    }
+    if (!this.currentEngine) {
+      // It's possible for us to get an error before the start message of an engine
+      // (somehow), in which case we still want to record that error.
+      if (!error) {
+        return;
+      }
+      log.error(`Error triggered on ${engineName} when no current engine exists: ${error}`);
+      this.currentEngine = new EngineRecord(engineName);
+    }
+    this.currentEngine.finished(error);
+    this.engines.push(this.currentEngine);
+    this.currentEngine = null;
+  }
+
+  onEngineApplied(engineName, counts) {
+    if (this._shouldIgnoreEngine(engineName)) {
+      return;
+    }
+    this.currentEngine.recordApplied(counts);
+  }
+
+  onEngineValidated(engineName, validationData) {
+    if (this._shouldIgnoreEngine(engineName, false)) {
+      return;
+    }
+    let engine = this.engines.find(e => e.name === engineName);
+    if (!engine && this.currentEngine && engineName === this.currentEngine.name) {
+      engine = this.currentEngine;
+    }
+    if (engine) {
+      engine.recordValidation(validationData);
+    } else {
+      log.warn(`Validation event triggered for engine ${engineName}, which hasn't been synced!`);
+    }
+  }
+
+  onEngineValidateError(engineName, error) {
+    if (this._shouldIgnoreEngine(engineName, false)) {
+      return;
+    }
+    let engine = this.engines.find(e => e.name === engineName);
+    if (!engine && this.currentEngine && engineName === this.currentEngine.name) {
+      engine = this.currentEngine;
+    }
+    if (engine) {
+      engine.recordValidationError(error);
+    } else {
+      log.warn(`Validation failure event triggered for engine ${engineName}, which hasn't been synced!`);
+    }
+  }
+
+  onEngineUploaded(engineName, counts) {
+    if (this._shouldIgnoreEngine(engineName)) {
+      return;
+    }
+    this.currentEngine.recordUploaded(counts);
+  }
+
+  _shouldIgnoreEngine(engineName, shouldBeCurrent = true) {
+    if (!this.allowedEngines.has(engineName)) {
+      log.info(`Notification for engine ${engineName}, but we aren't recording telemetry for it`);
+      return true;
+    }
+    if (shouldBeCurrent) {
+      if (!this.currentEngine || engineName != this.currentEngine.name) {
+        log.error(`Notification for engine ${engineName} but it isn't current`);
+        return true;
+      }
+    }
+    return false;
+  }
+}
+
+class SyncTelemetryImpl {
+  constructor(allowedEngines) {
+    log.level = Log.Level[Svc.Prefs.get("log.logger.telemetry", "Trace")];
+    // This is accessible so we can enable custom engines during tests.
+    this.allowedEngines = allowedEngines;
+    this.current = null;
+    this.setupObservers();
+
+    this.payloads = [];
+    this.discarded = 0;
+    this.maxPayloadCount = Svc.Prefs.get("telemetry.maxPayloadCount");
+    this.submissionInterval = Svc.Prefs.get("telemetry.submissionInterval") * 1000;
+    this.lastSubmissionTime = Telemetry.msSinceProcessStart();
+  }
+
+  getPingJSON(reason) {
+    return {
+      why: reason,
+      discarded: this.discarded || undefined,
+      version: PING_FORMAT_VERSION,
+      syncs: this.payloads.slice(),
+    };
+  }
+
+  finish(reason) {
+    // Note that we might be in the middle of a sync right now, and so we don't
+    // want to touch this.current.
+    let result = this.getPingJSON(reason);
+    this.payloads = [];
+    this.discarded = 0;
+    this.submit(result);
+  }
+
+  setupObservers() {
+    for (let topic of TOPICS) {
+      Observers.add(topic, this, this);
+    }
+  }
+
+  shutdown() {
+    this.finish("shutdown");
+    for (let topic of TOPICS) {
+      Observers.remove(topic, this, this);
+    }
+  }
+
+  submit(record) {
+    // We still call submit() with possibly illegal payloads so that tests can
+    // know that the ping was built. We don't end up submitting them, however.
+    if (record.syncs.length) {
+      log.trace(`submitting ${record.syncs.length} sync record(s) to telemetry`);
+      TelemetryController.submitExternalPing("sync", record);
+    }
+  }
+
+
+  onSyncStarted() {
+    if (this.current) {
+      log.warn("Observed weave:service:sync:start, but we're already recording a sync!");
+      // Just discard the old record, consistent with our handling of engines, above.
+      this.current = null;
+    }
+    this.current = new TelemetryRecord(this.allowedEngines);
+  }
+
+  _checkCurrent(topic) {
+    if (!this.current) {
+      log.warn(`Observed notification ${topic} but no current sync is being recorded.`);
+      return false;
+    }
+    return true;
+  }
+
+  onSyncFinished(error) {
+    if (!this.current) {
+      log.warn("onSyncFinished but we aren't recording");
+      return;
+    }
+    this.current.finished(error);
+    if (this.payloads.length < this.maxPayloadCount) {
+      this.payloads.push(this.current.toJSON());
+    } else {
+      ++this.discarded;
+    }
+    this.current = null;
+    if ((Telemetry.msSinceProcessStart() - this.lastSubmissionTime) > this.submissionInterval) {
+      this.finish("schedule");
+      this.lastSubmissionTime = Telemetry.msSinceProcessStart();
+    }
+  }
+
+  observe(subject, topic, data) {
+    log.trace(`observed ${topic} ${data}`);
+
+    switch (topic) {
+      case "profile-before-change":
+        this.shutdown();
+        break;
+
+      /* sync itself state changes */
+      case "weave:service:sync:start":
+        this.onSyncStarted();
+        break;
+
+      case "weave:service:sync:finish":
+        if (this._checkCurrent(topic)) {
+          this.onSyncFinished(null);
+        }
+        break;
+
+      case "weave:service:sync:error":
+        // argument needs to be truthy (this should always be the case)
+        this.onSyncFinished(subject || "Unknown");
+        break;
+
+      /* engine sync state changes */
+      case "weave:engine:sync:start":
+        if (this._checkCurrent(topic)) {
+          this.current.onEngineStart(data);
+        }
+        break;
+      case "weave:engine:sync:finish":
+        if (this._checkCurrent(topic)) {
+          this.current.onEngineStop(data, null);
+        }
+        break;
+
+      case "weave:engine:sync:error":
+        if (this._checkCurrent(topic)) {
+          // argument needs to be truthy (this should always be the case)
+          this.current.onEngineStop(data, subject || "Unknown");
+        }
+        break;
+
+      /* engine counts */
+      case "weave:engine:sync:applied":
+        if (this._checkCurrent(topic)) {
+          this.current.onEngineApplied(data, subject);
+        }
+        break;
+
+      case "weave:engine:sync:uploaded":
+        if (this._checkCurrent(topic)) {
+          this.current.onEngineUploaded(data, subject);
+        }
+        break;
+
+      case "weave:engine:validate:finish":
+        if (this._checkCurrent(topic)) {
+          this.current.onEngineValidated(data, subject);
+        }
+        break;
+
+      case "weave:engine:validate:error":
+        if (this._checkCurrent(topic)) {
+          this.current.onEngineValidateError(data, subject || "Unknown");
+        }
+        break;
+
+      default:
+        log.warn(`unexpected observer topic ${topic}`);
+        break;
+    }
+  }
+}
+
+this.SyncTelemetry = new SyncTelemetryImpl(ENGINES);

services/sync/modules/userapi.js

@@ -0,0 +1,224 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+  "UserAPI10Client",
+];
+
+var {utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/rest.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-sync/util.js");
+
+/**
+ * A generic client for the user API 1.0 service.
+ *
+ * http://docs.services.mozilla.com/reg/apis.html
+ *
+ * Instances are constructed with the base URI of the service.
+ */
+this.UserAPI10Client = function UserAPI10Client(baseURI) {
+  this._log = Log.repository.getLogger("Sync.UserAPI");
+  this._log.level = Log.Level[Svc.Prefs.get("log.logger.userapi")];
+
+  this.baseURI = baseURI;
+}
+UserAPI10Client.prototype = {
+  USER_CREATE_ERROR_CODES: {
+    2: "Incorrect or missing captcha.",
+    4: "User exists.",
+    6: "JSON parse failure.",
+    7: "Missing password field.",
+    9: "Requested password not strong enough.",
+    12: "No email address on file.",
+  },
+
+  /**
+   * Determine whether a specified username exists.
+   *
+   * Callback receives the following arguments:
+   *
+   *   (Error) Describes error that occurred or null if request was
+   *           successful.
+   *   (boolean) True if user exists. False if not. null if there was an error.
+   */
+  usernameExists: function usernameExists(username, cb) {
+    if (typeof(cb) != "function") {
+      throw new Error("cb must be a function.");
+    }
+
+    let url = this.baseURI + username;
+    let request = new RESTRequest(url);
+    request.get(this._onUsername.bind(this, cb, request));
+  },
+
+  /**
+   * Obtain the Weave (Sync) node for a specified user.
+   *
+   * The callback receives the following arguments:
+   *
+   *   (Error)  Describes error that occurred or null if request was successful.
+   *   (string) Username request is for.
+   *   (string) URL of user's node. If null and there is no error, no node could
+   *            be assigned at the time of the request.
+   */
+  getWeaveNode: function getWeaveNode(username, password, cb) {
+    if (typeof(cb) != "function") {
+      throw new Error("cb must be a function.");
+    }
+
+    let request = this._getRequest(username, "/node/weave", password);
+    request.get(this._onWeaveNode.bind(this, cb, request));
+  },
+
+  /**
+   * Change a password for the specified user.
+   *
+   * @param username
+   *        (string) The username whose password to change.
+   * @param oldPassword
+   *        (string) The old, current password.
+   * @param newPassword
+   *        (string) The new password to switch to.
+   */
+  changePassword: function changePassword(username, oldPassword, newPassword, cb) {
+    let request = this._getRequest(username, "/password", oldPassword);
+    request.onComplete = this._onChangePassword.bind(this, cb, request);
+    request.post(CommonUtils.encodeUTF8(newPassword));
+  },
+
+  createAccount: function createAccount(email, password, captchaChallenge,
+                                        captchaResponse, cb) {
+    let username = IdentityManager.prototype.usernameFromAccount(email);
+    let body = JSON.stringify({
+      "email":             email,
+      "password":          Utils.encodeUTF8(password),
+      "captcha-challenge": captchaChallenge,
+      "captcha-response":  captchaResponse
+    });
+
+    let url = this.baseURI + username;
+    let request = new RESTRequest(url);
+
+    if (this.adminSecret) {
+      request.setHeader("X-Weave-Secret", this.adminSecret);
+    }
+
+    request.onComplete = this._onCreateAccount.bind(this, cb, request);
+    request.put(body);
+  },
+
+  _getRequest: function _getRequest(username, path, password=null) {
+    let url = this.baseURI + username + path;
+    let request = new RESTRequest(url);
+
+    if (password) {
+      let up = username + ":" + password;
+      request.setHeader("authorization", "Basic " + btoa(up));
+    }
+
+    return request;
+  },
+
+  _onUsername: function _onUsername(cb, request, error) {
+    if (error) {
+      cb(error, null);
+      return;
+    }
+
+    let body = request.response.body;
+    if (body == "0") {
+      cb(null, false);
+      return;
+    } else if (body == "1") {
+      cb(null, true);
+      return;
+    } else {
+      cb(new Error("Unknown response from server: " + body), null);
+      return;
+    }
+  },
+
+  _onWeaveNode: function _onWeaveNode(cb, request, error) {
+    if (error) {
+      cb.network = true;
+      cb(error, null);
+      return;
+    }
+
+    let response = request.response;
+
+    if (response.status == 200) {
+      let body = response.body;
+      if (body == "null") {
+        cb(null, null);
+        return;
+      }
+
+      cb(null, body);
+      return;
+    }
+
+    error = new Error("Sync node retrieval failed.");
+    switch (response.status) {
+      case 400:
+        error.denied = true;
+        break;
+      case 404:
+        error.notFound = true;
+        break;
+      default:
+        error.message = "Unexpected response code: " + response.status;
+    }
+
+    cb(error, null);
+    return;
+  },
+
+  _onChangePassword: function _onChangePassword(cb, request, error) {
+    this._log.info("Password change response received: " +
+                   request.response.status);
+    if (error) {
+      cb(error);
+      return;
+    }
+
+    let response = request.response;
+    if (response.status != 200) {
+      cb(new Error("Password changed failed: " + response.body));
+      return;
+    }
+
+    cb(null);
+  },
+
+  _onCreateAccount: function _onCreateAccount(cb, request, error) {
+    let response = request.response;
+
+    this._log.info("Create account response: " + response.status + " " +
+                   response.body);
+
+    if (error) {
+      cb(new Error("HTTP transport error."), null);
+      return;
+    }
+
+    if (response.status == 200) {
+      cb(null, response.body);
+      return;
+    }
+
+    error = new Error("Could not create user.");
+    error.body = response.body;
+
+    cb(error, null);
+    return;
+  },
+};
+Object.freeze(UserAPI10Client.prototype);

services/sync/modules/util.js

@@ -0,0 +1,797 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["XPCOMUtils", "Services", "Utils", "Async", "Svc", "Str"];
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/observers.js");
+Cu.import("resource://services-common/stringbundle.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://services-common/async.js", this);
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://gre/modules/osfile.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+
+// FxAccountsCommon.js doesn't use a "namespace", so create one here.
+XPCOMUtils.defineLazyGetter(this, "FxAccountsCommon", function() {
+  let FxAccountsCommon = {};
+  Cu.import("resource://gre/modules/FxAccountsCommon.js", FxAccountsCommon);
+  return FxAccountsCommon;
+});
+
+/*
+ * Utility functions
+ */
+
+this.Utils = {
+  // Alias in functions from CommonUtils. These previously were defined here.
+  // In the ideal world, references to these would be removed.
+  nextTick: CommonUtils.nextTick,
+  namedTimer: CommonUtils.namedTimer,
+  makeURI: CommonUtils.makeURI,
+  encodeUTF8: CommonUtils.encodeUTF8,
+  decodeUTF8: CommonUtils.decodeUTF8,
+  safeAtoB: CommonUtils.safeAtoB,
+  byteArrayToString: CommonUtils.byteArrayToString,
+  bytesAsHex: CommonUtils.bytesAsHex,
+  hexToBytes: CommonUtils.hexToBytes,
+  encodeBase32: CommonUtils.encodeBase32,
+  decodeBase32: CommonUtils.decodeBase32,
+
+  // Aliases from CryptoUtils.
+  generateRandomBytes: CryptoUtils.generateRandomBytes,
+  computeHTTPMACSHA1: CryptoUtils.computeHTTPMACSHA1,
+  digestUTF8: CryptoUtils.digestUTF8,
+  digestBytes: CryptoUtils.digestBytes,
+  sha1: CryptoUtils.sha1,
+  sha1Base32: CryptoUtils.sha1Base32,
+  sha256: CryptoUtils.sha256,
+  makeHMACKey: CryptoUtils.makeHMACKey,
+  makeHMACHasher: CryptoUtils.makeHMACHasher,
+  hkdfExpand: CryptoUtils.hkdfExpand,
+  pbkdf2Generate: CryptoUtils.pbkdf2Generate,
+  deriveKeyFromPassphrase: CryptoUtils.deriveKeyFromPassphrase,
+  getHTTPMACSHA1Header: CryptoUtils.getHTTPMACSHA1Header,
+
+  /**
+   * The string to use as the base User-Agent in Sync requests.
+   * This string will look something like
+   *
+   *   Firefox/49.0a1 (Windows NT 6.1; WOW64; rv:46.0) FxSync/1.51.0.20160516142357.desktop
+   */
+  _userAgent: null,
+  get userAgent() {
+    if (!this._userAgent) {
+      let hph = Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler);
+      this._userAgent =
+        Services.appinfo.name + "/" + Services.appinfo.version +  // Product.
+        " (" + hph.oscpu + ")" +                                  // (oscpu)
+        " FxSync/" + WEAVE_VERSION + "." +                        // Sync.
+        Services.appinfo.appBuildID + ".";                        // Build.
+    }
+    return this._userAgent + Svc.Prefs.get("client.type", "desktop");
+  },
+
+  /**
+   * Wrap a function to catch all exceptions and log them
+   *
+   * @usage MyObj._catch = Utils.catch;
+   *        MyObj.foo = function() { this._catch(func)(); }
+   *
+   * Optionally pass a function which will be called if an
+   * exception occurs.
+   */
+  catch: function Utils_catch(func, exceptionCallback) {
+    let thisArg = this;
+    return function WrappedCatch() {
+      try {
+        return func.call(thisArg);
+      }
+      catch(ex) {
+        thisArg._log.debug("Exception calling " + (func.name || "anonymous function"), ex);
+        if (exceptionCallback) {
+          return exceptionCallback.call(thisArg, ex);
+        }
+        return null;
+      }
+    };
+  },
+
+  /**
+   * Wrap a function to call lock before calling the function then unlock.
+   *
+   * @usage MyObj._lock = Utils.lock;
+   *        MyObj.foo = function() { this._lock(func)(); }
+   */
+  lock: function lock(label, func) {
+    let thisArg = this;
+    return function WrappedLock() {
+      if (!thisArg.lock()) {
+        throw "Could not acquire lock. Label: \"" + label + "\".";
+      }
+
+      try {
+        return func.call(thisArg);
+      }
+      finally {
+        thisArg.unlock();
+      }
+    };
+  },
+
+  isLockException: function isLockException(ex) {
+    return ex && ex.indexOf && ex.indexOf("Could not acquire lock.") == 0;
+  },
+
+  /**
+   * Wrap functions to notify when it starts and finishes executing or if it
+   * threw an error.
+   *
+   * The message is a combination of a provided prefix, the local name, and
+   * the event. Possible events are: "start", "finish", "error". The subject
+   * is the function's return value on "finish" or the caught exception on
+   * "error". The data argument is the predefined data value.
+   *
+   * Example:
+   *
+   * @usage function MyObj(name) {
+   *          this.name = name;
+   *          this._notify = Utils.notify("obj:");
+   *        }
+   *        MyObj.prototype = {
+   *          foo: function() this._notify("func", "data-arg", function () {
+   *            //...
+   *          }(),
+   *        };
+   */
+  notify: function Utils_notify(prefix) {
+    return function NotifyMaker(name, data, func) {
+      let thisArg = this;
+      let notify = function(state, subject) {
+        let mesg = prefix + name + ":" + state;
+        thisArg._log.trace("Event: " + mesg);
+        Observers.notify(mesg, subject, data);
+      };
+
+      return function WrappedNotify() {
+        try {
+          notify("start", null);
+          let ret = func.call(thisArg);
+          notify("finish", ret);
+          return ret;
+        }
+        catch(ex) {
+          notify("error", ex);
+          throw ex;
+        }
+      };
+    };
+  },
+
+  /**
+   * GUIDs are 9 random bytes encoded with base64url (RFC 4648).
+   * That makes them 12 characters long with 72 bits of entropy.
+   */
+  makeGUID: function makeGUID() {
+    return CommonUtils.encodeBase64URL(Utils.generateRandomBytes(9));
+  },
+
+  _base64url_regex: /^[-abcdefghijklmnopqrstuvwxyz0123456789_]{12}$/i,
+  checkGUID: function checkGUID(guid) {
+    return !!guid && this._base64url_regex.test(guid);
+  },
+
+  /**
+   * Add a simple getter/setter to an object that defers access of a property
+   * to an inner property.
+   *
+   * @param obj
+   *        Object to add properties to defer in its prototype
+   * @param defer
+   *        Property of obj to defer to
+   * @param prop
+   *        Property name to defer (or an array of property names)
+   */
+  deferGetSet: function Utils_deferGetSet(obj, defer, prop) {
+    if (Array.isArray(prop))
+      return prop.map(prop => Utils.deferGetSet(obj, defer, prop));
+
+    let prot = obj.prototype;
+
+    // Create a getter if it doesn't exist yet
+    if (!prot.__lookupGetter__(prop)) {
+      prot.__defineGetter__(prop, function () {
+        return this[defer][prop];
+      });
+    }
+
+    // Create a setter if it doesn't exist yet
+    if (!prot.__lookupSetter__(prop)) {
+      prot.__defineSetter__(prop, function (val) {
+        this[defer][prop] = val;
+      });
+    }
+  },
+
+  lazyStrings: function Weave_lazyStrings(name) {
+    let bundle = "chrome://weave/locale/services/" + name + ".properties";
+    return () => new StringBundle(bundle);
+  },
+
+  deepEquals: function eq(a, b) {
+    // If they're triple equals, then it must be equals!
+    if (a === b)
+      return true;
+
+    // If they weren't equal, they must be objects to be different
+    if (typeof a != "object" || typeof b != "object")
+      return false;
+
+    // But null objects won't have properties to compare
+    if (a === null || b === null)
+      return false;
+
+    // Make sure all of a's keys have a matching value in b
+    for (let k in a)
+      if (!eq(a[k], b[k]))
+        return false;
+
+    // Do the same for b's keys but skip those that we already checked
+    for (let k in b)
+      if (!(k in a) && !eq(a[k], b[k]))
+        return false;
+
+    return true;
+  },
+
+  // Generator and discriminator for HMAC exceptions.
+  // Split these out in case we want to make them richer in future, and to
+  // avoid inevitable confusion if the message changes.
+  throwHMACMismatch: function throwHMACMismatch(shouldBe, is) {
+    throw "Record SHA256 HMAC mismatch: should be " + shouldBe + ", is " + is;
+  },
+
+  isHMACMismatch: function isHMACMismatch(ex) {
+    const hmacFail = "Record SHA256 HMAC mismatch: ";
+    return ex && ex.indexOf && (ex.indexOf(hmacFail) == 0);
+  },
+
+  /**
+   * Turn RFC 4648 base32 into our own user-friendly version.
+   *   ABCDEFGHIJKLMNOPQRSTUVWXYZ234567
+   * becomes
+   *   abcdefghijk8mn9pqrstuvwxyz234567
+   */
+  base32ToFriendly: function base32ToFriendly(input) {
+    return input.toLowerCase()
+                .replace(/l/g, '8')
+                .replace(/o/g, '9');
+  },
+
+  base32FromFriendly: function base32FromFriendly(input) {
+    return input.toUpperCase()
+                .replace(/8/g, 'L')
+                .replace(/9/g, 'O');
+  },
+
+  /**
+   * Key manipulation.
+   */
+
+  // Return an octet string in friendly base32 *with no trailing =*.
+  encodeKeyBase32: function encodeKeyBase32(keyData) {
+    return Utils.base32ToFriendly(
+             Utils.encodeBase32(keyData))
+           .slice(0, SYNC_KEY_ENCODED_LENGTH);
+  },
+
+  decodeKeyBase32: function decodeKeyBase32(encoded) {
+    return Utils.decodeBase32(
+             Utils.base32FromFriendly(
+               Utils.normalizePassphrase(encoded)))
+           .slice(0, SYNC_KEY_DECODED_LENGTH);
+  },
+
+  base64Key: function base64Key(keyData) {
+    return btoa(keyData);
+  },
+
+  /**
+   * N.B., salt should be base64 encoded, even though we have to decode
+   * it later!
+   */
+  derivePresentableKeyFromPassphrase : function derivePresentableKeyFromPassphrase(passphrase, salt, keyLength, forceJS) {
+    let k = CryptoUtils.deriveKeyFromPassphrase(passphrase, salt, keyLength,
+                                                forceJS);
+    return Utils.encodeKeyBase32(k);
+  },
+
+  /**
+   * N.B., salt should be base64 encoded, even though we have to decode
+   * it later!
+   */
+  deriveEncodedKeyFromPassphrase : function deriveEncodedKeyFromPassphrase(passphrase, salt, keyLength, forceJS) {
+    let k = CryptoUtils.deriveKeyFromPassphrase(passphrase, salt, keyLength,
+                                                forceJS);
+    return Utils.base64Key(k);
+  },
+
+  /**
+   * Take a base64-encoded 128-bit AES key, returning it as five groups of five
+   * uppercase alphanumeric characters, separated by hyphens.
+   * A.K.A. base64-to-base32 encoding.
+   */
+  presentEncodedKeyAsSyncKey : function presentEncodedKeyAsSyncKey(encodedKey) {
+    return Utils.encodeKeyBase32(atob(encodedKey));
+  },
+
+  /**
+   * Load a JSON file from disk in the profile directory.
+   *
+   * @param filePath
+   *        JSON file path load from profile. Loaded file will be
+   *        <profile>/<filePath>.json. i.e. Do not specify the ".json"
+   *        extension.
+   * @param that
+   *        Object to use for logging and "this" for callback.
+   * @param callback
+   *        Function to process json object as its first argument. If the file
+   *        could not be loaded, the first argument will be undefined.
+   */
+  jsonLoad: Task.async(function*(filePath, that, callback) {
+    let path = OS.Path.join(OS.Constants.Path.profileDir, "weave", filePath + ".json");
+
+    if (that._log) {
+      that._log.trace("Loading json from disk: " + filePath);
+    }
+
+    let json;
+
+    try {
+      json = yield CommonUtils.readJSON(path);
+    } catch (e) {
+      if (e instanceof OS.File.Error && e.becauseNoSuchFile) {
+        // Ignore non-existent files, but explicitly return null.
+        json = null;
+      } else {
+        if (that._log) {
+          that._log.debug("Failed to load json", e);
+        }
+      }
+    }
+
+    callback.call(that, json);
+  }),
+
+  /**
+   * Save a json-able object to disk in the profile directory.
+   *
+   * @param filePath
+   *        JSON file path save to <filePath>.json
+   * @param that
+   *        Object to use for logging and "this" for callback
+   * @param obj
+   *        Function to provide json-able object to save. If this isn't a
+   *        function, it'll be used as the object to make a json string.
+   * @param callback
+   *        Function called when the write has been performed. Optional.
+   *        The first argument will be a Components.results error
+   *        constant on error or null if no error was encountered (and
+   *        the file saved successfully).
+   */
+  jsonSave: Task.async(function*(filePath, that, obj, callback) {
+    let path = OS.Path.join(OS.Constants.Path.profileDir, "weave",
+                            ...(filePath + ".json").split("/"));
+    let dir = OS.Path.dirname(path);
+    let error = null;
+
+    try {
+      yield OS.File.makeDir(dir, { from: OS.Constants.Path.profileDir });
+
+      if (that._log) {
+        that._log.trace("Saving json to disk: " + path);
+      }
+
+      let json = typeof obj == "function" ? obj.call(that) : obj;
+
+      yield CommonUtils.writeJSON(json, path);
+    } catch (e) {
+      error = e
+    }
+
+    if (typeof callback == "function") {
+      callback.call(that, error);
+    }
+  }),
+
+  /**
+   * Move a json file in the profile directory. Will fail if a file exists at the
+   * destination.
+   *
+   * @returns a promise that resolves to undefined on success, or rejects on failure
+   *
+   * @param aFrom
+   *        Current path to the JSON file saved on disk, relative to profileDir/weave
+   *        .json will be appended to the file name.
+   * @param aTo
+   *        New path to the JSON file saved on disk, relative to profileDir/weave
+   *        .json will be appended to the file name.
+   * @param that
+   *        Object to use for logging
+   */
+  jsonMove(aFrom, aTo, that) {
+    let pathFrom = OS.Path.join(OS.Constants.Path.profileDir, "weave",
+                                ...(aFrom + ".json").split("/"));
+    let pathTo = OS.Path.join(OS.Constants.Path.profileDir, "weave",
+                              ...(aTo + ".json").split("/"));
+    if (that._log) {
+      that._log.trace("Moving " + pathFrom + " to " + pathTo);
+    }
+    return OS.File.move(pathFrom, pathTo, { noOverwrite: true });
+  },
+
+  /**
+   * Removes a json file in the profile directory.
+   *
+   * @returns a promise that resolves to undefined on success, or rejects on failure
+   *
+   * @param filePath
+   *        Current path to the JSON file saved on disk, relative to profileDir/weave
+   *        .json will be appended to the file name.
+   * @param that
+   *        Object to use for logging
+   */
+  jsonRemove(filePath, that) {
+    let path = OS.Path.join(OS.Constants.Path.profileDir, "weave",
+                            ...(filePath + ".json").split("/"));
+    if (that._log) {
+      that._log.trace("Deleting " + path);
+    }
+    return OS.File.remove(path, { ignoreAbsent: true });
+  },
+
+  getErrorString: function Utils_getErrorString(error, args) {
+    try {
+      return Str.errors.get(error, args || null);
+    } catch (e) {}
+
+    // basically returns "Unknown Error"
+    return Str.errors.get("error.reason.unknown");
+  },
+
+  /**
+   * Generate 26 characters.
+   */
+  generatePassphrase: function generatePassphrase() {
+    // Note that this is a different base32 alphabet to the one we use for
+    // other tasks. It's lowercase, uses different letters, and needs to be
+    // decoded with decodeKeyBase32, not just decodeBase32.
+    return Utils.encodeKeyBase32(CryptoUtils.generateRandomBytes(16));
+  },
+
+  /**
+   * The following are the methods supported for UI use:
+   *
+   * * isPassphrase:
+   *     determines whether a string is either a normalized or presentable
+   *     passphrase.
+   * * hyphenatePassphrase:
+   *     present a normalized passphrase for display. This might actually
+   *     perform work beyond just hyphenation; sorry.
+   * * hyphenatePartialPassphrase:
+   *     present a fragment of a normalized passphrase for display.
+   * * normalizePassphrase:
+   *     take a presentable passphrase and reduce it to a normalized
+   *     representation for storage. normalizePassphrase can safely be called
+   *     on normalized input.
+   * * normalizeAccount:
+   *     take user input for account/username, cleaning up appropriately.
+   */
+
+  isPassphrase: function(s) {
+    if (s) {
+      return /^[abcdefghijkmnpqrstuvwxyz23456789]{26}$/.test(Utils.normalizePassphrase(s));
+    }
+    return false;
+  },
+
+  /**
+   * Hyphenate a passphrase (26 characters) into groups.
+   * abbbbccccddddeeeeffffggggh
+   * =>
+   * a-bbbbc-cccdd-ddeee-effff-ggggh
+   */
+  hyphenatePassphrase: function hyphenatePassphrase(passphrase) {
+    // For now, these are the same.
+    return Utils.hyphenatePartialPassphrase(passphrase, true);
+  },
+
+  hyphenatePartialPassphrase: function hyphenatePartialPassphrase(passphrase, omitTrailingDash) {
+    if (!passphrase)
+      return null;
+
+    // Get the raw data input. Just base32.
+    let data = passphrase.toLowerCase().replace(/[^abcdefghijkmnpqrstuvwxyz23456789]/g, "");
+
+    // This is the neatest way to do this.
+    if ((data.length == 1) && !omitTrailingDash)
+      return data + "-";
+
+    // Hyphenate it.
+    let y = data.substr(0,1);
+    let z = data.substr(1).replace(/(.{1,5})/g, "-$1");
+
+    // Correct length? We're done.
+    if ((z.length == 30) || omitTrailingDash)
+      return y + z;
+
+    // Add a trailing dash if appropriate.
+    return (y + z.replace(/([^-]{5})$/, "$1-")).substr(0, SYNC_KEY_HYPHENATED_LENGTH);
+  },
+
+  normalizePassphrase: function normalizePassphrase(pp) {
+    // Short var name... have you seen the lines below?!
+    // Allow leading and trailing whitespace.
+    pp = pp.trim().toLowerCase();
+
+    // 20-char sync key.
+    if (pp.length == 23 &&
+        [5, 11, 17].every(i => pp[i] == '-')) {
+
+      return pp.slice(0, 5) + pp.slice(6, 11)
+             + pp.slice(12, 17) + pp.slice(18, 23);
+    }
+
+    // "Modern" 26-char key.
+    if (pp.length == 31 &&
+        [1, 7, 13, 19, 25].every(i => pp[i] == '-')) {
+
+      return pp.slice(0, 1) + pp.slice(2, 7)
+             + pp.slice(8, 13) + pp.slice(14, 19)
+             + pp.slice(20, 25) + pp.slice(26, 31);
+    }
+
+    // Something else -- just return.
+    return pp;
+  },
+
+  normalizeAccount: function normalizeAccount(acc) {
+    return acc.trim();
+  },
+
+  /**
+   * Create an array like the first but without elements of the second. Reuse
+   * arrays if possible.
+   */
+  arraySub: function arraySub(minuend, subtrahend) {
+    if (!minuend.length || !subtrahend.length)
+      return minuend;
+    return minuend.filter(i => subtrahend.indexOf(i) == -1);
+  },
+
+  /**
+   * Build the union of two arrays. Reuse arrays if possible.
+   */
+  arrayUnion: function arrayUnion(foo, bar) {
+    if (!foo.length)
+      return bar;
+    if (!bar.length)
+      return foo;
+    return foo.concat(Utils.arraySub(bar, foo));
+  },
+
+  bind2: function Async_bind2(object, method) {
+    return function innerBind() { return method.apply(object, arguments); };
+  },
+
+  /**
+   * Is there a master password configured, regardless of current lock state?
+   */
+  mpEnabled: function mpEnabled() {
+    let modules = Cc["@mozilla.org/security/pkcs11moduledb;1"]
+                    .getService(Ci.nsIPKCS11ModuleDB);
+    let sdrSlot = modules.findSlotByName("");
+    let status  = sdrSlot.status;
+    let slots = Ci.nsIPKCS11Slot;
+
+    return status != slots.SLOT_UNINITIALIZED && status != slots.SLOT_READY;
+  },
+
+  /**
+   * Is there a master password configured and currently locked?
+   */
+  mpLocked: function mpLocked() {
+    let modules = Cc["@mozilla.org/security/pkcs11moduledb;1"]
+                    .getService(Ci.nsIPKCS11ModuleDB);
+    let sdrSlot = modules.findSlotByName("");
+    let status  = sdrSlot.status;
+    let slots = Ci.nsIPKCS11Slot;
+
+    if (status == slots.SLOT_READY || status == slots.SLOT_LOGGED_IN
+                                   || status == slots.SLOT_UNINITIALIZED)
+      return false;
+
+    if (status == slots.SLOT_NOT_LOGGED_IN)
+      return true;
+
+    // something wacky happened, pretend MP is locked
+    return true;
+  },
+
+  // If Master Password is enabled and locked, present a dialog to unlock it.
+  // Return whether the system is unlocked.
+  ensureMPUnlocked: function ensureMPUnlocked() {
+    if (!Utils.mpLocked()) {
+      return true;
+    }
+    let sdr = Cc["@mozilla.org/security/sdr;1"]
+                .getService(Ci.nsISecretDecoderRing);
+    try {
+      sdr.encryptString("bacon");
+      return true;
+    } catch(e) {}
+    return false;
+  },
+
+  /**
+   * Return a value for a backoff interval.  Maximum is eight hours, unless
+   * Status.backoffInterval is higher.
+   *
+   */
+  calculateBackoff: function calculateBackoff(attempts, baseInterval,
+                                              statusInterval) {
+    let backoffInterval = attempts *
+                          (Math.floor(Math.random() * baseInterval) +
+                           baseInterval);
+    return Math.max(Math.min(backoffInterval, MAXIMUM_BACKOFF_INTERVAL),
+                    statusInterval);
+  },
+
+  /**
+   * Return a set of hostnames (including the protocol) which may have
+   * credentials for sync itself stored in the login manager.
+   *
+   * In general, these hosts will not have their passwords synced, will be
+   * reset when we drop sync credentials, etc.
+   */
+  getSyncCredentialsHosts: function() {
+    let result = new Set(this.getSyncCredentialsHostsLegacy());
+    for (let host of this.getSyncCredentialsHostsFxA()) {
+      result.add(host);
+    }
+    return result;
+  },
+
+  /*
+   * Get the "legacy" identity hosts.
+   */
+  getSyncCredentialsHostsLegacy: function() {
+    // the legacy sync host
+    return new Set([PWDMGR_HOST]);
+  },
+
+  /*
+   * Get the FxA identity hosts.
+   */
+  getSyncCredentialsHostsFxA: function() {
+    let result = new Set();
+    // the FxA host
+    result.add(FxAccountsCommon.FXA_PWDMGR_HOST);
+    // We used to include the FxA hosts (hence the Set() result) but we now
+    // don't give them special treatment (hence the Set() with exactly 1 item)
+    return result;
+  },
+
+  getDefaultDeviceName() {
+    // Generate a client name if we don't have a useful one yet
+    let env = Cc["@mozilla.org/process/environment;1"]
+                .getService(Ci.nsIEnvironment);
+    let user = env.get("USER") || env.get("USERNAME") ||
+               Svc.Prefs.get("account") || Svc.Prefs.get("username");
+    // A little hack for people using the the moz-build environment on Windows
+    // which sets USER to the literal "%USERNAME%" (yes, really)
+    if (user == "%USERNAME%" && env.get("USERNAME")) {
+      user = env.get("USERNAME");
+    }
+
+    let brand = new StringBundle("chrome://branding/locale/brand.properties");
+    let brandName = brand.get("brandShortName");
+
+    let appName;
+    try {
+      let syncStrings = new StringBundle("chrome://browser/locale/sync.properties");
+      appName = syncStrings.getFormattedString("sync.defaultAccountApplication", [brandName]);
+    } catch (ex) {}
+    appName = appName || brandName;
+
+    let system =
+      // 'device' is defined on unix systems
+      Cc["@mozilla.org/system-info;1"].getService(Ci.nsIPropertyBag2).get("device") ||
+      // hostname of the system, usually assigned by the user or admin
+      Cc["@mozilla.org/system-info;1"].getService(Ci.nsIPropertyBag2).get("host") ||
+      // fall back on ua info string
+      Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler).oscpu;
+
+    return Str.sync.get("client.name2", [user, appName, system]);
+  },
+
+  getDeviceName() {
+    const deviceName = Svc.Prefs.get("client.name", "");
+
+    if (deviceName === "") {
+      return this.getDefaultDeviceName();
+    }
+
+    return deviceName;
+  },
+
+  getDeviceType() {
+    return Svc.Prefs.get("client.type", DEVICE_TYPE_DESKTOP);
+  },
+
+  formatTimestamp(date) {
+    // Format timestamp as: "%Y-%m-%d %H:%M:%S"
+    let year = String(date.getFullYear());
+    let month = String(date.getMonth() + 1).padStart(2, "0");
+    let day = String(date.getDate()).padStart(2, "0");
+    let hours = String(date.getHours()).padStart(2, "0");
+    let minutes = String(date.getMinutes()).padStart(2, "0");
+    let seconds = String(date.getSeconds()).padStart(2, "0");
+
+    return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
+  }
+};
+
+XPCOMUtils.defineLazyGetter(Utils, "_utf8Converter", function() {
+  let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                    .createInstance(Ci.nsIScriptableUnicodeConverter);
+  converter.charset = "UTF-8";
+  return converter;
+});
+
+/*
+ * Commonly-used services
+ */
+this.Svc = {};
+Svc.Prefs = new Preferences(PREFS_BRANCH);
+Svc.DefaultPrefs = new Preferences({branch: PREFS_BRANCH, defaultBranch: true});
+Svc.Obs = Observers;
+
+var _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ?
+  "@mozilla.org/suite/sessionstore;1" :
+  "@mozilla.org/browser/sessionstore;1";
+
+[
+ ["Idle", "@mozilla.org/widget/idleservice;1", "nsIIdleService"],
+ ["Session", _sessionCID, "nsISessionStore"]
+].forEach(function([name, contract, iface]) {
+  XPCOMUtils.defineLazyServiceGetter(Svc, name, contract, iface);
+});
+
+XPCOMUtils.defineLazyModuleGetter(Svc, "FormHistory", "resource://gre/modules/FormHistory.jsm");
+
+Svc.__defineGetter__("Crypto", function() {
+  let cryptoSvc;
+  let ns = {};
+  Cu.import("resource://services-crypto/WeaveCrypto.js", ns);
+  cryptoSvc = new ns.WeaveCrypto();
+  delete Svc.Crypto;
+  return Svc.Crypto = cryptoSvc;
+});
+
+this.Str = {};
+["errors", "sync"].forEach(function(lazy) {
+  XPCOMUtils.defineLazyGetter(Str, lazy, Utils.lazyStrings(lazy));
+});
+
+Svc.Obs.add("xpcom-shutdown", function () {
+  for (let name in Svc)
+    delete Svc[name];
+});

services/sync/moz.build

@@ -0,0 +1,78 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+with Files('**'):
+    BUG_COMPONENT = ('Mozilla Services', 'Firefox Sync: Backend')
+
+DIRS += ['locales']
+
+XPCSHELL_TESTS_MANIFESTS += ['tests/unit/xpcshell.ini']
+
+EXTRA_COMPONENTS += [
+    'SyncComponents.manifest',
+    'Weave.js',
+]
+
+EXTRA_JS_MODULES['services-sync'] += [
+    'modules/addonsreconciler.js',
+    'modules/addonutils.js',
+    'modules/bookmark_validator.js',
+    'modules/browserid_identity.js',
+    'modules/collection_validator.js',
+    'modules/engines.js',
+    'modules/FxaMigrator.jsm',
+    'modules/identity.js',
+    'modules/jpakeclient.js',
+    'modules/keys.js',
+    'modules/main.js',
+    'modules/policies.js',
+    'modules/record.js',
+    'modules/resource.js',
+    'modules/rest.js',
+    'modules/service.js',
+    'modules/status.js',
+    'modules/SyncedTabs.jsm',
+    'modules/telemetry.js',
+    'modules/userapi.js',
+    'modules/util.js',
+]
+
+EXTRA_PP_JS_MODULES['services-sync'] += [
+    'modules/constants.js',
+]
+
+# Definitions used by constants.js
+DEFINES['weave_version'] = '1.54.0'
+DEFINES['weave_id'] = '{340c2bbc-ce74-4362-90b5-7c26312808ef}'
+
+EXTRA_JS_MODULES['services-sync'].engines += [
+    'modules/engines/addons.js',
+    'modules/engines/bookmarks.js',
+    'modules/engines/clients.js',
+    'modules/engines/extension-storage.js',
+    'modules/engines/forms.js',
+    'modules/engines/history.js',
+    'modules/engines/passwords.js',
+    'modules/engines/prefs.js',
+    'modules/engines/tabs.js',
+]
+
+EXTRA_JS_MODULES['services-sync'].stages += [
+    'modules/stages/cluster.js',
+    'modules/stages/declined.js',
+    'modules/stages/enginesync.js',
+]
+
+TESTING_JS_MODULES.services.sync += [
+    'modules-testing/fakeservices.js',
+    'modules-testing/fxa_utils.js',
+    'modules-testing/rotaryengine.js',
+    'modules-testing/utils.js',
+]
+
+JS_PREFERENCE_FILES += [
+    'services-sync.js',
+]

services/sync/services-sync.js

@@ -0,0 +1,95 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+pref("services.sync.serverURL", "https://auth.services.mozilla.com/");
+pref("services.sync.userURL", "user/");
+pref("services.sync.miscURL", "misc/");
+pref("services.sync.termsURL", "https://services.mozilla.com/tos/");
+pref("services.sync.privacyURL", "https://services.mozilla.com/privacy-policy/");
+pref("services.sync.statusURL", "https://services.mozilla.com/status/");
+pref("services.sync.syncKeyHelpURL", "https://services.mozilla.com/help/synckey");
+
+pref("services.sync.lastversion", "firstrun");
+pref("services.sync.sendVersionInfo", true);
+
+pref("services.sync.scheduler.eolInterval",         604800); // 1 week
+pref("services.sync.scheduler.idleInterval",         3600);  // 1 hour
+pref("services.sync.scheduler.activeInterval",       600);   // 10 minutes
+pref("services.sync.scheduler.immediateInterval",    90);    // 1.5 minutes
+pref("services.sync.scheduler.idleTime",             300);   // 5 minutes
+
+pref("services.sync.scheduler.fxa.singleDeviceInterval",     3600); // 1 hour
+pref("services.sync.scheduler.sync11.singleDeviceInterval", 86400); // 1 day
+
+pref("services.sync.errorhandler.networkFailureReportTimeout", 1209600); // 2 weeks
+
+pref("services.sync.engine.addons", true);
+pref("services.sync.engine.bookmarks", true);
+pref("services.sync.engine.history", true);
+pref("services.sync.engine.passwords", true);
+pref("services.sync.engine.prefs", true);
+pref("services.sync.engine.tabs", true);
+pref("services.sync.engine.tabs.filteredUrls", "^(about:.*|chrome://weave/.*|wyciwyg:.*|file:.*|blob:.*)$");
+
+pref("services.sync.jpake.serverURL", "https://setup.services.mozilla.com/");
+pref("services.sync.jpake.pollInterval", 1000);
+pref("services.sync.jpake.firstMsgMaxTries", 300); // 5 minutes
+pref("services.sync.jpake.lastMsgMaxTries", 300);  // 5 minutes
+pref("services.sync.jpake.maxTries", 10);
+
+// If true, add-on sync ignores changes to the user-enabled flag. This
+// allows people to have the same set of add-ons installed across all
+// profiles while maintaining different enabled states.
+pref("services.sync.addons.ignoreUserEnabledChanges", false);
+
+// Comma-delimited list of hostnames to trust for add-on install.
+pref("services.sync.addons.trustedSourceHostnames", "addons.mozilla.org");
+
+pref("services.sync.log.appender.console", "Fatal");
+pref("services.sync.log.appender.dump", "Error");
+pref("services.sync.log.appender.file.level", "Trace");
+pref("services.sync.log.appender.file.logOnError", true);
+pref("services.sync.log.appender.file.logOnSuccess", false);
+pref("services.sync.log.appender.file.maxErrorAge", 864000); // 10 days
+pref("services.sync.log.rootLogger", "Debug");
+pref("services.sync.log.logger.addonutils", "Debug");
+pref("services.sync.log.logger.declined", "Debug");
+pref("services.sync.log.logger.service.main", "Debug");
+pref("services.sync.log.logger.status", "Debug");
+pref("services.sync.log.logger.authenticator", "Debug");
+pref("services.sync.log.logger.network.resources", "Debug");
+pref("services.sync.log.logger.service.jpakeclient", "Debug");
+pref("services.sync.log.logger.engine.bookmarks", "Debug");
+pref("services.sync.log.logger.engine.clients", "Debug");
+pref("services.sync.log.logger.engine.forms", "Debug");
+pref("services.sync.log.logger.engine.history", "Debug");
+pref("services.sync.log.logger.engine.passwords", "Debug");
+pref("services.sync.log.logger.engine.prefs", "Debug");
+pref("services.sync.log.logger.engine.tabs", "Debug");
+pref("services.sync.log.logger.engine.addons", "Debug");
+pref("services.sync.log.logger.engine.extension-storage", "Debug");
+pref("services.sync.log.logger.engine.apps", "Debug");
+pref("services.sync.log.logger.identity", "Debug");
+pref("services.sync.log.logger.userapi", "Debug");
+pref("services.sync.log.cryptoDebug", false);
+
+pref("services.sync.fxa.termsURL", "https://accounts.firefox.com/legal/terms");
+pref("services.sync.fxa.privacyURL", "https://accounts.firefox.com/legal/privacy");
+
+pref("services.sync.telemetry.submissionInterval", 43200); // 12 hours in seconds
+pref("services.sync.telemetry.maxPayloadCount", 500);
+
+// Note that services.sync.validation.enabled is located in browser/app/profile/firefox.js
+
+// We consider validation this frequently. After considering validation, even
+// if we don't end up validating, we won't try again unless this much time has passed.
+pref("services.sync.validation.interval", 86400); // 24 hours in seconds
+
+// We only run validation `services.sync.validation.percentageChance` percent of
+// the time, even if it's been the right amount of time since the last validation,
+// and you meet the maxRecord checks.
+pref("services.sync.validation.percentageChance", 10);
+
+// We won't validate an engine if it has more than this many records on the server.
+pref("services.sync.validation.maxRecords", 100);

services/sync/tests/tps/addons/api/restartless-xpi@tests.mozilla.org.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5617">
+  <name>Restartless Test XPI</name>
+  <type id="1">Extension</type>
+  <guid>restartless-xpi@tests.mozilla.org</guid>
+  <slug>restartless-xpi</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="485">http://127.0.0.1:4567/addons/restartless.xpi</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/tps/addons/api/unsigned-xpi@tests.mozilla.org.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5612">
+  <name>Unsigned Test XPI</name>
+  <type id="1">Extension</type>
+  <guid>unsigned-xpi@tests.mozilla.org</guid>
+  <slug>unsigned-xpi</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="452">http://127.0.0.1:4567/addons/unsigned.xpi</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/tps/all_tests.json

@@ -0,0 +1,33 @@
+{ "tests": [
+    "test_bookmark_conflict.js",
+    "test_sync.js",
+    "test_prefs.js",
+    "test_tabs.js",
+    "test_passwords.js",
+    "test_history.js",
+    "test_formdata.js",
+    "test_bug530717.js",
+    "test_bug531489.js",
+    "test_bug538298.js",
+    "test_bug556509.js",
+    "test_bug562515.js",
+    "test_bug563989.js",
+    "test_bug535326.js",
+    "test_bug501528.js",
+    "test_bug575423.js",
+    "test_bug546807.js",
+    "test_history_collision.js",
+    "test_privbrw_passwords.js",
+    "test_privbrw_tabs.js",
+    "test_bookmarks_in_same_named_folder.js",
+    "test_client_wipe.js",
+    "test_special_tabs.js",
+    "test_addon_sanity.js",
+    "test_addon_restartless_xpi.js",
+    "test_addon_nonrestartless_xpi.js",
+    "test_addon_reconciling.js",
+    "test_addon_wipe.js"
+  ]
+}
+
+

services/sync/tests/tps/mozmill_sanity.js

@@ -0,0 +1,30 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+Components.utils.import('resource://tps/tps.jsm');
+
+var setupModule = function(module) {
+  module.controller = mozmill.getBrowserController();
+  assert.ok(true, "SetupModule passes");
+}
+
+var setupTest = function(module) {
+  assert.ok(true, "SetupTest passes");
+}
+
+var testTestStep = function() {
+  assert.ok(true, "test Passes");
+  controller.open("http://www.mozilla.org");
+
+  TPS.Login();
+  TPS.Sync(ACTIONS.ACTION_SYNC_WIPE_CLIENT);
+}
+
+var teardownTest = function () {
+  assert.ok(true, "teardownTest passes");
+}
+
+var teardownModule = function() {
+  assert.ok(true, "teardownModule passes");
+}

services/sync/tests/tps/mozmill_sanity2.js

@@ -0,0 +1,15 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var setupModule = function(module) {
+  module.controller = mozmill.getBrowserController();
+};
+
+var testGetNode = function() {
+  controller.open("about:support");
+  controller.waitForPageLoad();
+
+  var appbox = findElement.ID(controller.tabs.activeTab, "application-box");
+  assert.waitFor(() => appbox.getNode().textContent == 'Firefox', 'correct app name');
+};

services/sync/tests/tps/test_addon_nonrestartless_xpi.js

@@ -0,0 +1,105 @@
+
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test verifies that install of extensions that require restart
+// syncs between profiles.
+EnableEngines(["addons"]);
+
+var phases = {
+  "phase01": "profile1",
+  "phase02": "profile1",
+  "phase03": "profile2",
+  "phase04": "profile2",
+  "phase05": "profile1",
+  "phase06": "profile1",
+  "phase07": "profile2",
+  "phase08": "profile2",
+  "phase09": "profile1",
+  "phase10": "profile1",
+  "phase11": "profile2",
+  "phase12": "profile2",
+  "phase13": "profile1",
+  "phase14": "profile1",
+  "phase15": "profile2",
+  "phase16": "profile2"
+};
+
+const id = "unsigned-xpi@tests.mozilla.org";
+
+Phase("phase01", [
+  [Addons.verifyNot, [id]],
+  [Addons.install, [id]],
+  [Sync]
+]);
+Phase("phase02", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase03", [
+  [Addons.verifyNot, [id]],
+  [Sync]
+]);
+Phase("phase04", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+
+// Now we disable the add-on
+Phase("phase05", [
+  [EnsureTracking],
+  [Addons.setEnabled, [id], STATE_DISABLED],
+  [Sync]
+]);
+Phase("phase06", [
+  [Addons.verify, [id], STATE_DISABLED],
+  [Sync]
+]);
+Phase("phase07", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase08", [
+  [Addons.verify, [id], STATE_DISABLED],
+  [Sync]
+]);
+
+// Now we re-enable it again.
+Phase("phase09", [
+  [EnsureTracking],
+  [Addons.setEnabled, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase10", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase11", [
+  [Addons.verify, [id], STATE_DISABLED],
+  [Sync]
+]);
+Phase("phase12", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+
+// And we uninstall it
+
+Phase("phase13", [
+  [EnsureTracking],
+  [Addons.verify, [id], STATE_ENABLED],
+  [Addons.uninstall, [id]],
+  [Sync]
+]);
+Phase("phase14", [
+  [Addons.verifyNot, [id]],
+  [Sync]
+]);
+Phase("phase15", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase16", [
+  [Addons.verifyNot, [id]],
+  [Sync]
+]);

services/sync/tests/tps/test_addon_reconciling.js

@@ -0,0 +1,54 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test verifies that record reconciling works as expected. It makes
+// similar changes to add-ons in separate profiles and does a sync to verify
+// the proper action is taken.
+EnableEngines(["addons"]);
+
+var phases = {
+  "phase01": "profile1",
+  "phase02": "profile2",
+  "phase03": "profile1",
+  "phase04": "profile2",
+  "phase05": "profile1",
+  "phase06": "profile2"
+};
+
+const id = "restartless-xpi@tests.mozilla.org";
+
+// Install the add-on in 2 profiles.
+Phase("phase01", [
+  [Addons.verifyNot, [id]],
+  [Addons.install, [id]],
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase02", [
+  [Addons.verifyNot, [id]],
+  [Sync],
+  [Addons.verify, [id], STATE_ENABLED]
+]);
+
+// Now we disable in one and uninstall in the other.
+Phase("phase03", [
+  [Sync], // Get GUID updates, potentially.
+  [Addons.setEnabled, [id], STATE_DISABLED],
+  // We've changed the state, but don't want this profile to sync until phase5,
+  // so if we ran a validation now we'd be expecting to find errors.
+  [Addons.skipValidation]
+]);
+Phase("phase04", [
+  [EnsureTracking],
+  [Addons.uninstall, [id]],
+  [Sync]
+]);
+
+// When we sync, the uninstall should take precedence because it was newer.
+Phase("phase05", [
+  [Sync]
+]);
+Phase("phase06", [
+  [Sync],
+  [Addons.verifyNot, [id]]
+]);

services/sync/tests/tps/test_addon_restartless_xpi.js

@@ -0,0 +1,70 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test verifies that install of restartless extensions syncs to
+// other profiles.
+EnableEngines(["addons"]);
+
+var phases = {
+  "phase01": "profile1",
+  "phase02": "profile2",
+  "phase03": "profile1",
+  "phase04": "profile2",
+  "phase05": "profile1",
+  "phase06": "profile2",
+  "phase07": "profile1",
+  "phase08": "profile2"
+};
+
+const id = "restartless-xpi@tests.mozilla.org";
+
+// Verify install is synced
+Phase("phase01", [
+  [Addons.verifyNot, [id]],
+  [Addons.install, [id]],
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase02", [
+  [Addons.verifyNot, [id]],
+  [Sync],
+  [Addons.verify, [id], STATE_ENABLED]
+]);
+
+// Now disable and see that is is synced.
+Phase("phase03", [
+  [EnsureTracking],
+  [Addons.setEnabled, [id], STATE_DISABLED],
+  [Addons.verify, [id], STATE_DISABLED],
+  [Sync]
+]);
+Phase("phase04", [
+  [Sync],
+  [Addons.verify, [id], STATE_DISABLED]
+]);
+
+// Enable and see it is synced.
+Phase("phase05", [
+  [EnsureTracking],
+  [Addons.setEnabled, [id], STATE_ENABLED],
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync]
+]);
+Phase("phase06", [
+  [Sync],
+  [Addons.verify, [id], STATE_ENABLED]
+]);
+
+// Uninstall and see it is synced.
+Phase("phase07", [
+  [EnsureTracking],
+  [Addons.verify, [id], STATE_ENABLED],
+  [Addons.uninstall, [id]],
+  [Addons.verifyNot, [id]],
+  [Sync]
+]);
+Phase("phase08", [
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync],
+  [Addons.verifyNot, [id]]
+]);

services/sync/tests/tps/test_addon_sanity.js

@@ -0,0 +1,30 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+
+EnableEngines(["addons"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile1" };
+
+const id = "unsigned-xpi@tests.mozilla.org";
+
+Phase("phase1", [
+  [Addons.install, [id]],
+  // Non-restartless add-on shouldn't be found after install.
+  [Addons.verifyNot, [id]],
+
+  // But it should be marked for Sync.
+  [Sync]
+]);
+
+Phase("phase2", [
+  // Add-on should be present after restart
+  [Addons.verify, [id], STATE_ENABLED],
+  [Sync] // Sync to ensure everything is initialized enough for the addon validator to run
+]);

services/sync/tests/tps/test_addon_wipe.js

@@ -0,0 +1,35 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// This test ensures that a client wipe followed by an "initial" sync will
+// restore add-ons. This test should expose flaws in the reconciling logic,
+// specifically around AddonsReconciler. This test is in response to bug
+// 792990.
+
+EnableEngines(["addons"]);
+
+var phases = {
+  "phase01": "profile1",
+  "phase02": "profile1",
+  "phase03": "profile1"
+};
+
+const id1 = "restartless-xpi@tests.mozilla.org";
+const id2 = "unsigned-xpi@tests.mozilla.org";
+
+Phase("phase01", [
+  [Addons.install, [id1]],
+  [Addons.install, [id2]],
+  [Sync]
+]);
+Phase("phase02", [
+  [Addons.verify, [id1], STATE_ENABLED],
+  [Addons.verify, [id2], STATE_ENABLED],
+  [Sync, SYNC_WIPE_CLIENT],
+  [Sync]
+]);
+Phase("phase03", [
+  [Addons.verify, [id1], STATE_ENABLED],
+  [Addons.verify, [id2], STATE_ENABLED],
+  [Sync] // Sync to ensure that the addon validator can run without error
+]);

services/sync/tests/tps/test_bookmark_conflict.js

@@ -0,0 +1,143 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+
+// the initial list of bookmarks to add to the browser
+var bookmarksInitial = {
+  "menu": [
+    { folder: "foldera" },
+    { folder: "folderb" },
+    { folder: "folderc" },
+    { folder: "folderd" },
+  ],
+
+  "menu/foldera": [{ uri: "http://www.cnn.com", title: "CNN" }],
+  "menu/folderb": [{ uri: "http://www.apple.com", title: "Apple", tags: [] }],
+  "menu/folderc": [{ uri: "http://www.yahoo.com", title: "Yahoo" }],
+
+  "menu/folderd": []
+};
+
+// a list of bookmarks to delete during a 'delete' action on P2
+var bookmarksToDelete = {
+  "menu": [
+    { folder: "foldera" },
+    { folder: "folderb" },
+  ],
+  "menu/folderc": [{ uri: "http://www.yahoo.com", title: "Yahoo" }],
+};
+
+
+// the modifications to make on P1, after P2 has synced, but before P1 has gotten
+// P2's changes
+var bookmarkMods = {
+  "menu": [
+    { folder: "foldera" },
+    { folder: "folderb" },
+    { folder: "folderc" },
+    { folder: "folderd" },
+  ],
+
+  // we move this child out of its folder (p1), after deleting the folder (p2)
+  // and expect the child to come back to p2 after sync.
+  "menu/foldera": [{
+    uri: "http://www.cnn.com",
+    title: "CNN",
+    changes: { location: "menu/folderd" }
+  }],
+
+  // we rename this child (p1) after deleting the folder (p2), and expect the child
+  // to be moved into great grandparent (menu)
+  "menu/folderb": [{
+    uri: "http://www.apple.com",
+    title: "Apple",
+    tags: [],
+    changes: { title: "Mac" }
+  }],
+
+
+  // we move this child (p1) after deleting the child (p2) and expect it to survive
+  "menu/folderc": [{
+    uri: "http://www.yahoo.com",
+    title: "Yahoo",
+    changes: { location: "menu/folderd" }
+  }],
+
+  "menu/folderd": []
+};
+
+// a list of bookmarks to delete during a 'delete' action
+var bookmarksToDelete = {
+  "menu": [
+    { folder: "foldera" },
+    { folder: "folderb" },
+  ],
+  "menu/folderc": [
+    { uri: "http://www.yahoo.com", title: "Yahoo" },
+  ],
+};
+
+
+
+// expected bookmark state after conflict resolution
+var bookmarksExpected = {
+  "menu": [
+    { folder: "folderc" },
+    { folder: "folderd" },
+    { uri: "http://www.apple.com", title: "Mac", },
+  ],
+
+  "menu/folderc": [],
+
+  "menu/folderd": [
+    { uri: "http://www.cnn.com", title: "CNN" },
+    { uri: "http://www.yahoo.com", title: "Yahoo" }
+  ]
+};
+
+// Add bookmarks to profile1 and sync.
+Phase("phase1", [
+  [Bookmarks.add, bookmarksInitial],
+  [Bookmarks.verify, bookmarksInitial],
+  [Sync],
+  [Bookmarks.verify, bookmarksInitial],
+]);
+
+// Sync to profile2 and verify that the bookmarks are present. Delete
+// bookmarks/folders, verify that it's not present, and sync
+Phase("phase2", [
+  [Sync],
+  [Bookmarks.verify, bookmarksInitial],
+  [Bookmarks.delete, bookmarksToDelete],
+  [Bookmarks.verifyNot, bookmarksToDelete],
+  [Sync]
+]);
+
+// Using profile1, modify the bookmarks, and sync *after* the modification,
+// and then sync again to propagate the reconciliation changes.
+Phase("phase3", [
+  [Bookmarks.verify, bookmarksInitial],
+  [Bookmarks.modify, bookmarkMods],
+  [Sync],
+  [Bookmarks.verify, bookmarksExpected],
+  [Bookmarks.verifyNot, bookmarksToDelete],
+]);
+
+// Back in profile2, do a sync and verify that we're in the expected state
+Phase("phase4", [
+  [Sync],
+  [Bookmarks.verify, bookmarksExpected],
+  [Bookmarks.verifyNot, bookmarksToDelete],
+]);

services/sync/tests/tps/test_bookmarks_in_same_named_folder.js

@@ -0,0 +1,69 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// bug 558077
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1"};
+
+var bookmarks_initial_1 = {
+  "menu": [
+    { folder: "aaa",
+      description: "foo"
+    },
+    { uri: "http://www.mozilla.com"
+    }
+  ],
+  "menu/aaa": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo"
+    },
+    { uri: "http://www.google.com",
+      title: "testing Google"
+    }
+  ]
+};
+
+var bookmarks_initial_2 = {
+  "menu": [
+    { folder: "aaa",
+      description: "bar"
+    },
+    { uri: "http://www.mozilla.com"
+    }
+  ],
+  "menu/aaa": [
+    { uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s",
+      title: "Bugzilla"
+    },
+    { uri: "http://www.apple.com",
+      tags: [ "apple" ]
+    }
+  ]
+};
+
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial_1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial_1],
+  [Bookmarks.add, bookmarks_initial_2],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  // XXX [Bookmarks.verify, bookmarks_initial_1],
+  [Bookmarks.verify, bookmarks_initial_2]
+]);

services/sync/tests/tps/test_bug501528.js

@@ -0,0 +1,79 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["passwords"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Password lists
+ */
+
+var passwords_initial = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "secret",
+    usernameField: "uname",
+    passwordField: "pword",
+    changes: {
+      password: "SeCrEt$$$"
+    }
+  },
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "jack",
+    password: "secretlogin"
+  }
+];
+
+var passwords_after_first_update = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "SeCrEt$$$",
+    usernameField: "uname",
+    passwordField: "pword"
+  },
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "jack",
+    password: "secretlogin"
+  }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Passwords.add, passwords_initial],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Passwords.add, passwords_initial],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Passwords.verify, passwords_initial],
+  [Passwords.modify, passwords_initial],
+  [Passwords.verify, passwords_after_first_update],
+  [Sync]
+]);
+
+Phase('phase4', [
+  [Sync],
+  [Passwords.verify, passwords_after_first_update],
+]);
+

services/sync/tests/tps/test_bug530717.js

@@ -0,0 +1,69 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["prefs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1"};
+
+/*
+ * Preference lists
+ */
+
+var prefs1 = [
+  { name: "browser.startup.homepage",
+    value: "http://www.getfirefox.com"
+  },
+  { name: "browser.urlbar.maxRichResults",
+    value: 20
+  },
+  { name: "privacy.clearOnShutdown.siteSettings",
+    value: true
+  }
+];
+
+var prefs2 = [
+  { name: "browser.startup.homepage",
+    value: "http://www.mozilla.com"
+  },
+  { name: "browser.urlbar.maxRichResults",
+    value: 18
+  },
+  { name: "privacy.clearOnShutdown.siteSettings",
+    value: false
+  }
+];
+
+/*
+ * Test phases
+ */
+
+// Add prefs to profile1 and sync.
+Phase('phase1', [
+  [Prefs.modify, prefs1],
+  [Prefs.verify, prefs1],
+  [Sync]
+]);
+
+// Sync profile2 and verify same prefs are present.
+Phase('phase2', [
+  [Sync],
+  [Prefs.verify, prefs1]
+]);
+
+// Using profile1, change some prefs, then do another sync with wipe-client.
+// Verify that the cloud's prefs are restored, and the recent local changes
+// discarded.
+Phase('phase3', [
+  [Prefs.modify, prefs2],
+  [Prefs.verify, prefs2],
+  [Sync, SYNC_WIPE_CLIENT],
+  [Prefs.verify, prefs1]
+]);
+

services/sync/tests/tps/test_bug531489.js

@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1"};
+
+/*
+ * Bookmark asset lists: these define bookmarks that are used during the test
+ */
+
+// the initial list of bookmarks to add to the browser
+var bookmarks_initial = {
+  "menu": [
+    { folder: "foldera" },
+    { uri: "http://www.google.com",
+      title: "Google"
+    }
+  ],
+  "menu/foldera": [
+    { uri: "http://www.google.com",
+      title: "Google"
+    }
+  ],
+  "toolbar": [
+    { uri: "http://www.google.com",
+      title: "Google"
+    }
+  ]
+};
+
+/*
+ * Test phases
+ */
+
+// Add three bookmarks with the same url to different locations and sync.
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial],
+  [Bookmarks.verify, bookmarks_initial],
+  [Sync]
+]);
+
+// Sync to profile2 and verify that all three bookmarks are present
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+
+// Sync again to profile1 and verify that all three bookmarks are still
+// present.
+Phase('phase3', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+

services/sync/tests/tps/test_bug535326.js

@@ -0,0 +1,130 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["tabs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2"};
+
+var tabs1 = [
+  { uri: "data:text/html,<html><head><title>Howdy</title></head><body>Howdy</body></html>",
+    title: "Howdy",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>America</title></head><body>America</body></html>",
+    title: "America",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Apple</title></head><body>Apple</body></html>",
+    title: "Apple",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>This</title></head><body>This</body></html>",
+    title: "This",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Bug</title></head><body>Bug</body></html>",
+    title: "Bug",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>IRC</title></head><body>IRC</body></html>",
+    title: "IRC",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Tinderbox</title></head><body>Tinderbox</body></html>",
+    title: "Tinderbox",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Fox</title></head><body>Fox</body></html>",
+    title: "Fox",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Hello</title></head><body>Hello</body></html>",
+    title: "Hello",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Eagle</title></head><body>Eagle</body></html>",
+    title: "Eagle",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Train</title></head><body>Train</body></html>",
+    title: "Train",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Macbook</title></head><body>Macbook</body></html>",
+    title: "Macbook",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Clock</title></head><body>Clock</body></html>",
+    title: "Clock",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Google</title></head><body>Google</body></html>",
+    title: "Google",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Human</title></head><body>Human</body></html>",
+    title: "Human",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Jetpack</title></head><body>Jetpack</body></html>",
+    title: "Jetpack",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Selenium</title></head><body>Selenium</body></html>",
+    title: "Selenium",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Mozilla</title></head><body>Mozilla</body></html>",
+    title: "Mozilla",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>",
+    title: "Firefox",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Weave</title></head><body>Weave</body></html>",
+    title: "Weave",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Android</title></head><body>Android</body></html>",
+    title: "Android",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Bye</title></head><body>Bye</body></html>",
+    title: "Bye",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Hi</title></head><body>Hi</body></html>",
+    title: "Hi",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Final</title></head><body>Final</body></html>",
+    title: "Final",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Fennec</title></head><body>Fennec</body></html>",
+    title: "Fennec",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Mobile</title></head><body>Mobile</body></html>",
+    title: "Mobile",
+    profile: "profile1"
+  }
+];
+
+Phase('phase1', [
+  [Tabs.add, tabs1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Tabs.verify, tabs1]
+]);
+

services/sync/tests/tps/test_bug538298.js

@@ -0,0 +1,93 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Bookmark asset lists: these define bookmarks that are used during the test
+ */
+
+// the initial list of bookmarks to add to the browser
+var bookmarks_initial = {
+  "toolbar": [
+    { uri: "http://www.google.com",
+      title: "Google"
+    },
+    { uri: "http://www.cnn.com",
+      title: "CNN",
+      changes: {
+        position: "Google"
+      }
+    },
+    { uri: "http://www.mozilla.com",
+      title: "Mozilla"
+    },
+    { uri: "http://www.firefox.com",
+      title: "Firefox",
+      changes: {
+        position: "Mozilla"
+      }
+    }
+  ]
+};
+
+var bookmarks_after_move = {
+  "toolbar": [
+    { uri: "http://www.cnn.com",
+      title: "CNN"
+    },
+    { uri: "http://www.google.com",
+      title: "Google"
+    },
+    { uri: "http://www.firefox.com",
+      title: "Firefox"
+    },
+    { uri: "http://www.mozilla.com",
+      title: "Mozilla"
+    }
+  ]
+};
+
+/*
+ * Test phases
+ */
+
+// Add four bookmarks to the toolbar and sync.
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial],
+  [Bookmarks.verify, bookmarks_initial],
+  [Sync]
+]);
+
+// Sync to profile2 and verify that all four bookmarks are present.
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+
+// Change the order of the toolbar bookmarks, and sync.
+Phase('phase3', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial],
+  [Bookmarks.modify, bookmarks_initial],
+  [Bookmarks.verify, bookmarks_after_move],
+  [Sync],
+]);
+
+// Go back to profile2, sync, and verify that the bookmarks are reordered
+// as expected.
+Phase('phase4', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_after_move]
+]);
+

services/sync/tests/tps/test_bug546807.js

@@ -0,0 +1,54 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+
+EnableEngines(["tabs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2"};
+
+/*
+ * Tabs data
+ */
+
+var tabs1 = [
+  { uri: "about:config",
+    profile: "profile1"
+  },
+  { uri: "about:credits",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Apple</title></head><body>Apple</body></html>",
+    title: "Apple",
+    profile: "profile1"
+  }
+];
+
+var tabs_absent = [
+  { uri: "about:config",
+    profile: "profile1"
+  },
+  { uri: "about:credits",
+    profile: "profile1"
+  },
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Tabs.add, tabs1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Tabs.verifyNot, tabs_absent]
+]);
+

services/sync/tests/tps/test_bug556509.js

@@ -0,0 +1,45 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2"};
+
+
+// the initial list of bookmarks to add to the browser
+var bookmarks_initial = {
+  "menu": [
+    { folder: "testfolder",
+      description: "it's just me, a test folder"
+    }
+  ],
+  "menu/testfolder": [
+    { uri: "http://www.mozilla.com",
+      title: "Mozilla"
+    }
+  ]
+};
+
+/*
+ * Test phases
+ */
+
+// Add a bookmark folder which has a description, and sync.
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial],
+  [Bookmarks.verify, bookmarks_initial],
+  [Sync]
+]);
+
+// Sync to profile2 and verify that the bookmark folder is created, along
+// with its description.
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);

services/sync/tests/tps/test_bug562515.js

@@ -0,0 +1,105 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Bookmark lists
+ */
+
+// the initial list of bookmarks to add to the browser
+var bookmarks_initial = {
+  "menu": [
+    { uri: "http://www.google.com",
+      loadInSidebar: true,
+      tags: [ "google", "computers", "internet", "www"]
+    },
+    { uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s",
+      title: "Bugzilla",
+      keyword: "bz"
+    },
+    { folder: "foldera" },
+    { uri: "http://www.mozilla.com" },
+    { separator: true },
+    { folder: "folderb" }
+  ],
+  "menu/foldera": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo"
+    },
+    { uri: "http://www.cnn.com",
+      description: "This is a description of the site a at www.cnn.com"
+    },
+    { livemark: "Livemark1",
+      feedUri: "http://rss.wunderground.com/blog/JeffMasters/rss.xml",
+      siteUri: "http://www.wunderground.com/blog/JeffMasters/show.html"
+    }
+  ],
+  "menu/folderb": [
+    { uri: "http://www.apple.com",
+      tags: [ "apple", "mac" ]
+    }
+  ],
+  "toolbar": [
+    { uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0",
+      title: "Visited Today"
+    }
+  ]
+};
+
+// a list of bookmarks to delete during a 'delete' action
+var bookmarks_to_delete = {
+  "menu": [
+    { uri: "http://www.google.com",
+      loadInSidebar: true,
+      tags: [ "google", "computers", "internet", "www"]
+    }
+  ],
+  "menu/foldera": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo"
+    }
+  ]
+};
+
+/*
+ * Test phases
+ */
+
+// add bookmarks to profile1 and sync
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial],
+  [Bookmarks.verify, bookmarks_initial],
+  [Sync]
+]);
+
+// sync to profile2 and verify that the bookmarks are present
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+
+// delete some bookmarks from profile1, then sync with "wipe-client"
+// set; finally, verify that the deleted bookmarks were restored.
+Phase('phase3', [
+  [Bookmarks.delete, bookmarks_to_delete],
+  [Bookmarks.verifyNot, bookmarks_to_delete],
+  [Sync, SYNC_WIPE_CLIENT],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+
+// sync profile2 again, verify no bookmarks have been deleted
+Phase('phase4', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);

services/sync/tests/tps/test_bug563989.js

@@ -0,0 +1,107 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["bookmarks"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Bookmark asset lists: these define bookmarks that are used during the test
+ */
+
+// the initial list of bookmarks to add to the browser
+var bookmarks_initial = {
+  "menu": [
+    { uri: "http://www.google.com",
+      loadInSidebar: true,
+      tags: [ "google", "computers", "internet", "www" ]
+    },
+    { uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s",
+      title: "Bugzilla",
+      keyword: "bz"
+    },
+    { folder: "foldera" },
+    { uri: "http://www.mozilla.com" },
+    { separator: true },
+    { folder: "folderb" }
+  ],
+  "menu/foldera": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo"
+    },
+    { uri: "http://www.cnn.com",
+      description: "This is a description of the site a at www.cnn.com"
+    },
+    { livemark: "Livemark1",
+      feedUri: "http://rss.wunderground.com/blog/JeffMasters/rss.xml",
+      siteUri: "http://www.wunderground.com/blog/JeffMasters/show.html"
+    }
+  ],
+  "menu/folderb": [
+    { uri: "http://www.apple.com",
+      tags: [ "apple", "mac" ]
+    }
+  ],
+  "toolbar": [
+    { uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0",
+      title: "Visited Today"
+    }
+  ]
+};
+
+// a list of bookmarks to delete during a 'delete' action
+var bookmarks_to_delete = {
+  "menu/folderb": [
+    { uri: "http://www.apple.com",
+      tags: [ "apple", "mac" ]
+    }
+  ],
+  "toolbar": [
+    { uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0",
+      title: "Visited Today"
+    }
+  ]
+};
+
+/*
+ * Test phases
+ */
+
+// Add bookmarks to profile1 and sync.
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial],
+  [Bookmarks.verify, bookmarks_initial],
+  [Sync],
+]);
+
+// Sync to profile2 and verify that the bookmarks are present.  Delete 
+// some bookmarks, and verify that they're not present, but don't sync again.
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial],
+  [Bookmarks.delete, bookmarks_to_delete],
+  [Bookmarks.verifyNot, bookmarks_to_delete],
+  [Bookmarks.skipValidation]
+]);
+
+// Using profile1, sync again with wipe-server set to true.  Verify our
+// initial bookmarks are still all present.
+Phase('phase3', [
+  [Sync, SYNC_WIPE_REMOTE],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+
+// Back in profile2, do a sync and verify that the bookmarks we had
+// deleted earlier are now restored.
+Phase('phase4', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial]
+]);

services/sync/tests/tps/test_bug575423.js

@@ -0,0 +1,85 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+
+EnableEngines(["history"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2"};
+
+/*
+ * History data
+ */
+
+// the history data to add to the browser
+var history1 = [
+  { uri: "http://www.google.com/",
+    title: "Google",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -1
+      }
+    ]
+  },
+  { uri: "http://www.cnn.com/",
+    title: "CNN",
+    visits: [
+      { type: 1,
+        date: -1
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  }
+];
+
+// Another history data to add to the browser
+var history2 = [
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  },
+  { uri: "http://www.google.com/language_tools?hl=en",
+    title: "Language Tools",
+    visits: [
+      { type: 1, 
+        date: 0
+      },
+      { type: 2, 
+        date: -40
+      }
+    ]
+  }
+];
+
+/*
+ * Test phases
+ */
+Phase('phase1', [
+  [History.add, history1],
+  [Sync],
+  [History.add, history2],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [History.verify, history2]
+]);
+

services/sync/tests/tps/test_client_wipe.js

@@ -0,0 +1,164 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1"};
+
+/*
+ * Bookmark lists
+ */
+
+// the initial list of bookmarks to add to the browser
+var bookmarks_initial = {
+  toolbar: [
+    { uri: "http://www.google.com",
+      title: "Google"
+    },
+    { uri: "http://www.cnn.com",
+      title: "CNN",
+      changes: {
+        position: "Google"
+      }
+    },
+    { uri: "http://www.mozilla.com",
+      title: "Mozilla"
+    },
+    { uri: "http://www.firefox.com",
+      title: "Firefox",
+      changes: {
+        position: "Mozilla"
+      }
+    }
+  ]
+};
+
+var bookmarks_after_move = {
+  toolbar: [
+    { uri: "http://www.cnn.com",
+      title: "CNN"
+    },
+    { uri: "http://www.google.com",
+      title: "Google"
+    },
+    { uri: "http://www.firefox.com",
+      title: "Firefox"
+    },
+    { uri: "http://www.mozilla.com",
+      title: "Mozilla"
+    }
+  ]
+};
+
+/*
+ * Password data
+ */
+
+// Initial password data
+var passwords_initial = [
+   { hostname: "http://www.example.com",
+     submitURL: "http://login.example.com",
+     username: "joe",
+     password: "secret",
+     usernameField: "uname",
+     passwordField: "pword",
+     changes: {
+       password: "SeCrEt$$$"
+     }
+   },
+   { hostname: "http://www.example.com",
+     realm: "login",
+     username: "jack",
+     password: "secretlogin"
+   }
+];
+
+// Password after first modify action has been performed
+var passwords_after_change = [
+   { hostname: "http://www.example.com",
+     submitURL: "http://login.example.com",
+     username: "joe",
+     password: "SeCrEt$$$",
+     usernameField: "uname",
+     passwordField: "pword",
+     changes: {
+        username: "james"
+     }
+   },
+   { hostname: "http://www.example.com",
+     realm: "login",
+     username: "jack",
+     password: "secretlogin"
+   }
+];
+
+/*
+ * Prefs to use in the test
+ */
+var prefs1 = [
+  { name: "browser.startup.homepage",
+    value: "http://www.getfirefox.com"
+  },
+  { name: "browser.urlbar.maxRichResults",
+    value: 20
+  },
+  { name: "privacy.clearOnShutdown.siteSettings",
+    value: true
+  }
+];
+
+var prefs2 = [
+  { name: "browser.startup.homepage",
+    value: "http://www.mozilla.com"
+  },
+  { name: "browser.urlbar.maxRichResults",
+    value: 18
+  },
+  { name: "privacy.clearOnShutdown.siteSettings",
+    value: false
+  }
+];
+
+/*
+ * Test phases
+ */
+
+// Add prefs,passwords and bookmarks to profile1 and sync.
+Phase('phase1', [
+  [Passwords.add, passwords_initial],
+  [Bookmarks.add, bookmarks_initial],
+  [Prefs.modify, prefs1],
+  [Prefs.verify, prefs1],
+  [Sync]
+]);
+
+// Sync profile2 and verify same prefs,passwords and bookmarks are present.
+Phase('phase2', [
+  [Sync],
+  [Prefs.verify, prefs1],
+  [Passwords.verify, passwords_initial],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+
+// Using profile1, change some prefs,bookmarks and pwds, then do another sync with wipe-client.
+// Verify that the cloud's  settings are restored, and the recent local changes
+// discarded.
+Phase('phase3', [
+  [Prefs.modify, prefs2],
+  [Passwords.modify, passwords_initial],
+  [Bookmarks.modify, bookmarks_initial],
+  [Prefs.verify, prefs2],
+  [Passwords.verify, passwords_after_change],
+  [Bookmarks.verify, bookmarks_after_move],
+  [Sync, SYNC_WIPE_CLIENT],
+  [Prefs.verify, prefs1],
+  [Passwords.verify, passwords_initial],
+  [Bookmarks.verify, bookmarks_initial]
+]);
+

services/sync/tests/tps/test_formdata.js

@@ -0,0 +1,97 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["forms"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Form data asset lists: these define form values that are used in the tests.
+ */
+
+var formdata1 = [
+  { fieldname: "testing",
+    value: "success",
+    date: -1
+  },
+  { fieldname: "testing",
+    value: "failure",
+    date: -2
+  },
+  { fieldname: "username",
+    value: "joe"
+  }
+];
+
+// This is currently pointless - it *looks* like it is trying to check that
+// one of the entries in formdata1 has been removed, but (a) the delete code
+// isn't active (see comments below), and (b) the way the verification works
+// means it would never do the right thing - it only checks all the entries
+// here exist, but not that they are the only entries in the DB.
+var formdata2 = [
+  { fieldname: "testing",
+    value: "success",
+    date: -1
+  },
+  { fieldname: "username",
+    value: "joe"
+  }
+];
+
+var formdata_delete = [
+  { fieldname: "testing",
+    value: "failure"
+  }
+];
+
+var formdata_new = [
+  { fieldname: "new-field",
+    value: "new-value"
+  }
+]
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Formdata.add, formdata1],
+  [Formdata.verify, formdata1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Formdata.verify, formdata1],
+]);
+
+/*
+ * Note: Weave does not support syncing deleted form data, so those
+ * tests are disabled below.  See bug 568363.
+ */
+
+Phase('phase3', [
+  [Sync],
+  [Formdata.delete, formdata_delete],
+//[Formdata.verifyNot, formdata_delete],
+  [Formdata.verify, formdata2],
+  // add new data after the first Sync, ensuring the tracker works.
+  [Formdata.add, formdata_new],
+  [Sync],
+]);
+
+Phase('phase4', [
+  [Sync],
+  [Formdata.verify, formdata2],
+  [Formdata.verify, formdata_new],
+//[Formdata.verifyNot, formdata_delete]
+]);
+
+

services/sync/tests/tps/test_history.js

@@ -0,0 +1,167 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["history"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2" };
+
+/*
+ * History asset lists: these define history entries that are used during
+ * the test
+ */
+
+// the initial list of history items to add to the browser
+var history1 = [
+  { uri: "http://www.google.com/",
+    title: "Google",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -1
+      }
+    ]
+  },
+  { uri: "http://www.cnn.com/",
+    title: "CNN",
+    visits: [
+      { type: 1,
+        date: -1
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  },
+  { uri: "http://www.google.com/language_tools?hl=en",
+    title: "Language Tools",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -40
+      }
+    ]
+  },
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 1,
+        date: -1
+      },
+      { type: 1,
+        date: -20
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  }
+];
+
+// a list of items to delete from the history
+var history_to_delete = [
+  { uri: "http://www.cnn.com/" },
+  { begin: -24,
+    end: -1
+  },
+  { host: "www.google.com" }
+];
+
+// a list which reflects items that should be in the history after
+// the above items are deleted
+var history2 = [
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  }
+];
+
+// a list which includes history entries that should not be present
+// after deletion of the history_to_delete entries
+var history_not = [
+  { uri: "http://www.google.com/",
+    title: "Google",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -1
+      }
+    ]
+  },
+  { uri: "http://www.cnn.com/",
+    title: "CNN",
+    visits: [
+      { type: 1,
+        date: -1
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  },
+  { uri: "http://www.google.com/language_tools?hl=en",
+    title: "Language Tools",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -40
+      }
+    ]
+  },
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1,
+        date: -1
+      },
+      { type: 1,
+        date: -20
+      }
+    ]
+  }
+];
+
+/*
+ * Test phases
+ * Note:  there is no test phase in which deleted history entries are
+ * synced to other clients.  This functionality is not supported by
+ * Sync, see bug 446517.
+ */
+
+Phase('phase1', [
+  [History.add, history1],
+  [Sync],
+]);
+
+Phase('phase2', [
+  [Sync],
+  [History.verify, history1],
+  [History.delete, history_to_delete],
+  [History.verify, history2],
+  [History.verifyNot, history_not],
+  [Sync]
+]);
+

services/sync/tests/tps/test_history_collision.js

@@ -0,0 +1,125 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["history"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * History lists
+ */
+
+// the initial list of history to add to the browser
+var history1 = [
+   { uri: "http://www.google.com/",
+     title: "Google",
+     visits: [
+       { type: 1,
+         date: 0
+       }
+     ]
+   },
+   { uri: "http://www.cnn.com/",
+     title: "CNN",
+     visits: [
+       { type: 1,
+         date: -1
+       },
+       { type: 2,
+         date: -36
+       }
+     ]
+   },
+   { uri: "http://www.mozilla.com/",
+     title: "Mozilla",
+     visits: [
+       { type: 1,
+         date: 0
+       },
+       { type: 2,
+         date: -36
+       }
+     ]
+   }
+];
+
+// the history to delete
+var history_to_delete = [
+   { uri: "http://www.cnn.com/",
+     title: "CNN"
+   },
+   { begin: -36,
+     end: -1
+   }
+];
+
+var history_not = [
+   { uri: "http://www.cnn.com/",
+     title: "CNN",
+     visits: [
+       { type: 1,
+         date: -1
+       },
+       { type: 2,
+         date: -36
+       }
+     ]
+   }
+];
+
+var history_after_delete = [
+   { uri: "http://www.google.com/",
+     title: "Google",
+     visits: [
+       { type: 1,
+         date: 0
+       }
+     ]
+   },
+   { uri: "http://www.mozilla.com/",
+     title: "Mozilla",
+     visits: [
+       { type: 1,
+         date: 0
+       }
+     ]
+   }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [History.add, history1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [History.add, history1],
+  [Sync, SYNC_WIPE_REMOTE]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [History.verify, history1],
+  [History.delete, history_to_delete],
+  [History.verify, history_after_delete],
+  [History.verifyNot, history_not],
+  [Sync]
+]);
+
+Phase('phase4', [
+  [Sync],
+  [History.verify, history_after_delete],
+  [History.verifyNot, history_not]
+]);
+

services/sync/tests/tps/test_mozmill_sanity.js

@@ -0,0 +1,24 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2" };
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [RunMozmillTest, 'mozmill_sanity.js'],
+]);
+
+Phase('phase2', [
+  [Sync],
+  [RunMozmillTest, 'mozmill_sanity2.js'],
+]);

services/sync/tests/tps/test_passwords.js

@@ -0,0 +1,113 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["passwords"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Password asset lists: these define password entries that are used during
+ * the test
+ */
+
+// initial password list to be loaded into the browser
+var passwords_initial = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "SeCrEt123",
+    usernameField: "uname",
+    passwordField: "pword",
+    changes: {
+      password: "zippity-do-dah"
+    }
+  },
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+// expected state of passwords after the changes in the above list are applied
+var passwords_after_first_update = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "zippity-do-dah",
+    usernameField: "uname",
+    passwordField: "pword"
+  },
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+var passwords_to_delete = [
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+var passwords_absent = [
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+// expected state of passwords after the delete operation
+var passwords_after_second_update = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "zippity-do-dah",
+    usernameField: "uname",
+    passwordField: "pword"
+  }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Passwords.add, passwords_initial],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Passwords.verify, passwords_initial],
+  [Passwords.modify, passwords_initial],
+  [Passwords.verify, passwords_after_first_update],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Passwords.verify, passwords_after_first_update],
+  [Passwords.delete, passwords_to_delete],
+  [Passwords.verify, passwords_after_second_update],
+  [Passwords.verifyNot, passwords_absent],
+  [Sync]
+]);
+
+Phase('phase4', [
+  [Sync],
+  [Passwords.verify, passwords_after_second_update],
+  [Passwords.verifyNot, passwords_absent]
+]);

services/sync/tests/tps/test_prefs.js

@@ -0,0 +1,57 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["prefs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1"};
+
+var prefs1 = [
+  { name: "browser.startup.homepage",
+    value: "http://www.getfirefox.com"
+  },
+  { name: "browser.urlbar.maxRichResults",
+    value: 20
+  },
+  { name: "privacy.clearOnShutdown.siteSettings",
+    value: true
+  }
+];
+
+var prefs2 = [
+  { name: "browser.startup.homepage",
+    value: "http://www.mozilla.com"
+  },
+  { name: "browser.urlbar.maxRichResults",
+    value: 18
+  },
+  { name: "privacy.clearOnShutdown.siteSettings",
+    value: false
+  }
+];
+
+Phase('phase1', [
+  [Prefs.modify, prefs1],
+  [Prefs.verify, prefs1],
+  [Sync],
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Prefs.verify, prefs1],
+  [Prefs.modify, prefs2],
+  [Prefs.verify, prefs2],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Prefs.verify, prefs2]
+]);
+

services/sync/tests/tps/test_privbrw_passwords.js

@@ -0,0 +1,104 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["passwords"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Password data
+ */
+
+// Initial password data
+var passwords_initial = [
+   { hostname: "http://www.example.com",
+     submitURL: "http://login.example.com",
+     username: "joe",
+     password: "secret",
+     usernameField: "uname",
+     passwordField: "pword",
+     changes: {
+       password: "SeCrEt$$$"
+     }
+   },
+   { hostname: "http://www.example.com",
+     realm: "login",
+     username: "jack",
+     password: "secretlogin"
+   }
+];
+
+// Password after first modify action has been performed
+var passwords_after_first_change = [
+   { hostname: "http://www.example.com",
+     submitURL: "http://login.example.com",
+     username: "joe",
+     password: "SeCrEt$$$",
+     usernameField: "uname",
+     passwordField: "pword",
+     changes: {
+        username: "james"
+     }
+   },
+   { hostname: "http://www.example.com",
+     realm: "login",
+     username: "jack",
+     password: "secretlogin"
+   }
+];
+
+// Password after second modify action has been performed
+var passwords_after_second_change = [
+   { hostname: "http://www.example.com",
+     submitURL: "http://login.example.com",
+     username: "james",
+     password: "SeCrEt$$$",
+     usernameField: "uname",
+     passwordField: "pword"
+   },
+   { hostname: "http://www.example.com",
+     realm: "login",
+     username: "jack",
+     password: "secretlogin"
+   }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Passwords.add, passwords_initial],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Passwords.verify, passwords_initial],
+  [Passwords.modify, passwords_initial],
+  [Passwords.verify, passwords_after_first_change],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Windows.add, { private: true }],
+  [Passwords.verify, passwords_after_first_change],
+  [Passwords.modify, passwords_after_first_change],
+  [Passwords.verify, passwords_after_second_change],
+  [Sync]
+]);
+
+Phase('phase4', [
+  [Sync],
+  [Passwords.verify, passwords_after_second_change]
+]);
+

services/sync/tests/tps/test_privbrw_tabs.js

@@ -0,0 +1,87 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["tabs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Tabs data
+ */
+
+var tabs1 = [
+  { uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>",
+    title: "Firefox",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Weave</title></head><body>Weave</body></html>",
+    title: "Weave",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Apple</title></head><body>Apple</body></html>",
+    title: "Apple",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>IRC</title></head><body>IRC</body></html>",
+    title: "IRC",
+    profile: "profile1"
+  }
+];
+
+var tabs2 = [
+  { uri: "data:text/html,<html><head><title>Tinderbox</title></head><body>Tinderbox</body></html>",
+    title: "Tinderbox",
+    profile: "profile2"
+  },
+  { uri: "data:text/html,<html><head><title>Fox</title></head><body>Fox</body></html>",
+    title: "Fox",
+    profile: "profile2"
+  }
+];
+
+var tabs3 = [
+  { uri: "data:text/html,<html><head><title>Jetpack</title></head><body>Jetpack</body></html>",
+    title: "Jetpack",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Selenium</title></head><body>Selenium</body></html>",
+    title: "Selenium",
+    profile: "profile1"
+  }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Tabs.add, tabs1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Tabs.verify, tabs1],
+  [Tabs.add, tabs2],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Windows.add, { private: true }],
+  [Tabs.add, tabs3],
+  [Sync]
+]);
+
+Phase('phase4', [
+  [Sync],
+  [Tabs.verifyNot, tabs3]
+]);

services/sync/tests/tps/test_special_tabs.js

@@ -0,0 +1,78 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Bug 532173 - Dont sync tabs like about:* , weave firstrun etc
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["tabs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2" };
+
+var tabs1 = [
+  { uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>",
+    title: "Firefox",
+    profile: "profile1"
+  },
+  { uri: "about:plugins",
+    title: "About",
+    profile: "profile1"
+  },
+  { uri: "about:credits",
+    title: "Credits",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Mozilla</title></head><body>Mozilla</body></html>",
+    title: "Mozilla",
+    profile: "profile1"
+  },
+  { uri: "http://www.mozilla.com/en-US/firefox/sync/firstrun.html",
+    title: "Firstrun",
+    profile: "profile1"
+  }
+];
+
+var tabs2 = [
+  { uri: "data:text/html,<html><head><title>Firefox</title></head><body>Firefox</body></html>",
+    title: "Firefox",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Mozilla</title></head><body>Mozilla</body></html>",
+    title: "Mozilla",
+    profile: "profile1"
+  }
+];
+
+var tabs3 = [
+  { uri: "http://www.mozilla.com/en-US/firefox/sync/firstrun.html",
+    title: "Firstrun",
+    profile: "profile1"
+  },
+  { uri: "about:plugins",
+    title: "About",
+    profile: "profile1"
+  },
+  { uri: "about:credits",
+    title: "Credits",
+    profile: "profile1"
+  }
+];
+
+/*
+ * Test phases
+ */
+Phase('phase1', [
+  [Tabs.add, tabs1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Tabs.verify, tabs2],
+  [Tabs.verifyNot, tabs3]
+]);
+

services/sync/tests/tps/test_sync.js

@@ -0,0 +1,424 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1",
+               "phase4": "profile2" };
+
+/*
+ * Bookmark asset lists: these define bookmarks that are used during the test
+ */
+
+// the initial list of bookmarks to be added to the browser
+var bookmarks_initial = {
+  "menu": [
+    { uri: "http://www.google.com",
+      loadInSidebar: true,
+      tags: ["google", "computers", "internet", "www"],
+      changes: {
+        title: "Google",
+        loadInSidebar: false,
+        tags: ["google", "computers", "misc"],
+      }
+    },
+    { uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s",
+      title: "Bugzilla",
+      keyword: "bz",
+      changes: {
+        keyword: "bugzilla" 
+      }
+    },
+    { folder: "foldera" },
+    { uri: "http://www.mozilla.com" },
+    { separator: true },
+    { folder: "folderb" },
+  ],
+  "menu/foldera": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo",
+      changes: {
+        location: "menu/folderb"
+      }
+    },
+    { uri: "http://www.cnn.com",
+      description: "This is a description of the site a at www.cnn.com",
+      changes: {
+        uri: "http://money.cnn.com",
+        description: "new description",
+      }
+    },
+    { livemark: "Livemark1",
+      feedUri: "http://rss.wunderground.com/blog/JeffMasters/rss.xml",
+      siteUri: "http://www.wunderground.com/blog/JeffMasters/show.html",
+      changes: {
+        livemark: "LivemarkOne"
+      }
+    },
+  ],
+  "menu/folderb": [
+    { uri: "http://www.apple.com",
+      tags: ["apple", "mac"],
+      changes: {
+        uri: "http://www.apple.com/iphone/",
+        title: "iPhone",
+        location: "menu",
+        position: "Google",
+        tags: []
+      }
+    }
+  ],
+  toolbar: [
+    { uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0",
+      title: "Visited Today"
+    }
+  ]
+};
+
+// the state of bookmarks after the first 'modify' action has been performed
+// on them
+var bookmarks_after_first_modify = {
+  "menu": [
+    { uri: "http://www.apple.com/iphone/",
+      title: "iPhone",
+      before: "Google",
+      tags: []
+    },
+    { uri: "http://www.google.com",
+      title: "Google",
+      loadInSidebar: false,
+      tags: [ "google", "computers", "misc"]
+    },
+    { uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s",
+      title: "Bugzilla",
+      keyword: "bugzilla"
+    },
+    { folder: "foldera" },
+    { uri: "http://www.mozilla.com" },
+    { separator: true },
+    { folder: "folderb",
+      changes: {
+        location: "menu/foldera",
+        folder: "Folder B",
+        description: "folder description"
+      }
+    }
+  ],
+  "menu/foldera": [
+    { uri: "http://money.cnn.com",
+      title: "http://www.cnn.com",
+      description: "new description"
+    },
+    { livemark: "LivemarkOne",
+      feedUri: "http://rss.wunderground.com/blog/JeffMasters/rss.xml",
+      siteUri: "http://www.wunderground.com/blog/JeffMasters/show.html"
+    }
+  ],
+  "menu/folderb": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo"
+    }
+  ],
+  "toolbar": [
+    { uri: "place:queryType=0&sort=8&maxResults=10&beginTimeRef=1&beginTime=0",
+      title: "Visited Today"
+    }
+  ]
+};
+
+// a list of bookmarks to delete during a 'delete' action
+var bookmarks_to_delete = {
+  "menu": [
+    { uri: "http://www.google.com",
+      title: "Google",
+      loadInSidebar: false,
+      tags: [ "google", "computers", "misc" ]
+    }
+  ]
+};
+
+// the state of bookmarks after the second 'modify' action has been performed
+// on them
+var bookmarks_after_second_modify = {
+  "menu": [
+    { uri: "http://www.apple.com/iphone/",
+      title: "iPhone"
+    },
+    { uri: "http://bugzilla.mozilla.org/show_bug.cgi?id=%s",
+      title: "Bugzilla",
+      keyword: "bugzilla"
+    },
+    { folder: "foldera" },
+    { uri: "http://www.mozilla.com" },
+    { separator: true },
+  ],
+  "menu/foldera": [
+    { uri: "http://money.cnn.com",
+      title: "http://www.cnn.com",
+      description: "new description"
+    },
+    { livemark: "LivemarkOne",
+      feedUri: "http://rss.wunderground.com/blog/JeffMasters/rss.xml",
+      siteUri: "http://www.wunderground.com/blog/JeffMasters/show.html"
+    },
+    { folder: "Folder B",
+      description: "folder description"
+    }
+  ],
+  "menu/foldera/Folder B": [
+    { uri: "http://www.yahoo.com",
+      title: "testing Yahoo"
+    }
+  ]
+};
+
+// a list of bookmarks which should not be present after the last
+// 'delete' and 'modify' actions
+var bookmarks_absent = {
+  "menu": [
+    { uri: "http://www.google.com",
+      title: "Google"
+    },
+    { folder: "folderb" },
+    { folder: "Folder B" }
+  ]
+};
+
+/*
+ * History asset lists: these define history entries that are used during
+ * the test
+ */
+
+// the initial list of history items to add to the browser
+var history_initial = [
+  { uri: "http://www.google.com/",
+    title: "Google",
+    visits: [
+      { type: 1, date: 0 },
+      { type: 2, date: -1 }
+    ]
+  },
+  { uri: "http://www.cnn.com/",
+    title: "CNN",
+    visits: [
+      { type: 1, date: -1 },
+      { type: 2, date: -36 }
+    ]
+  },
+  { uri: "http://www.google.com/language_tools?hl=en",
+    title: "Language Tools",
+    visits: [
+      { type: 1, date: 0 },
+      { type: 2, date: -40 }
+    ]
+  },
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1, date: 0 },
+      { type: 1, date: -1 },
+      { type: 1, date: -20 },
+      { type: 2, date: -36 }
+    ]
+  }
+];
+
+// a list of history entries to delete during a 'delete' action
+var history_to_delete = [
+  { uri: "http://www.cnn.com/" },
+  { begin: -24,
+    end: -1 },
+  { host: "www.google.com" }
+];
+
+// the expected history entries after the first 'delete' action
+var history_after_delete = [
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  }
+];
+
+// history entries expected to not exist after a 'delete' action
+var history_absent = [
+  { uri: "http://www.google.com/",
+    title: "Google",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -1
+      }
+    ]
+  },
+  { uri: "http://www.cnn.com/",
+    title: "CNN",
+    visits: [
+      { type: 1,
+        date: -1
+      },
+      { type: 2,
+        date: -36
+      }
+    ]
+  },
+  { uri: "http://www.google.com/language_tools?hl=en",
+    title: "Language Tools",
+    visits: [
+      { type: 1,
+        date: 0
+      },
+      { type: 2,
+        date: -40
+      }
+    ]
+  },
+  { uri: "http://www.mozilla.com/",
+    title: "Mozilla",
+    visits: [
+      { type: 1,
+        date: -1
+      },
+      { type: 1,
+        date: -20
+      }
+    ]
+  }
+];
+
+/*
+ * Password asset lists: these define password entries that are used during
+ * the test
+ */
+
+// the initial list of passwords to add to the browser
+var passwords_initial = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "SeCrEt123",
+    usernameField: "uname",
+    passwordField: "pword",
+    changes: {
+      password: "zippity-do-dah"
+    }
+  },
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+// the expected state of passwords after the first 'modify' action
+var passwords_after_first_modify = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "zippity-do-dah",
+    usernameField: "uname",
+    passwordField: "pword"
+  },
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+// a list of passwords to delete during a 'delete' action
+var passwords_to_delete = [
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+// a list of passwords expected to be absent after 'delete' and 'modify'
+// actions
+var passwords_absent = [
+  { hostname: "http://www.example.com",
+    realm: "login",
+    username: "joe",
+    password: "secretlogin"
+  }
+];
+
+// the expected state of passwords after the seconds 'modify' action
+var passwords_after_second_modify = [
+  { hostname: "http://www.example.com",
+    submitURL: "http://login.example.com",
+    username: "joe",
+    password: "zippity-do-dah",
+    usernameField: "uname",
+    passwordField: "pword"
+  }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Bookmarks.add, bookmarks_initial],
+  [Passwords.add, passwords_initial],
+  [History.add, history_initial],
+  [Sync],
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_initial],
+  [Passwords.verify, passwords_initial],
+  [History.verify, history_initial],
+  [Bookmarks.modify, bookmarks_initial],
+  [Passwords.modify, passwords_initial],
+  [History.delete, history_to_delete],
+  [Bookmarks.verify, bookmarks_after_first_modify],
+  [Passwords.verify, passwords_after_first_modify],
+  [History.verify, history_after_delete],
+  [History.verifyNot, history_absent],
+  [Sync],
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_after_first_modify],
+  [Passwords.verify, passwords_after_first_modify],
+  [History.verify, history_after_delete],
+  [Bookmarks.modify, bookmarks_after_first_modify],
+  [Passwords.modify, passwords_after_first_modify],
+  [Bookmarks.delete, bookmarks_to_delete],
+  [Passwords.delete, passwords_to_delete],
+  [Bookmarks.verify, bookmarks_after_second_modify],
+  [Passwords.verify, passwords_after_second_modify],
+  [Bookmarks.verifyNot, bookmarks_absent],
+  [Passwords.verifyNot, passwords_absent],
+  [Sync],
+]);
+
+Phase('phase4', [
+  [Sync],
+  [Bookmarks.verify, bookmarks_after_second_modify],
+  [Passwords.verify, passwords_after_second_modify],
+  [Bookmarks.verifyNot, bookmarks_absent],
+  [Passwords.verifyNot, passwords_absent],
+  [History.verifyNot, history_absent],
+]);
+
+

services/sync/tests/tps/test_tabs.js

@@ -0,0 +1,59 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * The list of phases mapped to their corresponding profiles.  The object
+ * here must be in strict JSON format, as it will get parsed by the Python
+ * testrunner (no single quotes, extra comma's, etc).
+ */
+EnableEngines(["tabs"]);
+
+var phases = { "phase1": "profile1",
+               "phase2": "profile2",
+               "phase3": "profile1"};
+
+/*
+ * Tab lists.
+ */
+
+var tabs1 = [
+  { uri: "http://mozqa.com/data/firefox/layout/mozilla.html",
+    title: "Mozilla",
+    profile: "profile1"
+  },
+  { uri: "data:text/html,<html><head><title>Hello</title></head><body>Hello</body></html>",
+    title: "Hello",
+    profile: "profile1"
+  }
+];
+
+var tabs2 = [
+  { uri: "http://mozqa.com/data/firefox/layout/mozilla_community.html",
+    title: "Mozilla Community",
+    profile: "profile2"
+  },
+  { uri: "data:text/html,<html><head><title>Bye</title></head><body>Bye</body></html>",
+    profile: "profile2"
+  }
+];
+
+/*
+ * Test phases
+ */
+
+Phase('phase1', [
+  [Tabs.add, tabs1],
+  [Sync]
+]);
+
+Phase('phase2', [
+  [Sync],
+  [Tabs.verify, tabs1],
+  [Tabs.add, tabs2],
+  [Sync]
+]);
+
+Phase('phase3', [
+  [Sync],
+  [Tabs.verify, tabs2]
+]);

services/sync/tests/unit/addon1-search.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5617">
+  <name>Non-Restartless Test Extension</name>
+  <type id="1">Extension</type>
+  <guid>addon1@tests.mozilla.org</guid>
+  <slug>addon11</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>xpcshell@tests.mozilla.org</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="485">http://127.0.0.1:8888/addon1.xpi</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/unit/bootstrap1-search.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5617">
+  <name>Restartless Test Extension</name>
+  <type id="1">Extension</type>
+  <guid>bootstrap1@tests.mozilla.org</guid>
+  <slug>bootstrap1</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>xpcshell@tests.mozilla.org</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="485">http://127.0.0.1:8888/bootstrap1.xpi</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/unit/fake_login_manager.js

@@ -0,0 +1,38 @@
+Cu.import("resource://services-sync/util.js");
+
+// ----------------------------------------
+// Fake Sample Data
+// ----------------------------------------
+
+var fakeSampleLogins = [
+  // Fake nsILoginInfo object.
+  {hostname: "www.boogle.com",
+   formSubmitURL: "http://www.boogle.com/search",
+   httpRealm: "",
+   username: "",
+   password: "",
+   usernameField: "test_person",
+   passwordField: "test_password"}
+];
+
+// ----------------------------------------
+// Fake Login Manager
+// ----------------------------------------
+
+function FakeLoginManager(fakeLogins) {
+  this.fakeLogins = fakeLogins;
+
+  let self = this;
+
+  // Use a fake nsILoginManager object.
+  delete Services.logins;
+  Services.logins = {
+      removeAllLogins: function() { self.fakeLogins = []; },
+      getAllLogins: function() { return self.fakeLogins; },
+      addLogin: function(login) {
+        getTestLogger().info("nsILoginManager.addLogin() called " +
+                             "with hostname '" + login.hostname + "'.");
+        self.fakeLogins.push(login);
+      }
+  };
+}

services/sync/tests/unit/head_appinfo.js

@@ -0,0 +1,57 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+var gSyncProfile;
+
+gSyncProfile = do_get_profile();
+
+// Init FormHistoryStartup and pretend we opened a profile.
+var fhs = Cc["@mozilla.org/satchel/form-history-startup;1"]
+            .getService(Ci.nsIObserver);
+fhs.observe(null, "profile-after-change", null);
+
+// An app is going to have some prefs set which xpcshell tests don't.
+Services.prefs.setCharPref("identity.sync.tokenserver.uri", "http://token-server");
+
+// Set the validation prefs to attempt validation every time to avoid non-determinism.
+Services.prefs.setIntPref("services.sync.validation.interval", 0);
+Services.prefs.setIntPref("services.sync.validation.percentageChance", 100);
+Services.prefs.setIntPref("services.sync.validation.maxRecords", -1);
+Services.prefs.setBoolPref("services.sync.validation.enabled", true);
+
+// Make sure to provide the right OS so crypto loads the right binaries
+function getOS() {
+  switch (mozinfo.os) {
+    case "win":
+      return "WINNT";
+    case "mac":
+      return "Darwin";
+    default:
+      return "Linux";
+  }
+}
+
+Cu.import("resource://testing-common/AppInfo.jsm", this);
+updateAppInfo({
+  name: "XPCShell",
+  ID: "xpcshell@tests.mozilla.org",
+  version: "1",
+  platformVersion: "",
+  OS: getOS(),
+});
+
+// Register resource aliases. Normally done in SyncComponents.manifest.
+function addResourceAlias() {
+  const resProt = Services.io.getProtocolHandler("resource")
+                          .QueryInterface(Ci.nsIResProtocolHandler);
+  for (let s of ["common", "sync", "crypto"]) {
+    let uri = Services.io.newURI("resource://gre/modules/services-" + s + "/", null,
+                                 null);
+    resProt.setSubstitution("services-" + s, uri);
+  }
+}
+addResourceAlias();

services/sync/tests/unit/head_errorhandler_common.js

@@ -0,0 +1,112 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-sync/engines.js");
+
+// Common code for test_errorhandler_{1,2}.js -- pulled out to make it less
+// monolithic and take less time to execute.
+const EHTestsCommon = {
+
+  service_unavailable(request, response) {
+    let body = "Service Unavailable";
+    response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
+    response.setHeader("Retry-After", "42");
+    response.bodyOutputStream.write(body, body.length);
+  },
+
+  sync_httpd_setup() {
+    let global = new ServerWBO("global", {
+      syncID: Service.syncID,
+      storageVersion: STORAGE_VERSION,
+      engines: {clients: {version: Service.clientsEngine.version,
+                          syncID: Service.clientsEngine.syncID},
+                catapult: {version: Service.engineManager.get("catapult").version,
+                           syncID: Service.engineManager.get("catapult").syncID}}
+    });
+    let clientsColl = new ServerCollection({}, true);
+
+    // Tracking info/collections.
+    let collectionsHelper = track_collections_helper();
+    let upd = collectionsHelper.with_updated_collection;
+
+    let handler_401 = httpd_handler(401, "Unauthorized");
+    return httpd_setup({
+      // Normal server behaviour.
+      "/1.1/johndoe/storage/meta/global": upd("meta", global.handler()),
+      "/1.1/johndoe/info/collections": collectionsHelper.handler,
+      "/1.1/johndoe/storage/crypto/keys":
+        upd("crypto", (new ServerWBO("keys")).handler()),
+      "/1.1/johndoe/storage/clients": upd("clients", clientsColl.handler()),
+
+      // Credentials are wrong or node reallocated.
+      "/1.1/janedoe/storage/meta/global": handler_401,
+      "/1.1/janedoe/info/collections": handler_401,
+
+      // Maintenance or overloaded (503 + Retry-After) at info/collections.
+      "/maintenance/1.1/broken.info/info/collections": EHTestsCommon.service_unavailable,
+
+      // Maintenance or overloaded (503 + Retry-After) at meta/global.
+      "/maintenance/1.1/broken.meta/storage/meta/global": EHTestsCommon.service_unavailable,
+      "/maintenance/1.1/broken.meta/info/collections": collectionsHelper.handler,
+
+      // Maintenance or overloaded (503 + Retry-After) at crypto/keys.
+      "/maintenance/1.1/broken.keys/storage/meta/global": upd("meta", global.handler()),
+      "/maintenance/1.1/broken.keys/info/collections": collectionsHelper.handler,
+      "/maintenance/1.1/broken.keys/storage/crypto/keys": EHTestsCommon.service_unavailable,
+
+      // Maintenance or overloaded (503 + Retry-After) at wiping collection.
+      "/maintenance/1.1/broken.wipe/info/collections": collectionsHelper.handler,
+      "/maintenance/1.1/broken.wipe/storage/meta/global": upd("meta", global.handler()),
+      "/maintenance/1.1/broken.wipe/storage/crypto/keys":
+        upd("crypto", (new ServerWBO("keys")).handler()),
+      "/maintenance/1.1/broken.wipe/storage": EHTestsCommon.service_unavailable,
+      "/maintenance/1.1/broken.wipe/storage/clients": upd("clients", clientsColl.handler()),
+      "/maintenance/1.1/broken.wipe/storage/catapult": EHTestsCommon.service_unavailable
+    });
+  },
+
+  CatapultEngine: (function() {
+    function CatapultEngine() {
+      SyncEngine.call(this, "Catapult", Service);
+    }
+    CatapultEngine.prototype = {
+      __proto__: SyncEngine.prototype,
+      exception: null, // tests fill this in
+      _sync: function _sync() {
+        if (this.exception) {
+          throw this.exception;
+        }
+      }
+    };
+
+    return CatapultEngine;
+  }()),
+
+
+  generateCredentialsChangedFailure() {
+    // Make sync fail due to changed credentials. We simply re-encrypt
+    // the keys with a different Sync Key, without changing the local one.
+    let newSyncKeyBundle = new SyncKeyBundle("johndoe", "23456234562345623456234562");
+    let keys = Service.collectionKeys.asWBO();
+    keys.encrypt(newSyncKeyBundle);
+    keys.upload(Service.resource(Service.cryptoKeysURL));
+  },
+
+  setUp(server) {
+    return configureIdentity({ username: "johndoe" }).then(
+      () => {
+        Service.serverURL  = server.baseURI + "/";
+        Service.clusterURL = server.baseURI + "/";
+      }
+    ).then(
+      () => EHTestsCommon.generateAndUploadKeys()
+    );
+  },
+
+  generateAndUploadKeys() {
+    generateNewKeys(Service.collectionKeys);
+    let serverKeys = Service.collectionKeys.asWBO("crypto", "keys");
+    serverKeys.encrypt(Service.identity.syncKeyBundle);
+    return serverKeys.upload(Service.resource(Service.cryptoKeysURL)).success;
+  }
+};

services/sync/tests/unit/head_helpers.js

@@ -0,0 +1,446 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://testing-common/services/common/utils.js");
+Cu.import("resource://testing-common/PlacesTestUtils.jsm");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyGetter(this, 'SyncPingSchema', function() {
+  let ns = {};
+  Cu.import("resource://gre/modules/FileUtils.jsm", ns);
+  let stream = Cc["@mozilla.org/network/file-input-stream;1"]
+               .createInstance(Ci.nsIFileInputStream);
+  let jsonReader = Cc["@mozilla.org/dom/json;1"]
+                   .createInstance(Components.interfaces.nsIJSON);
+  let schema;
+  try {
+    let schemaFile = do_get_file("sync_ping_schema.json");
+    stream.init(schemaFile, ns.FileUtils.MODE_RDONLY, ns.FileUtils.PERMS_FILE, 0);
+    schema = jsonReader.decodeFromStream(stream, stream.available());
+  } finally {
+    stream.close();
+  }
+
+  // Allow tests to make whatever engines they want, this shouldn't cause
+  // validation failure.
+  schema.definitions.engine.properties.name = { type: "string" };
+  return schema;
+});
+
+XPCOMUtils.defineLazyGetter(this, 'SyncPingValidator', function() {
+  let ns = {};
+  Cu.import("resource://testing-common/ajv-4.1.1.js", ns);
+  let ajv = new ns.Ajv({ async: "co*" });
+  return ajv.compile(SyncPingSchema);
+});
+
+var provider = {
+  getFile: function(prop, persistent) {
+    persistent.value = true;
+    switch (prop) {
+      case "ExtPrefDL":
+        return [Services.dirsvc.get("CurProcD", Ci.nsIFile)];
+      default:
+        throw Cr.NS_ERROR_FAILURE;
+    }
+  },
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIDirectoryServiceProvider])
+};
+Services.dirsvc.QueryInterface(Ci.nsIDirectoryService).registerProvider(provider);
+
+// This is needed for loadAddonTestFunctions().
+var gGlobalScope = this;
+
+function ExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/mozapps/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the AddonManager test functions by importing its test file.
+ *
+ * This should be called in the global scope of any test file needing to
+ * interface with the AddonManager. It should only be called once, or the
+ * universe will end.
+ */
+function loadAddonTestFunctions() {
+  const path = ExtensionsTestPath("/head_addons.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+}
+
+function webExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/components/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the WebExtension test functions by importing its test file.
+ */
+function loadWebExtensionTestFunctions() {
+  const path = webExtensionsTestPath("/head_sync.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+}
+
+function getAddonInstall(name) {
+  let f = do_get_file(ExtensionsTestPath("/addons/" + name + ".xpi"));
+  let cb = Async.makeSyncCallback();
+  AddonManager.getInstallForFile(f, cb);
+
+  return Async.waitForSyncCallback(cb);
+}
+
+/**
+ * Obtains an addon from the add-on manager by id.
+ *
+ * This is merely a synchronous wrapper.
+ *
+ * @param  id
+ *         ID of add-on to fetch
+ * @return addon object on success or undefined or null on failure
+ */
+function getAddonFromAddonManagerByID(id) {
+   let cb = Async.makeSyncCallback();
+   AddonManager.getAddonByID(id, cb);
+   return Async.waitForSyncCallback(cb);
+}
+
+/**
+ * Installs an add-on synchronously from an addonInstall
+ *
+ * @param  install addonInstall instance to install
+ */
+function installAddonFromInstall(install) {
+  let cb = Async.makeSyncCallback();
+  let listener = {onInstallEnded: cb};
+  AddonManager.addInstallListener(listener);
+  install.install();
+  Async.waitForSyncCallback(cb);
+  AddonManager.removeAddonListener(listener);
+
+  do_check_neq(null, install.addon);
+  do_check_neq(null, install.addon.syncGUID);
+
+  return install.addon;
+}
+
+/**
+ * Convenience function to install an add-on from the extensions unit tests.
+ *
+ * @param  name
+ *         String name of add-on to install. e.g. test_install1
+ * @return addon object that was installed
+ */
+function installAddon(name) {
+  let install = getAddonInstall(name);
+  do_check_neq(null, install);
+  return installAddonFromInstall(install);
+}
+
+/**
+ * Convenience function to uninstall an add-on synchronously.
+ *
+ * @param addon
+ *        Addon instance to uninstall
+ */
+function uninstallAddon(addon) {
+  let cb = Async.makeSyncCallback();
+  let listener = {onUninstalled: function(uninstalled) {
+    if (uninstalled.id == addon.id) {
+      AddonManager.removeAddonListener(listener);
+      cb(uninstalled);
+    }
+  }};
+
+  AddonManager.addAddonListener(listener);
+  addon.uninstall();
+  Async.waitForSyncCallback(cb);
+}
+
+function generateNewKeys(collectionKeys, collections=null) {
+  let wbo = collectionKeys.generateNewKeysWBO(collections);
+  let modified = new_timestamp();
+  collectionKeys.setContents(wbo.cleartext, modified);
+}
+
+// Helpers for testing open tabs.
+// These reflect part of the internal structure of TabEngine,
+// and stub part of Service.wm.
+
+function mockShouldSkipWindow (win) {
+  return win.closed ||
+         win.mockIsPrivate;
+}
+
+function mockGetTabState (tab) {
+  return tab;
+}
+
+function mockGetWindowEnumerator(url, numWindows, numTabs, indexes, moreURLs) {
+  let elements = [];
+
+  function url2entry(url) {
+    return {
+      url: ((typeof url == "function") ? url() : url),
+      title: "title"
+    };
+  }
+
+  for (let w = 0; w < numWindows; ++w) {
+    let tabs = [];
+    let win = {
+      closed: false,
+      mockIsPrivate: false,
+      gBrowser: {
+        tabs: tabs,
+      },
+    };
+    elements.push(win);
+
+    for (let t = 0; t < numTabs; ++t) {
+      tabs.push(TestingUtils.deepCopy({
+        index: indexes ? indexes() : 1,
+        entries: (moreURLs ? [url].concat(moreURLs()) : [url]).map(url2entry),
+        attributes: {
+          image: "image"
+        },
+        lastAccessed: 1499
+      }));
+    }
+  }
+
+  // Always include a closed window and a private window.
+  elements.push({
+    closed: true,
+    mockIsPrivate: false,
+    gBrowser: {
+      tabs: [],
+    },
+  });
+ 
+  elements.push({
+    closed: false,
+    mockIsPrivate: true,
+    gBrowser: {
+      tabs: [],
+    },
+  });
+
+  return {
+    hasMoreElements: function () {
+      return elements.length;
+    },
+    getNext: function () {
+      return elements.shift();
+    },
+  };
+}
+
+// Helper that allows checking array equality.
+function do_check_array_eq(a1, a2) {
+  do_check_eq(a1.length, a2.length);
+  for (let i = 0; i < a1.length; ++i) {
+    do_check_eq(a1[i], a2[i]);
+  }
+}
+
+// Helper function to get the sync telemetry and add the typically used test
+// engine names to its list of allowed engines.
+function get_sync_test_telemetry() {
+  let ns = {};
+  Cu.import("resource://services-sync/telemetry.js", ns);
+  let testEngines = ["rotary", "steam", "sterling", "catapult"];
+  for (let engineName of testEngines) {
+    ns.SyncTelemetry.allowedEngines.add(engineName);
+  }
+  ns.SyncTelemetry.submissionInterval = -1;
+  return ns.SyncTelemetry;
+}
+
+function assert_valid_ping(record) {
+  // This is called as the test harness tears down due to shutdown. This
+  // will typically have no recorded syncs, and the validator complains about
+  // it. So ignore such records (but only ignore when *both* shutdown and
+  // no Syncs - either of them not being true might be an actual problem)
+  if (record && (record.why != "shutdown" || record.syncs.length != 0)) {
+    if (!SyncPingValidator(record)) {
+      deepEqual([], SyncPingValidator.errors, "Sync telemetry ping validation failed");
+    }
+    equal(record.version, 1);
+    record.syncs.forEach(p => {
+      lessOrEqual(p.when, Date.now());
+      if (p.devices) {
+        ok(!p.devices.some(device => device.id == p.deviceID));
+        equal(new Set(p.devices.map(device => device.id)).size,
+              p.devices.length, "Duplicate device ids in ping devices list");
+      }
+    });
+  }
+}
+
+// Asserts that `ping` is a ping that doesn't contain any failure information
+function assert_success_ping(ping) {
+  ok(!!ping);
+  assert_valid_ping(ping);
+  ping.syncs.forEach(record => {
+    ok(!record.failureReason);
+    equal(undefined, record.status);
+    greater(record.engines.length, 0);
+    for (let e of record.engines) {
+      ok(!e.failureReason);
+      equal(undefined, e.status);
+      if (e.validation) {
+        equal(undefined, e.validation.problems);
+        equal(undefined, e.validation.failureReason);
+      }
+      if (e.outgoing) {
+        for (let o of e.outgoing) {
+          equal(undefined, o.failed);
+          notEqual(undefined, o.sent);
+        }
+      }
+      if (e.incoming) {
+        equal(undefined, e.incoming.failed);
+        equal(undefined, e.incoming.newFailed);
+        notEqual(undefined, e.incoming.applied || e.incoming.reconciled);
+      }
+    }
+  });
+}
+
+// Hooks into telemetry to validate all pings after calling.
+function validate_all_future_pings() {
+  let telem = get_sync_test_telemetry();
+  telem.submit = assert_valid_ping;
+}
+
+function wait_for_ping(callback, allowErrorPings, getFullPing = false) {
+  return new Promise(resolve => {
+    let telem = get_sync_test_telemetry();
+    let oldSubmit = telem.submit;
+    telem.submit = function(record) {
+      telem.submit = oldSubmit;
+      if (allowErrorPings) {
+        assert_valid_ping(record);
+      } else {
+        assert_success_ping(record);
+      }
+      if (getFullPing) {
+        resolve(record);
+      } else {
+        equal(record.syncs.length, 1);
+        resolve(record.syncs[0]);
+      }
+    };
+    callback();
+  });
+}
+
+// Short helper for wait_for_ping
+function sync_and_validate_telem(allowErrorPings, getFullPing = false) {
+  return wait_for_ping(() => Service.sync(), allowErrorPings, getFullPing);
+}
+
+// Used for the (many) cases where we do a 'partial' sync, where only a single
+// engine is actually synced, but we still want to ensure we're generating a
+// valid ping. Returns a promise that resolves to the ping, or rejects with the
+// thrown error after calling an optional callback.
+function sync_engine_and_validate_telem(engine, allowErrorPings, onError) {
+  return new Promise((resolve, reject) => {
+    let telem = get_sync_test_telemetry();
+    let caughtError = null;
+    // Clear out status, so failures from previous syncs won't show up in the
+    // telemetry ping.
+    let ns = {};
+    Cu.import("resource://services-sync/status.js", ns);
+    ns.Status._engines = {};
+    ns.Status.partial = false;
+    // Ideally we'd clear these out like we do with engines, (probably via
+    // Status.resetSync()), but this causes *numerous* tests to fail, so we just
+    // assume that if no failureReason or engine failures are set, and the
+    // status properties are the same as they were initially, that it's just
+    // a leftover.
+    // This is only an issue since we're triggering the sync of just one engine,
+    // without doing any other parts of the sync.
+    let initialServiceStatus = ns.Status._service;
+    let initialSyncStatus = ns.Status._sync;
+
+    let oldSubmit = telem.submit;
+    telem.submit = function(ping) {
+      telem.submit = oldSubmit;
+      ping.syncs.forEach(record => {
+        if (record && record.status) {
+          // did we see anything to lead us to believe that something bad actually happened
+          let realProblem = record.failureReason || record.engines.some(e => {
+            if (e.failureReason || e.status) {
+              return true;
+            }
+            if (e.outgoing && e.outgoing.some(o => o.failed > 0)) {
+              return true;
+            }
+            return e.incoming && e.incoming.failed;
+          });
+          if (!realProblem) {
+            // no, so if the status is the same as it was initially, just assume
+            // that its leftover and that we can ignore it.
+            if (record.status.sync && record.status.sync == initialSyncStatus) {
+              delete record.status.sync;
+            }
+            if (record.status.service && record.status.service == initialServiceStatus) {
+              delete record.status.service;
+            }
+            if (!record.status.sync && !record.status.service) {
+              delete record.status;
+            }
+          }
+        }
+      });
+      if (allowErrorPings) {
+        assert_valid_ping(ping);
+      } else {
+        assert_success_ping(ping);
+      }
+      equal(ping.syncs.length, 1);
+      if (caughtError) {
+        if (onError) {
+          onError(ping.syncs[0]);
+        }
+        reject(caughtError);
+      } else {
+        resolve(ping.syncs[0]);
+      }
+    }
+    Svc.Obs.notify("weave:service:sync:start");
+    try {
+      engine.sync();
+    } catch (e) {
+      caughtError = e;
+    }
+    if (caughtError) {
+      Svc.Obs.notify("weave:service:sync:error", caughtError);
+    } else {
+      Svc.Obs.notify("weave:service:sync:finish");
+    }
+  });
+}
+
+// Avoid an issue where `client.name2` containing unicode characters causes
+// a number of tests to fail, due to them assuming that we do not need to utf-8
+// encode or decode data sent through the mocked server (see bug 1268912).
+Utils.getDefaultDeviceName = function() {
+  return "Test device name";
+};
+
+

services/sync/tests/unit/missing-sourceuri.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5617">
+  <name>Restartless Test Extension</name>
+  <type id="1">Extension</type>
+  <guid>missing-sourceuri@tests.mozilla.org</guid>
+  <slug>missing-sourceuri</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>{3e3ba16c-1675-4e88-b9c8-afef81b3d2ef}</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="485"></install>
+  <created epoch="1252903662">
+    2009-09-14T04:47:42Z
+  </created>
+  <last_updated epoch="1315255329">
+    2011-09-05T20:42:09Z
+  </last_updated>
+</addon>
+</searchresults>

services/sync/tests/unit/missing-xpi-search.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5617">
+  <name>Restartless Test Extension</name>
+  <type id="1">Extension</type>
+  <guid>missing-xpi@tests.mozilla.org</guid>
+  <slug>missing-xpi</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>{3e3ba16c-1675-4e88-b9c8-afef81b3d2ef}</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="485">http://127.0.0.1:8888/THIS_DOES_NOT_EXIST.xpi</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/unit/prefs_test_prefs_store.js

@@ -0,0 +1,25 @@
+// This is a "preferences" file used by test_prefs_store.js
+
+// The prefs that control what should be synced.
+// Most of these are "default" prefs, so the value itself will not sync.
+pref("services.sync.prefs.sync.testing.int", true);
+pref("services.sync.prefs.sync.testing.string", true);
+pref("services.sync.prefs.sync.testing.bool", true);
+pref("services.sync.prefs.sync.testing.dont.change", true);
+// this one is a user pref, so it *will* sync.
+user_pref("services.sync.prefs.sync.testing.turned.off", false);
+pref("services.sync.prefs.sync.testing.nonexistent", true);
+pref("services.sync.prefs.sync.testing.default", true);
+
+// The preference values - these are all user_prefs, otherwise their value
+// will not be synced.
+user_pref("testing.int", 123);
+user_pref("testing.string", "ohai");
+user_pref("testing.bool", true);
+user_pref("testing.dont.change", "Please don't change me.");
+user_pref("testing.turned.off", "I won't get synced.");
+user_pref("testing.not.turned.on", "I won't get synced either!");
+
+// A pref that exists but still has the default value - will be synced with
+// null as the value.
+pref("testing.default", "I'm the default value");

services/sync/tests/unit/rewrite-search.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5617">
+  <name>Rewrite Test Extension</name>
+  <type id="1">Extension</type>
+  <guid>rewrite@tests.mozilla.org</guid>
+  <slug>rewrite</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>xpcshell@tests.mozilla.org</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="485">http://127.0.0.1:8888/require.xpi?src=api</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/unit/sync_ping_schema.json

@@ -0,0 +1,198 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "schema for Sync pings, documentation avaliable in toolkit/components/telemetry/docs/sync-ping.rst",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["version", "syncs", "why"],
+  "properties": {
+    "version": { "type": "integer", "minimum": 0 },
+    "discarded": { "type": "integer", "minimum": 1 },
+    "why": { "enum": ["shutdown", "schedule"] },
+    "syncs": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/payload" }
+    }
+  },
+  "definitions": {
+    "payload": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["when", "uid", "took"],
+      "properties": {
+        "didLogin": { "type": "boolean" },
+        "when": { "type": "integer" },
+        "uid": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{32}$"
+        },
+        "devices": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/device" }
+        },
+        "deviceID": {
+          "type": "string",
+          "pattern": "^[0-9a-f]{64}$"
+        },
+        "status": {
+          "type": "object",
+          "anyOf": [
+            { "required": ["sync"] },
+            { "required": ["service"] }
+          ],
+          "additionalProperties": false,
+          "properties": {
+            "sync": { "type": "string" },
+            "service": { "type": "string" }
+          }
+        },
+        "why": { "enum": ["startup", "schedule", "score", "user", "tabs"] },
+        "took": { "type": "integer", "minimum": -1 },
+        "failureReason": { "$ref": "#/definitions/error" },
+        "engines": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/definitions/engine" }
+        }
+      }
+    },
+    "device": {
+      "required": ["os", "id", "version"],
+      "additionalProperties": false,
+      "type": "object",
+      "properties": {
+        "id": { "type": "string", "pattern": "^[0-9a-f]{64}$" },
+        "os": { "type": "string" },
+        "version": { "type": "string" }
+      }
+    },
+    "engine": {
+      "required": ["name"],
+      "additionalProperties": false,
+      "properties": {
+        "failureReason": { "$ref": "#/definitions/error" },
+        "name": { "enum": ["addons", "bookmarks", "clients", "forms", "history", "passwords", "prefs", "tabs"] },
+        "took": { "type": "integer", "minimum": 1 },
+        "status": { "type": "string" },
+        "incoming": {
+          "type": "object",
+          "additionalProperties": false,
+          "anyOf": [
+            {"required": ["applied"]},
+            {"required": ["failed"]},
+            {"required": ["newFailed"]},
+            {"required": ["reconciled"]}
+          ],
+          "properties": {
+            "applied": { "type": "integer", "minimum": 1 },
+            "failed": { "type": "integer", "minimum": 1 },
+            "newFailed": { "type": "integer", "minimum": 1 },
+            "reconciled": { "type": "integer", "minimum": 1 }
+          }
+        },
+        "outgoing": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "$ref": "#/definitions/outgoingBatch" }
+        },
+        "validation": {
+          "type": "object",
+          "additionalProperties": false,
+          "anyOf": [
+            { "required": ["checked"] },
+            { "required": ["failureReason"] }
+          ],
+          "properties": {
+            "checked": { "type": "integer", "minimum": 0 },
+            "failureReason": { "$ref": "#/definitions/error" },
+            "took": { "type": "integer" },
+            "version": { "type": "integer" },
+            "problems": {
+              "type": "array",
+              "minItems": 1,
+              "$ref": "#/definitions/validationProblem"
+            }
+          }
+        }
+      }
+    },
+    "outgoingBatch": {
+      "type": "object",
+      "additionalProperties": false,
+      "anyOf": [
+        {"required": ["sent"]},
+        {"required": ["failed"]}
+      ],
+      "properties": {
+        "sent": { "type": "integer", "minimum": 1 },
+        "failed": { "type": "integer", "minimum": 1 }
+      }
+    },
+    "error": {
+      "oneOf": [
+        { "$ref": "#/definitions/httpError" },
+        { "$ref": "#/definitions/nsError" },
+        { "$ref": "#/definitions/shutdownError" },
+        { "$ref": "#/definitions/authError" },
+        { "$ref": "#/definitions/otherError" },
+        { "$ref": "#/definitions/unexpectedError" },
+        { "$ref": "#/definitions/sqlError" }
+      ]
+    },
+    "httpError": {
+      "required": ["name", "code"],
+      "properties": {
+        "name": { "enum": ["httperror"] },
+        "code": { "type": "integer" }
+      }
+    },
+    "nsError": {
+      "required": ["name", "code"],
+      "properties": {
+        "name": { "enum": ["nserror"] },
+        "code": { "type": "integer" }
+      }
+    },
+    "shutdownError": {
+      "required": ["name"],
+      "properties": {
+        "name": { "enum": ["shutdownerror"] }
+      }
+    },
+    "authError": {
+      "required": ["name"],
+      "properties": {
+        "name": { "enum": ["autherror"] },
+        "from": { "enum": ["tokenserver", "fxaccounts", "hawkclient"] }
+      }
+    },
+    "otherError": {
+      "required": ["name"],
+      "properties": {
+        "name": { "enum": ["othererror"] },
+        "error": { "type": "string" }
+      }
+    },
+    "unexpectedError": {
+      "required": ["name"],
+      "properties": {
+        "name": { "enum": ["unexpectederror"] },
+        "error": { "type": "string" }
+      }
+    },
+    "sqlError": {
+      "required": ["name"],
+      "properties": {
+        "name": { "enum": ["sqlerror"] },
+        "code": { "type": "integer" }
+      }
+    },
+    "validationProblem": {
+      "required": ["name", "count"],
+      "properties": {
+        "name": { "type": "string" },
+        "count": { "type": "integer" }
+      }
+    }
+  }
+}

services/sync/tests/unit/systemaddon-search.xml

@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<searchresults total_results="1">
+  <addon id="5618">
+  <name>System Add-on Test</name>
+  <type id="1">Extension</type>
+  <guid>system1@tests.mozilla.org</guid>
+  <slug>addon11</slug>
+  <version>1.0</version>
+
+  <compatible_applications><application>
+      <name>Firefox</name>
+      <application_id>1</application_id>
+      <min_version>3.6</min_version>
+      <max_version>*</max_version>
+      <appID>xpcshell@tests.mozilla.org</appID>
+    </application></compatible_applications>
+  <all_compatible_os><os>ALL</os></all_compatible_os>
+
+  <install os="ALL" size="999">http://127.0.0.1:8888/system.xpi</install>
+    <created epoch="1252903662">
+      2009-09-14T04:47:42Z
+    </created>
+    <last_updated epoch="1315255329">
+      2011-09-05T20:42:09Z
+    </last_updated>
+    </addon>
+</searchresults>

services/sync/tests/unit/test_addon_utils.js

@@ -0,0 +1,141 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://services-sync/addonutils.js");
+Cu.import("resource://services-sync/util.js");
+
+const HTTP_PORT = 8888;
+const SERVER_ADDRESS = "http://127.0.0.1:8888";
+
+var prefs = new Preferences();
+
+prefs.set("extensions.getAddons.get.url",
+          SERVER_ADDRESS + "/search/guid:%IDS%");
+
+loadAddonTestFunctions();
+startupManager();
+
+function createAndStartHTTPServer(port=HTTP_PORT) {
+  try {
+    let server = new HttpServer();
+
+    let bootstrap1XPI = ExtensionsTestPath("/addons/test_bootstrap1_1.xpi");
+
+    server.registerFile("/search/guid:missing-sourceuri%40tests.mozilla.org",
+                        do_get_file("missing-sourceuri.xml"));
+
+    server.registerFile("/search/guid:rewrite%40tests.mozilla.org",
+                        do_get_file("rewrite-search.xml"));
+
+    server.start(port);
+
+    return server;
+  } catch (ex) {
+    _("Got exception starting HTTP server on port " + port);
+    _("Error: " + Log.exceptionStr(ex));
+    do_throw(ex);
+  }
+}
+
+function run_test() {
+  initTestLogging("Trace");
+
+  run_next_test();
+}
+
+add_test(function test_handle_empty_source_uri() {
+  _("Ensure that search results without a sourceURI are properly ignored.");
+
+  let server = createAndStartHTTPServer();
+
+  const ID = "missing-sourceuri@tests.mozilla.org";
+
+  let cb = Async.makeSpinningCallback();
+  AddonUtils.installAddons([{id: ID, requireSecureURI: false}], cb);
+  let result = cb.wait();
+
+  do_check_true("installedIDs" in result);
+  do_check_eq(0, result.installedIDs.length);
+
+  do_check_true("skipped" in result);
+  do_check_true(result.skipped.includes(ID));
+
+  server.stop(run_next_test);
+});
+
+add_test(function test_ignore_untrusted_source_uris() {
+  _("Ensures that source URIs from insecure schemes are rejected.");
+
+  let ioService = Cc["@mozilla.org/network/io-service;1"]
+                  .getService(Ci.nsIIOService);
+
+  const bad = ["http://example.com/foo.xpi",
+               "ftp://example.com/foo.xpi",
+               "silly://example.com/foo.xpi"];
+
+  const good = ["https://example.com/foo.xpi"];
+
+  for (let s of bad) {
+    let sourceURI = ioService.newURI(s, null, null);
+    let addon = {sourceURI: sourceURI, name: "bad", id: "bad"};
+
+    let canInstall = AddonUtils.canInstallAddon(addon);
+    do_check_false(canInstall, "Correctly rejected a bad URL");
+  }
+
+  for (let s of good) {
+    let sourceURI = ioService.newURI(s, null, null);
+    let addon = {sourceURI: sourceURI, name: "good", id: "good"};
+
+    let canInstall = AddonUtils.canInstallAddon(addon);
+    do_check_true(canInstall, "Correctly accepted a good URL");
+  }
+  run_next_test();
+});
+
+add_test(function test_source_uri_rewrite() {
+  _("Ensure that a 'src=api' query string is rewritten to 'src=sync'");
+
+  // This tests for conformance with bug 708134 so server-side metrics aren't
+  // skewed.
+
+  // We resort to monkeypatching because of the API design.
+  let oldFunction = AddonUtils.__proto__.installAddonFromSearchResult;
+
+  let installCalled = false;
+  AddonUtils.__proto__.installAddonFromSearchResult =
+    function testInstallAddon(addon, metadata, cb) {
+
+    do_check_eq(SERVER_ADDRESS + "/require.xpi?src=sync",
+                addon.sourceURI.spec);
+
+    installCalled = true;
+
+    AddonUtils.getInstallFromSearchResult(addon, function (error, install) {
+      do_check_null(error);
+      do_check_eq(SERVER_ADDRESS + "/require.xpi?src=sync",
+                  install.sourceURI.spec);
+
+      cb(null, {id: addon.id, addon: addon, install: install});
+    }, false);
+  };
+
+  let server = createAndStartHTTPServer();
+
+  let installCallback = Async.makeSpinningCallback();
+  let installOptions = {
+    id: "rewrite@tests.mozilla.org",
+    requireSecureURI: false,
+  }
+  AddonUtils.installAddons([installOptions], installCallback);
+
+  installCallback.wait();
+  do_check_true(installCalled);
+  AddonUtils.__proto__.installAddonFromSearchResult = oldFunction;
+
+  server.stop(run_next_test);
+});

services/sync/tests/unit/test_addons_engine.js

@@ -0,0 +1,253 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/AddonManager.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-sync/addonsreconciler.js");
+Cu.import("resource://services-sync/engines/addons.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+
+var prefs = new Preferences();
+prefs.set("extensions.getAddons.get.url",
+          "http://localhost:8888/search/guid:%IDS%");
+prefs.set("extensions.install.requireSecureOrigin", false);
+
+loadAddonTestFunctions();
+startupManager();
+
+var engineManager = Service.engineManager;
+
+engineManager.register(AddonsEngine);
+var engine = engineManager.get("addons");
+var reconciler = engine._reconciler;
+var tracker = engine._tracker;
+
+function advance_test() {
+  reconciler._addons = {};
+  reconciler._changes = [];
+
+  let cb = Async.makeSpinningCallback();
+  reconciler.saveState(null, cb);
+  cb.wait();
+
+  run_next_test();
+}
+
+// This is a basic sanity test for the unit test itself. If this breaks, the
+// add-ons API likely changed upstream.
+add_test(function test_addon_install() {
+  _("Ensure basic add-on APIs work as expected.");
+
+  let install = getAddonInstall("test_bootstrap1_1");
+  do_check_neq(install, null);
+  do_check_eq(install.type, "extension");
+  do_check_eq(install.name, "Test Bootstrap 1");
+
+  advance_test();
+});
+
+add_test(function test_find_dupe() {
+  _("Ensure the _findDupe() implementation is sane.");
+
+  // This gets invoked at the top of sync, which is bypassed by this
+  // test, so we do it manually.
+  engine._refreshReconcilerState();
+
+  let addon = installAddon("test_bootstrap1_1");
+
+  let record = {
+    id:            Utils.makeGUID(),
+    addonID:       addon.id,
+    enabled:       true,
+    applicationID: Services.appinfo.ID,
+    source:        "amo"
+  };
+
+  let dupe = engine._findDupe(record);
+  do_check_eq(addon.syncGUID, dupe);
+
+  record.id = addon.syncGUID;
+  dupe = engine._findDupe(record);
+  do_check_eq(null, dupe);
+
+  uninstallAddon(addon);
+  advance_test();
+});
+
+add_test(function test_get_changed_ids() {
+  _("Ensure getChangedIDs() has the appropriate behavior.");
+
+  _("Ensure getChangedIDs() returns an empty object by default.");
+  let changes = engine.getChangedIDs();
+  do_check_eq("object", typeof(changes));
+  do_check_eq(0, Object.keys(changes).length);
+
+  _("Ensure tracker changes are populated.");
+  let now = new Date();
+  let changeTime = now.getTime() / 1000;
+  let guid1 = Utils.makeGUID();
+  tracker.addChangedID(guid1, changeTime);
+
+  changes = engine.getChangedIDs();
+  do_check_eq("object", typeof(changes));
+  do_check_eq(1, Object.keys(changes).length);
+  do_check_true(guid1 in changes);
+  do_check_eq(changeTime, changes[guid1]);
+
+  tracker.clearChangedIDs();
+
+  _("Ensure reconciler changes are populated.");
+  let addon = installAddon("test_bootstrap1_1");
+  tracker.clearChangedIDs(); // Just in case.
+  changes = engine.getChangedIDs();
+  do_check_eq("object", typeof(changes));
+  do_check_eq(1, Object.keys(changes).length);
+  do_check_true(addon.syncGUID in changes);
+  _("Change time: " + changeTime + ", addon change: " + changes[addon.syncGUID]);
+  do_check_true(changes[addon.syncGUID] >= changeTime);
+
+  let oldTime = changes[addon.syncGUID];
+  let guid2 = addon.syncGUID;
+  uninstallAddon(addon);
+  changes = engine.getChangedIDs();
+  do_check_eq(1, Object.keys(changes).length);
+  do_check_true(guid2 in changes);
+  do_check_true(changes[guid2] > oldTime);
+
+  _("Ensure non-syncable add-ons aren't picked up by reconciler changes.");
+  reconciler._addons  = {};
+  reconciler._changes = [];
+  let record = {
+    id:             "DUMMY",
+    guid:           Utils.makeGUID(),
+    enabled:        true,
+    installed:      true,
+    modified:       new Date(),
+    type:           "UNSUPPORTED",
+    scope:          0,
+    foreignInstall: false
+  };
+  reconciler.addons["DUMMY"] = record;
+  reconciler._addChange(record.modified, CHANGE_INSTALLED, record);
+
+  changes = engine.getChangedIDs();
+  _(JSON.stringify(changes));
+  do_check_eq(0, Object.keys(changes).length);
+
+  advance_test();
+});
+
+add_test(function test_disabled_install_semantics() {
+  _("Ensure that syncing a disabled add-on preserves proper state.");
+
+  // This is essentially a test for bug 712542, which snuck into the original
+  // add-on sync drop. It ensures that when an add-on is installed that the
+  // disabled state and incoming syncGUID is preserved, even on the next sync.
+  const USER       = "foo";
+  const PASSWORD   = "password";
+  const PASSPHRASE = "abcdeabcdeabcdeabcdeabcdea";
+  const ADDON_ID   = "addon1@tests.mozilla.org";
+
+  let server = new SyncServer();
+  server.start();
+  new SyncTestingInfrastructure(server.server, USER, PASSWORD, PASSPHRASE);
+
+  generateNewKeys(Service.collectionKeys);
+
+  let contents = {
+    meta: {global: {engines: {addons: {version: engine.version,
+                                      syncID:  engine.syncID}}}},
+    crypto: {},
+    addons: {}
+  };
+
+  server.registerUser(USER, "password");
+  server.createContents(USER, contents);
+
+  let amoServer = new HttpServer();
+  amoServer.registerFile("/search/guid:addon1%40tests.mozilla.org",
+                         do_get_file("addon1-search.xml"));
+
+  let installXPI = ExtensionsTestPath("/addons/test_install1.xpi");
+  amoServer.registerFile("/addon1.xpi", do_get_file(installXPI));
+  amoServer.start(8888);
+
+  // Insert an existing record into the server.
+  let id = Utils.makeGUID();
+  let now = Date.now() / 1000;
+
+  let record = encryptPayload({
+    id:            id,
+    applicationID: Services.appinfo.ID,
+    addonID:       ADDON_ID,
+    enabled:       false,
+    deleted:       false,
+    source:        "amo",
+  });
+  let wbo = new ServerWBO(id, record, now - 2);
+  server.insertWBO(USER, "addons", wbo);
+
+  _("Performing sync of add-ons engine.");
+  engine._sync();
+
+  // At this point the non-restartless extension should be staged for install.
+
+  // Don't need this server any more.
+  let cb = Async.makeSpinningCallback();
+  amoServer.stop(cb);
+  cb.wait();
+
+  // We ensure the reconciler has recorded the proper ID and enabled state.
+  let addon = reconciler.getAddonStateFromSyncGUID(id);
+  do_check_neq(null, addon);
+  do_check_eq(false, addon.enabled);
+
+  // We fake an app restart and perform another sync, just to make sure things
+  // are sane.
+  restartManager();
+
+  engine._sync();
+
+  // The client should not upload a new record. The old record should be
+  // retained and unmodified.
+  let collection = server.getCollection(USER, "addons");
+  do_check_eq(1, collection.count());
+
+  let payload = collection.payloads()[0];
+  do_check_neq(null, collection.wbo(id));
+  do_check_eq(ADDON_ID, payload.addonID);
+  do_check_false(payload.enabled);
+
+  server.stop(advance_test);
+});
+
+add_test(function cleanup() {
+  // There's an xpcom-shutdown hook for this, but let's give this a shot.
+  reconciler.stopListening();
+  run_next_test();
+});
+
+function run_test() {
+  initTestLogging("Trace");
+  Log.repository.getLogger("Sync.Engine.Addons").level =
+    Log.Level.Trace;
+  Log.repository.getLogger("Sync.Store.Addons").level = Log.Level.Trace;
+  Log.repository.getLogger("Sync.Tracker.Addons").level =
+    Log.Level.Trace;
+  Log.repository.getLogger("Sync.AddonsRepository").level =
+    Log.Level.Trace;
+
+  reconciler.startListening();
+
+  // Don't flush to disk in the middle of an event listener!
+  // This causes test hangs on WinXP.
+  reconciler._shouldPersist = false;
+
+  advance_test();
+}

services/sync/tests/unit/test_addons_reconciler.js

@@ -0,0 +1,195 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/AddonManager.jsm");
+Cu.import("resource://services-sync/addonsreconciler.js");
+Cu.import("resource://services-sync/engines/addons.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+
+loadAddonTestFunctions();
+startupManager();
+
+function run_test() {
+  initTestLogging("Trace");
+  Log.repository.getLogger("Sync.AddonsReconciler").level = Log.Level.Trace;
+  Log.repository.getLogger("Sync.AddonsReconciler").level =
+    Log.Level.Trace;
+
+  Svc.Prefs.set("engine.addons", true);
+  Service.engineManager.register(AddonsEngine);
+
+  run_next_test();
+}
+
+add_test(function test_defaults() {
+  _("Ensure new objects have reasonable defaults.");
+
+  let reconciler = new AddonsReconciler();
+
+  do_check_false(reconciler._listening);
+  do_check_eq("object", typeof(reconciler.addons));
+  do_check_eq(0, Object.keys(reconciler.addons).length);
+  do_check_eq(0, reconciler._changes.length);
+  do_check_eq(0, reconciler._listeners.length);
+
+  run_next_test();
+});
+
+add_test(function test_load_state_empty_file() {
+  _("Ensure loading from a missing file results in defaults being set.");
+
+  let reconciler = new AddonsReconciler();
+
+  reconciler.loadState(null, function(error, loaded) {
+    do_check_eq(null, error);
+    do_check_false(loaded);
+
+    do_check_eq("object", typeof(reconciler.addons));
+    do_check_eq(0, Object.keys(reconciler.addons).length);
+    do_check_eq(0, reconciler._changes.length);
+
+    run_next_test();
+  });
+});
+
+add_test(function test_install_detection() {
+  _("Ensure that add-on installation results in appropriate side-effects.");
+
+  let reconciler = new AddonsReconciler();
+  reconciler.startListening();
+
+  let before = new Date();
+  let addon = installAddon("test_bootstrap1_1");
+  let after = new Date();
+
+  do_check_eq(1, Object.keys(reconciler.addons).length);
+  do_check_true(addon.id in reconciler.addons);
+  let record = reconciler.addons[addon.id];
+
+  const KEYS = ["id", "guid", "enabled", "installed", "modified", "type",
+                "scope", "foreignInstall"];
+  for (let key of KEYS) {
+    do_check_true(key in record);
+    do_check_neq(null, record[key]);
+  }
+
+  do_check_eq(addon.id, record.id);
+  do_check_eq(addon.syncGUID, record.guid);
+  do_check_true(record.enabled);
+  do_check_true(record.installed);
+  do_check_true(record.modified >= before && record.modified <= after);
+  do_check_eq("extension", record.type);
+  do_check_false(record.foreignInstall);
+
+  do_check_eq(1, reconciler._changes.length);
+  let change = reconciler._changes[0];
+  do_check_true(change[0] >= before && change[1] <= after);
+  do_check_eq(CHANGE_INSTALLED, change[1]);
+  do_check_eq(addon.id, change[2]);
+
+  uninstallAddon(addon);
+
+  run_next_test();
+});
+
+add_test(function test_uninstall_detection() {
+  _("Ensure that add-on uninstallation results in appropriate side-effects.");
+
+  let reconciler = new AddonsReconciler();
+  reconciler.startListening();
+
+  reconciler._addons = {};
+  reconciler._changes = [];
+
+  let addon = installAddon("test_bootstrap1_1");
+  let id = addon.id;
+  let guid = addon.syncGUID;
+
+  reconciler._changes = [];
+  uninstallAddon(addon);
+
+  do_check_eq(1, Object.keys(reconciler.addons).length);
+  do_check_true(id in reconciler.addons);
+
+  let record = reconciler.addons[id];
+  do_check_false(record.installed);
+
+  do_check_eq(1, reconciler._changes.length);
+  let change = reconciler._changes[0];
+  do_check_eq(CHANGE_UNINSTALLED, change[1]);
+  do_check_eq(id, change[2]);
+
+  run_next_test();
+});
+
+add_test(function test_load_state_future_version() {
+  _("Ensure loading a file from a future version results in no data loaded.");
+
+  const FILENAME = "TEST_LOAD_STATE_FUTURE_VERSION";
+
+  let reconciler = new AddonsReconciler();
+
+  // First we populate our new file.
+  let state = {version: 100, addons: {foo: {}}, changes: [[1, 1, "foo"]]};
+  let cb = Async.makeSyncCallback();
+
+  // jsonSave() expects an object with ._log, so we give it a reconciler
+  // instance.
+  Utils.jsonSave(FILENAME, reconciler, state, cb);
+  Async.waitForSyncCallback(cb);
+
+  reconciler.loadState(FILENAME, function(error, loaded) {
+    do_check_eq(null, error);
+    do_check_false(loaded);
+
+    do_check_eq("object", typeof(reconciler.addons));
+    do_check_eq(1, Object.keys(reconciler.addons).length);
+    do_check_eq(1, reconciler._changes.length);
+
+    run_next_test();
+  });
+});
+
+add_test(function test_prune_changes_before_date() {
+  _("Ensure that old changes are pruned properly.");
+
+  let reconciler = new AddonsReconciler();
+  reconciler._ensureStateLoaded();
+  reconciler._changes = [];
+
+  let now = new Date();
+  const HOUR_MS = 1000 * 60 * 60;
+
+  _("Ensure pruning an empty changes array works.");
+  reconciler.pruneChangesBeforeDate(now);
+  do_check_eq(0, reconciler._changes.length);
+
+  let old = new Date(now.getTime() - HOUR_MS);
+  let young = new Date(now.getTime() - 1000);
+  reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]);
+  reconciler._changes.push([young, CHANGE_INSTALLED, "bar"]);
+  do_check_eq(2, reconciler._changes.length);
+
+  _("Ensure pruning with an old time won't delete anything.");
+  let threshold = new Date(old.getTime() - 1);
+  reconciler.pruneChangesBeforeDate(threshold);
+  do_check_eq(2, reconciler._changes.length);
+
+  _("Ensure pruning a single item works.");
+  threshold = new Date(young.getTime() - 1000);
+  reconciler.pruneChangesBeforeDate(threshold);
+  do_check_eq(1, reconciler._changes.length);
+  do_check_neq(undefined, reconciler._changes[0]);
+  do_check_eq(young, reconciler._changes[0][0]);
+  do_check_eq("bar", reconciler._changes[0][2]);
+
+  _("Ensure pruning all changes works.");
+  reconciler._changes.push([old, CHANGE_INSTALLED, "foo"]);
+  reconciler.pruneChangesBeforeDate(now);
+  do_check_eq(0, reconciler._changes.length);
+
+  run_next_test();
+});