wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-10 02:08:38 +09:00
Code Issues Projects Releases Packages Wiki Activity

Import Tycho weave client

Browse source
This commit is contained in:
wolfbeast 2018-10-06 06:57:51 +02:00 • committed by Roy Tam
commit c53787dfdb
183 changed files with 7272 additions and 16759 deletions
Download patch file Download diff file Expand all files Collapse all files

546
services/sync/modules/FxaMigrator.jsm Normal file
View file

@ -0,0 +1,546 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict;"
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/Task.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
"resource://gre/modules/FxAccounts.jsm");
XPCOMUtils.defineLazyGetter(this, "WeaveService", function() {
return Cc["@mozilla.org/weave/service;1"]
.getService(Components.interfaces.nsISupports)
.wrappedJSObject;
});
XPCOMUtils.defineLazyModuleGetter(this, "Weave",
"resource://services-sync/main.js");
// FxAccountsCommon.js doesn't use a "namespace", so create one here.
let fxAccountsCommon = {};
Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
// We send this notification whenever the "user" migration state changes.
const OBSERVER_STATE_CHANGE_TOPIC = "fxa-migration:state-changed";
// We also send the state notification when we *receive* this. This allows
// consumers to avoid loading this module until it receives a notification
// from us (which may never happen if there's no migration to do)
const OBSERVER_STATE_REQUEST_TOPIC = "fxa-migration:state-request";
// We send this notification whenever the migration is paused waiting for
// something internal to complete.
const OBSERVER_INTERNAL_STATE_CHANGE_TOPIC = "fxa-migration:internal-state-changed";
// We use this notification so Sync's healthreport module can record telemetry
// (actually via "health report") for us.
const OBSERVER_INTERNAL_TELEMETRY_TOPIC = "fxa-migration:internal-telemetry";
const OBSERVER_TOPICS = [
"xpcom-shutdown",
"weave:service:sync:start",
"weave:service:sync:finish",
"weave:service:sync:error",
"weave:eol",
OBSERVER_STATE_REQUEST_TOPIC,
fxAccountsCommon.ONLOGIN_NOTIFICATION,
fxAccountsCommon.ONLOGOUT_NOTIFICATION,
fxAccountsCommon.ONVERIFIED_NOTIFICATION,
];
// A list of preference names we write to the migration sentinel. We only
// write ones that have a user-set value.
const FXA_SENTINEL_PREFS = [
"identity.fxaccounts.auth.uri",
"identity.fxaccounts.remote.force_auth.uri",
"identity.fxaccounts.remote.signup.uri",
"identity.fxaccounts.remote.signin.uri",
"identity.fxaccounts.settings.uri",
"services.sync.tokenServerURI",
];
function Migrator() {
// Leave the log-level as Debug - Sync will setup log appenders such that
// these messages generally will not be seen unless other log related
// prefs are set.
this.log.level = Log.Level.Debug;
this._nextUserStatePromise = Promise.resolve();
for (let topic of OBSERVER_TOPICS) {
Services.obs.addObserver(this, topic, false);
}
// ._state is an optimization so we avoid sending redundant observer
// notifications when the state hasn't actually changed.
this._state = null;
}
Migrator.prototype = {
log: Log.repository.getLogger("Sync.SyncMigration"),
// What user action is necessary to push the migration forward?
// A |null| state means there is nothing to do. Note that a null state implies
// either. (a) no migration is necessary or (b) that the migrator module is
// waiting for something outside of the user's control - eg, sync to complete,
// the migration sentinel to be uploaded, etc. In most cases the wait will be
// short, but edge cases (eg, no network, sync bugs that prevent it stopping
// until shutdown) may require a significantly longer wait.
STATE_USER_FXA: "waiting for user to be signed in to FxA",
STATE_USER_FXA_VERIFIED: "waiting for a verified FxA user",
// What internal state are we at? This is primarily used for FHR reporting so
// we can determine why exactly we might be stalled.
STATE_INTERNAL_WAITING_SYNC_COMPLETE: "waiting for sync to complete",
STATE_INTERNAL_WAITING_WRITE_SENTINEL: "waiting for sentinel to be written",
STATE_INTERNAL_WAITING_START_OVER: "waiting for sync to reset itself",
STATE_INTERNAL_COMPLETE: "migration complete",
// Flags for the telemetry we record. The UI will call a helper to record
// the fact some UI was interacted with.
TELEMETRY_ACCEPTED: "accepted",
TELEMETRY_DECLINED: "declined",
TELEMETRY_UNLINKED: "unlinked",
finalize() {
for (let topic of OBSERVER_TOPICS) {
Services.obs.removeObserver(this, topic);
}
},
observe(subject, topic, data) {
this.log.debug("observed " + topic);
switch (topic) {
case "xpcom-shutdown":
this.finalize();
break;
case OBSERVER_STATE_REQUEST_TOPIC:
// someone has requested the state - send it.
this._queueCurrentUserState(true);
break;
default:
// some other observer that may affect our state has fired, so update.
this._queueCurrentUserState().then(
() => this.log.debug("update state from observer " + topic + " complete")
).catch(err => {
let msg = "Failed to handle topic " + topic + ": " + err;
Cu.reportError(msg);
this.log.error(msg);
});
}
},
// Try and move to a state where we are blocked on a user action.
// This needs to be restartable, and the states may, in edge-cases, end
// up going backwards (eg, user logs out while we are waiting to be told
// about verification)
// This is called by our observer notifications - so if there is already
// a promise in-flight, it's possible we will miss something important - so
// we wait for the in-flight one to complete then fire another (ie, this
// is effectively a queue of promises)
_queueCurrentUserState(forceObserver = false) {
return this._nextUserStatePromise = this._nextUserStatePromise.then(
() => this._promiseCurrentUserState(forceObserver),
err => {
let msg = "Failed to determine the current user state: " + err;
Cu.reportError(msg);
this.log.error(msg);
return this._promiseCurrentUserState(forceObserver)
}
);
},
_promiseCurrentUserState: Task.async(function* (forceObserver) {
this.log.trace("starting _promiseCurrentUserState");
let update = (newState, email=null) => {
this.log.info("Migration state: '${state}' => '${newState}'",
{state: this._state, newState: newState});
if (forceObserver || newState !== this._state) {
this._state = newState;
let subject = Cc["@mozilla.org/supports-string;1"]
.createInstance(Ci.nsISupportsString);
subject.data = email || "";
Services.obs.notifyObservers(subject, OBSERVER_STATE_CHANGE_TOPIC, newState);
}
return newState;
}
// If we have no sync user, or are already using an FxA account we must
// be done.
if (WeaveService.fxAccountsEnabled) {
// should not be necessary, but if we somehow ended up with FxA enabled
// and sync blocked it would be bad - so better safe than sorry.
this.log.debug("FxA enabled - there's nothing to do!")
this._unblockSync();
return update(null);
}
// so we need to migrate - let's see how far along we are.
// If sync isn't in EOL mode, then we are still waiting for the server
// to offer the migration process - so no user action necessary.
let isEOL = false;
try {
isEOL = !!Services.prefs.getCharPref("services.sync.errorhandler.alert.mode");
} catch (e) {}
if (!isEOL) {
return update(null);
}
// So we are in EOL mode - have we a user?
let fxauser = yield fxAccounts.getSignedInUser();
if (!fxauser) {
// See if there is a migration sentinel so we can send the email
// address that was used on a different device for this account (ie, if
// this is a "join the party" migration rather than the first)
let sentinel = yield this._getSyncMigrationSentinel();
return update(this.STATE_USER_FXA, sentinel && sentinel.email);
}
if (!fxauser.verified) {
return update(this.STATE_USER_FXA_VERIFIED, fxauser.email);
}
// So we just have housekeeping to do - we aren't blocked on a user, so
// reflect that.
this.log.info("No next user state - doing some housekeeping");
update(null);
// We need to disable sync from automatically starting,
// and if we are currently syncing wait for it to complete.
this._blockSync();
// Are we currently syncing?
if (Weave.Service._locked) {
// our observers will kick us further along when complete.
this.log.info("waiting for sync to complete")
Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
this.STATE_INTERNAL_WAITING_SYNC_COMPLETE);
return null;
}
// Write the migration sentinel if necessary.
Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
this.STATE_INTERNAL_WAITING_WRITE_SENTINEL);
yield this._setMigrationSentinelIfNecessary();
// Get the list of enabled engines to we can restore that state.
let enginePrefs = this._getEngineEnabledPrefs();
// Must be ready to perform the actual migration.
this.log.info("Performing final sync migration steps");
// Do the actual migration. We setup one observer for when the new identity
// is about to be initialized so we can reset some key preferences - but
// there's no promise associated with this.
let observeStartOverIdentity;
Services.obs.addObserver(observeStartOverIdentity = () => {
this.log.info("observed that startOver is about to re-initialize the identity");
Services.obs.removeObserver(observeStartOverIdentity, "weave:service:start-over:init-identity");
// We've now reset all sync prefs - set the engine related prefs back to
// what they were.
for (let [prefName, prefType, prefVal] of enginePrefs) {
this.log.debug("Restoring pref ${prefName} (type=${prefType}) to ${prefVal}",
{prefName, prefType, prefVal});
switch (prefType) {
case Services.prefs.PREF_BOOL:
Services.prefs.setBoolPref(prefName, prefVal);
break;
case Services.prefs.PREF_STRING:
Services.prefs.setCharPref(prefName, prefVal);
break;
default:
// _getEngineEnabledPrefs doesn't return any other type...
Cu.reportError("unknown engine pref type for " + prefName + ": " + prefType);
}
}
}, "weave:service:start-over:init-identity", false);
// And another observer for the startOver being fully complete - the only
// reason for this is so we can wait until everything is fully reset.
let startOverComplete = new Promise((resolve, reject) => {
let observe;
Services.obs.addObserver(observe = () => {
this.log.info("observed that startOver is complete");
Services.obs.removeObserver(observe, "weave:service:start-over:finish");
resolve();
}, "weave:service:start-over:finish", false);
});
Weave.Service.startOver();
// need to wait for an observer.
Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
this.STATE_INTERNAL_WAITING_START_OVER);
yield startOverComplete;
// observer fired, now kick things off with the FxA user.
this.log.info("scheduling initial FxA sync.");
// Note we technically don't need to unblockSync as by now all sync prefs
// have been reset - but it doesn't hurt.
this._unblockSync();
Weave.Service.scheduler.scheduleNextSync(0);
// Tell the front end that migration is now complete -- Sync is now
// configured with an FxA user.
forceObserver = true;
this.log.info("Migration complete");
update(null);
Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
this.STATE_INTERNAL_COMPLETE);
return null;
}),
/* Return an object with the preferences we care about */
_getSentinelPrefs() {
let result = {};
for (let pref of FXA_SENTINEL_PREFS) {
if (Services.prefs.prefHasUserValue(pref)) {
result[pref] = Services.prefs.getCharPref(pref);
}
}
return result;
},
/* Apply any preferences we've obtained from the sentinel */
_applySentinelPrefs(savedPrefs) {
for (let pref of FXA_SENTINEL_PREFS) {
if (savedPrefs[pref]) {
Services.prefs.setCharPref(pref, savedPrefs[pref]);
}
}
},
/* Ask sync to upload the migration sentinel */
_setSyncMigrationSentinel: Task.async(function* () {
yield WeaveService.whenLoaded();
let signedInUser = yield fxAccounts.getSignedInUser();
let sentinel = {
email: signedInUser.email,
uid: signedInUser.uid,
verified: signedInUser.verified,
prefs: this._getSentinelPrefs(),
};
yield Weave.Service.setFxAMigrationSentinel(sentinel);
}),
/* Ask sync to upload the migration sentinal if we (or any other linked device)
haven't previously written one.
*/
_setMigrationSentinelIfNecessary: Task.async(function* () {
if (!(yield this._getSyncMigrationSentinel())) {
this.log.info("writing the migration sentinel");
yield this._setSyncMigrationSentinel();
}
}),
/* Ask sync to return a migration sentinel if one exists, otherwise return null */
_getSyncMigrationSentinel: Task.async(function* () {
yield WeaveService.whenLoaded();
let sentinel = yield Weave.Service.getFxAMigrationSentinel();
this.log.debug("got migration sentinel ${}", sentinel);
return sentinel;
}),
_getDefaultAccountName: Task.async(function* (sentinel) {
// Requires looking to see if other devices have written a migration
// sentinel (eg, see _haveSynchedMigrationSentinel), and if not, see if
// the legacy account name appears to be a valid email address (via the
// services.sync.account pref), otherwise return null.
// NOTE: Sync does all this synchronously via nested event loops, but we
// expose a promise to make future migration to an async-sync easier.
if (sentinel && sentinel.email) {
this.log.info("defaultAccountName found via sentinel: ${}", sentinel.email);
return sentinel.email;
}
// No previous migrations, so check the existing account name.
let account = Weave.Service.identity.account;
if (account && account.contains("@")) {
this.log.info("defaultAccountName found via legacy account name: {}", account);
return account;
}
this.log.info("defaultAccountName could not find an account");
return null;
}),
// Prevent sync from automatically starting
_blockSync() {
Weave.Service.scheduler.blockSync();
},
_unblockSync() {
Weave.Service.scheduler.unblockSync();
},
/* Return a list of [prefName, prefType, prefVal] for all engine related
preferences.
*/
_getEngineEnabledPrefs() {
let result = [];
for (let engine of Weave.Service.engineManager.getAll()) {
let prefName = "services.sync.engine." + engine.prefName;
let prefVal;
try {
prefVal = Services.prefs.getBoolPref(prefName);
result.push([prefName, Services.prefs.PREF_BOOL, prefVal]);
} catch (ex) {} /* just skip this pref */
}
// and the declined list.
try {
let prefName = "services.sync.declinedEngines";
let prefVal = Services.prefs.getCharPref(prefName);
result.push([prefName, Services.prefs.PREF_STRING, prefVal]);
} catch (ex) {}
return result;
},
/* return true if all engines are enabled, false otherwise. */
_allEnginesEnabled() {
return Weave.Service.engineManager.getAll().every(e => e.enabled);
},
/*
* Some helpers for the UI to try and move to the next state.
*/
// Open a UI for the user to create a Firefox Account. This should only be
// called while we are in the STATE_USER_FXA state. When the user completes
// the creation we'll see an ONLOGIN_NOTIFICATION notification from FxA and
// we'll move to either the STATE_USER_FXA_VERIFIED state or we'll just
// complete the migration if they login as an already verified user.
createFxAccount: Task.async(function* (win) {
let {url, options} = yield this.getFxAccountCreationOptions();
win.switchToTabHavingURI(url, true, options);
// An FxA observer will fire when the user completes this, which will
// cause us to move to the next "user blocked" state and notify via our
// observer notification.
}),
// Returns an object with properties "url" and "options", suitable for
// opening FxAccounts to create/signin to FxA suitable for the migration
// state. The caller of this is responsible for the actual opening of the
// page.
// This should only be called while we are in the STATE_USER_FXA state. When
// the user completes the creation we'll see an ONLOGIN_NOTIFICATION
// notification from FxA and we'll move to either the STATE_USER_FXA_VERIFIED
// state or we'll just complete the migration if they login as an already
// verified user.
getFxAccountCreationOptions: Task.async(function* (win) {
// warn if we aren't in the expected state - but go ahead anyway!
if (this._state != this.STATE_USER_FXA) {
this.log.warn("getFxAccountCreationOptions called in an unexpected state: ${}", this._state);
}
// We need to obtain the sentinel and apply any prefs that might be
// specified *before* attempting to setup FxA as the prefs might
// specify custom servers etc.
let sentinel = yield this._getSyncMigrationSentinel();
if (sentinel && sentinel.prefs) {
this._applySentinelPrefs(sentinel.prefs);
}
// If we already have a sentinel then we assume the user has previously
// created the specified account, so just ask to sign-in.
let action = sentinel ? "signin" : "signup";
// See if we can find a default account name to use.
let email = yield this._getDefaultAccountName(sentinel);
let tail = email ? "&email=" + encodeURIComponent(email) : "";
// A special flag so server-side metrics can tell this is part of migration.
tail += "&migration=sync11";
// We want to ask FxA to offer a "Customize Sync" checkbox iff any engines
// are disabled.
let customize = !this._allEnginesEnabled();
tail += "&customizeSync=" + customize;
// We assume the caller of this is going to actually use it, so record
// telemetry now.
this.recordTelemetry(this.TELEMETRY_ACCEPTED);
return {
url: "about:accounts?action=" + action + tail,
options: {ignoreFragment: true, replaceQueryString: true}
};
}),
// Ask the FxA servers to re-send a verification mail for the currently
// logged in user. This should only be called while we are in the
// STATE_USER_FXA_VERIFIED state. When the user clicks on the link in
// the mail we should see an ONVERIFIED_NOTIFICATION which will cause us
// to complete the migration.
resendVerificationMail: Task.async(function * (win) {
// warn if we aren't in the expected state - but go ahead anyway!
if (this._state != this.STATE_USER_FXA_VERIFIED) {
this.log.warn("resendVerificationMail called in an unexpected state: ${}", this._state);
}
let ok = true;
try {
yield fxAccounts.resendVerificationEmail();
} catch (ex) {
this.log.error("Failed to resend verification mail: ${}", ex);
ok = false;
}
this.recordTelemetry(this.TELEMETRY_ACCEPTED);
let fxauser = yield fxAccounts.getSignedInUser();
let sb = Services.strings.createBundle("chrome://browser/locale/accounts.properties");
let heading = ok ?
sb.formatStringFromName("verificationSentHeading", [fxauser.email], 1) :
sb.GetStringFromName("verificationNotSentHeading");
let title = sb.GetStringFromName(ok ? "verificationSentTitle" : "verificationNotSentTitle");
let description = sb.GetStringFromName(ok ? "verificationSentDescription"
: "verificationNotSentDescription");
let factory = Cc["@mozilla.org/prompter;1"]
.getService(Ci.nsIPromptFactory);
let prompt = factory.getPrompt(win, Ci.nsIPrompt);
let bag = prompt.QueryInterface(Ci.nsIWritablePropertyBag2);
bag.setPropertyAsBool("allowTabModal", true);
prompt.alert(title, heading + "\n\n" + description);
}),
// "forget" about the current Firefox account. This should only be called
// while we are in the STATE_USER_FXA_VERIFIED state. After this we will
// see an ONLOGOUT_NOTIFICATION, which will cause the migrator to return back
// to the STATE_USER_FXA state, from where they can choose a different account.
forgetFxAccount: Task.async(function * () {
// warn if we aren't in the expected state - but go ahead anyway!
if (this._state != this.STATE_USER_FXA_VERIFIED) {
this.log.warn("forgetFxAccount called in an unexpected state: ${}", this._state);
}
return fxAccounts.signOut();
}),
recordTelemetry(flag) {
// Note the value is the telemetry field name - but this is an
// implementation detail which could be changed later.
switch (flag) {
case this.TELEMETRY_ACCEPTED:
case this.TELEMETRY_UNLINKED:
case this.TELEMETRY_DECLINED:
Services.obs.notifyObservers(null, OBSERVER_INTERNAL_TELEMETRY_TOPIC, flag);
break;
default:
throw new Error("Unexpected telemetry flag: " + flag);
}
},
get learnMoreLink() {
try {
var url = Services.prefs.getCharPref("app.support.baseURL");
} catch (err) {
return null;
}
url += "sync-upgrade";
let sb = Services.strings.createBundle("chrome://weave/locale/services/sync.properties");
return {
text: sb.GetStringFromName("sync.eol.learnMore.label"),
href: Services.urlFormatter.formatURL(url),
};
},
};
// We expose a singleton
this.EXPORTED_SYMBOLS = ["fxaMigrator"];
let fxaMigrator = new Migrator();

301
services/sync/modules/SyncedTabs.jsm
View file

@ -1,301 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = ["SyncedTabs"];
const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components;
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Task.jsm");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://gre/modules/PlacesUtils.jsm", this);
Cu.import("resource://services-sync/main.js");
Cu.import("resource://gre/modules/Preferences.jsm");
// The Sync XPCOM service
XPCOMUtils.defineLazyGetter(this, "weaveXPCService", function() {
return Cc["@mozilla.org/weave/service;1"]
.getService(Ci.nsISupports)
.wrappedJSObject;
});
// from MDN...
function escapeRegExp(string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}
// A topic we fire whenever we have new tabs available. This might be due
// to a request made by this module to refresh the tab list, or as the result
// of a regularly scheduled sync. The intent is that consumers just listen
// for this notification and update their UI in response.
const TOPIC_TABS_CHANGED = "services.sync.tabs.changed";
// The interval, in seconds, before which we consider the existing list
// of tabs "fresh enough" and don't force a new sync.
const TABS_FRESH_ENOUGH_INTERVAL = 30;
let log = Log.repository.getLogger("Sync.RemoteTabs");
// A new scope to do the logging thang...
(function() {
let level = Preferences.get("services.sync.log.logger.tabs");
if (level) {
let appender = new Log.DumpAppender();
log.level = appender.level = Log.Level[level] || Log.Level.Debug;
log.addAppender(appender);
}
})();
// A private singleton that does the work.
let SyncedTabsInternal = {
/* Make a "tab" record. Returns a promise */
_makeTab: Task.async(function* (client, tab, url, showRemoteIcons) {
let icon;
if (showRemoteIcons) {
icon = tab.icon;
}
if (!icon) {
try {
icon = (yield PlacesUtils.promiseFaviconLinkUrl(url)).spec;
} catch (ex) { /* no favicon avaiable */ }
}
if (!icon) {
icon = "";
}
return {
type: "tab",
title: tab.title || url,
url,
icon,
client: client.id,
lastUsed: tab.lastUsed,
};
}),
/* Make a "client" record. Returns a promise for consistency with _makeTab */
_makeClient: Task.async(function* (client) {
return {
id: client.id,
type: "client",
name: Weave.Service.clientsEngine.getClientName(client.id),
isMobile: Weave.Service.clientsEngine.isMobile(client.id),
lastModified: client.lastModified * 1000, // sec to ms
tabs: []
};
}),
_tabMatchesFilter(tab, filter) {
let reFilter = new RegExp(escapeRegExp(filter), "i");
return tab.url.match(reFilter) || tab.title.match(reFilter);
},
getTabClients: Task.async(function* (filter) {
log.info("Generating tab list with filter", filter);
let result = [];
// If Sync isn't ready, don't try and get anything.
if (!weaveXPCService.ready) {
log.debug("Sync isn't yet ready, so returning an empty tab list");
return result;
}
// A boolean that controls whether we should show the icon from the remote tab.
const showRemoteIcons = Preferences.get("services.sync.syncedTabs.showRemoteIcons", true);
let engine = Weave.Service.engineManager.get("tabs");
let seenURLs = new Set();
let parentIndex = 0;
let ntabs = 0;
for (let [guid, client] of Object.entries(engine.getAllClients())) {
if (!Weave.Service.clientsEngine.remoteClientExists(client.id)) {
continue;
}
let clientRepr = yield this._makeClient(client);
log.debug("Processing client", clientRepr);
for (let tab of client.tabs) {
let url = tab.urlHistory[0];
log.debug("remote tab", url);
// Note there are some issues with tracking "seen" tabs, including:
// * We really can't return the entire urlHistory record as we are
// only checking the first entry - others might be different.
// * We don't update the |lastUsed| timestamp to reflect the
// most-recently-seen time.
// In a followup we should consider simply dropping this |seenUrls|
// check and return duplicate records - it seems the user will be more
// confused by tabs not showing up on a device (because it was detected
// as a dupe so it only appears on a different device) than being
// confused by seeing the same tab on different clients.
if (!url || seenURLs.has(url)) {
continue;
}
let tabRepr = yield this._makeTab(client, tab, url, showRemoteIcons);
if (filter && !this._tabMatchesFilter(tabRepr, filter)) {
continue;
}
seenURLs.add(url);
clientRepr.tabs.push(tabRepr);
}
// We return all clients, even those without tabs - the consumer should
// filter it if they care.
ntabs += clientRepr.tabs.length;
result.push(clientRepr);
}
log.info(`Final tab list has ${result.length} clients with ${ntabs} tabs.`);
return result;
}),
syncTabs(force) {
if (!force) {
// Don't bother refetching tabs if we already did so recently
let lastFetch = Preferences.get("services.sync.lastTabFetch", 0);
let now = Math.floor(Date.now() / 1000);
if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL) {
log.info("_refetchTabs was done recently, do not doing it again");
return Promise.resolve(false);
}
}
// If Sync isn't configured don't try and sync, else we will get reports
// of a login failure.
if (Weave.Status.checkSetup() == Weave.CLIENT_NOT_CONFIGURED) {
log.info("Sync client is not configured, so not attempting a tab sync");
return Promise.resolve(false);
}
// Ask Sync to just do the tabs engine if it can.
// Sync is currently synchronous, so do it after an event-loop spin to help
// keep the UI responsive.
return new Promise((resolve, reject) => {
Services.tm.currentThread.dispatch(() => {
try {
log.info("Doing a tab sync.");
Weave.Service.sync(["tabs"]);
resolve(true);
} catch (ex) {
log.error("Sync failed", ex);
reject(ex);
};
}, Ci.nsIThread.DISPATCH_NORMAL);
});
},
observe(subject, topic, data) {
log.trace(`observed topic=${topic}, data=${data}, subject=${subject}`);
switch (topic) {
case "weave:engine:sync:finish":
if (data != "tabs") {
return;
}
// The tabs engine just finished syncing
// Set our lastTabFetch pref here so it tracks both explicit sync calls
// and normally scheduled ones.
Preferences.set("services.sync.lastTabFetch", Math.floor(Date.now() / 1000));
Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
break;
case "weave:service:start-over":
// start-over needs to notify so consumers find no tabs.
Preferences.reset("services.sync.lastTabFetch");
Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
break;
case "nsPref:changed":
Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
break;
default:
break;
}
},
// Returns true if Sync is configured to Sync tabs, false otherwise
get isConfiguredToSyncTabs() {
if (!weaveXPCService.ready) {
log.debug("Sync isn't yet ready; assuming tab engine is enabled");
return true;
}
let engine = Weave.Service.engineManager.get("tabs");
return engine && engine.enabled;
},
get hasSyncedThisSession() {
let engine = Weave.Service.engineManager.get("tabs");
return engine && engine.hasSyncedThisSession;
},
};
Services.obs.addObserver(SyncedTabsInternal, "weave:engine:sync:finish", false);
Services.obs.addObserver(SyncedTabsInternal, "weave:service:start-over", false);
// Observe the pref the indicates the state of the tabs engine has changed.
// This will force consumers to re-evaluate the state of sync and update
// accordingly.
Services.prefs.addObserver("services.sync.engine.tabs", SyncedTabsInternal, false);
// The public interface.
this.SyncedTabs = {
// A mock-point for tests.
_internal: SyncedTabsInternal,
// We make the topic for the observer notification public.
TOPIC_TABS_CHANGED,
// Returns true if Sync is configured to Sync tabs, false otherwise
get isConfiguredToSyncTabs() {
return this._internal.isConfiguredToSyncTabs;
},
// Returns true if a tab sync has completed once this session. If this
// returns false, then getting back no clients/tabs possibly just means we
// are waiting for that first sync to complete.
get hasSyncedThisSession() {
return this._internal.hasSyncedThisSession;
},
// Return a promise that resolves with an array of client records, each with
// a .tabs array. Note that part of the contract for this module is that the
// returned objects are not shared between invocations, so callers are free
// to mutate the returned objects (eg, sort, truncate) however they see fit.
getTabClients(query) {
return this._internal.getTabClients(query);
},
// Starts a background request to start syncing tabs. Returns a promise that
// resolves when the sync is complete, but there's no resolved value -
// callers should be listening for TOPIC_TABS_CHANGED.
// If |force| is true we always sync. If false, we only sync if the most
// recent sync wasn't "recently".
syncTabs(force) {
return this._internal.syncTabs(force);
},
sortTabClientsByLastUsed(clients, maxTabs = Infinity) {
// First sort and filter the list of tabs for each client. Note that
// this module promises that the objects it returns are never
// shared, so we are free to mutate those objects directly.
for (let client of clients) {
let tabs = client.tabs;
tabs.sort((a, b) => b.lastUsed - a.lastUsed);
if (Number.isFinite(maxTabs)) {
client.tabs = tabs.slice(0, maxTabs);
}
}
// Now sort the clients - the clients are sorted in the order of the
// most recent tab for that client (ie, it is important the tabs for
// each client are already sorted.)
clients.sort((a, b) => {
if (a.tabs.length == 0) {
return 1; // b comes first.
}
if (b.tabs.length == 0) {
return -1; // a comes first.
}
return b.tabs[0].lastUsed - a.tabs[0].lastUsed;
});
},
};

35
services/sync/modules/addonsreconciler.js
View file

@ -17,7 +17,7 @@
"use strict";
var Cu = Components.utils;
const Cu = Components.utils;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/util.js");
@ -218,12 +218,11 @@ AddonsReconciler.prototype = {
}
this._addons = json.addons;
for (let id in this._addons) {
let record = this._addons[id];
for each (let record in this._addons) {
record.modified = new Date(record.modified);
}
for (let [time, change, id] of json.changes) {
for each (let [time, change, id] in json.changes) {
this._changes.push([new Date(time), change, id]);
}
@ -247,9 +246,9 @@ AddonsReconciler.prototype = {
let file = path || DEFAULT_STATE_FILE;
let state = {version: 1, addons: {}, changes: []};
for (let [id, record] of Object.entries(this._addons)) {
for (let [id, record] in Iterator(this._addons)) {
state.addons[id] = {};
for (let [k, v] of Object.entries(record)) {
for (let [k, v] in Iterator(record)) {
if (k == "modified") {
state.addons[id][k] = v.getTime();
}
@ -259,7 +258,7 @@ AddonsReconciler.prototype = {
}
}
for (let [time, change, id] of this._changes) {
for each (let [time, change, id] in this._changes) {
state.changes.push([time.getTime(), change, id]);
}
@ -351,14 +350,14 @@ AddonsReconciler.prototype = {
AddonManager.getAllAddons(function (addons) {
let ids = {};
for (let addon of addons) {
for each (let addon in addons) {
ids[addon.id] = true;
this.rectifyStateFromAddon(addon);
}
// Look for locally-defined add-ons that no longer exist and update their
// record.
for (let [id, addon] of Object.entries(this._addons)) {
for (let [id, addon] in Iterator(this._addons)) {
if (id in ids) {
continue;
}
@ -374,7 +373,7 @@ AddonsReconciler.prototype = {
}
let installFound = false;
for (let install of installs) {
for each (let install in installs) {
if (install.addon && install.addon.id == id &&
install.state == AddonManager.STATE_INSTALLED) {
@ -417,7 +416,7 @@ AddonsReconciler.prototype = {
* Addon instance being updated.
*/
rectifyStateFromAddon: function rectifyStateFromAddon(addon) {
this._log.debug(`Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})`);
this._log.debug("Rectifying state for addon: " + addon.id);
this._ensureStateLoaded();
let id = addon.id;
@ -434,8 +433,7 @@ AddonsReconciler.prototype = {
modified: now,
type: addon.type,
scope: addon.scope,
foreignInstall: addon.foreignInstall,
isSyncable: addon.isSyncable,
foreignInstall: addon.foreignInstall
};
this._addons[id] = record;
this._log.debug("Adding change because add-on not present locally: " +
@ -445,7 +443,6 @@ AddonsReconciler.prototype = {
}
let record = this._addons[id];
record.isSyncable = addon.isSyncable;
if (!record.installed) {
// It is possible the record is marked as uninstalled because an
@ -486,11 +483,12 @@ AddonsReconciler.prototype = {
this._log.info("Change recorded for " + state.id);
this._changes.push([date, change, state.id]);
for (let listener of this._listeners) {
for each (let listener in this._listeners) {
try {
listener.changeListener.call(listener, date, change, state);
} catch (ex) {
this._log.warn("Exception calling change listener", ex);
this._log.warn("Exception calling change listener: " +
Utils.exceptionStr(ex));
}
}
},
@ -556,8 +554,7 @@ AddonsReconciler.prototype = {
* @return Object on success on null on failure.
*/
getAddonStateFromSyncGUID: function getAddonStateFromSyncGUID(guid) {
for (let id in this.addons) {
let addon = this.addons[id];
for each (let addon in this.addons) {
if (addon.guid == guid) {
return addon;
}
@ -636,7 +633,7 @@ AddonsReconciler.prototype = {
}
}
catch (ex) {
this._log.warn("Exception", ex);
this._log.warn("Exception: " + Utils.exceptionStr(ex));
}
},

94
services/sync/modules/addonutils.js
View file

@ -6,7 +6,7 @@
this.EXPORTED_SYMBOLS = ["AddonUtils"];
var {interfaces: Ci, utils: Cu} = Components;
const {interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Log.jsm");
@ -38,10 +38,21 @@ AddonUtilsInternal.prototype = {
* Function to be called with result of operation.
*/
getInstallFromSearchResult:
function getInstallFromSearchResult(addon, cb) {
function getInstallFromSearchResult(addon, cb, requireSecureURI=true) {
this._log.debug("Obtaining install for " + addon.id);
// Verify that the source URI uses TLS. We don't allow installs from
// insecure sources for security reasons. The Addon Manager ensures that
// cert validation, etc is performed.
if (requireSecureURI) {
let scheme = addon.sourceURI.scheme;
if (scheme != "https") {
cb(new Error("Insecure source URI scheme: " + scheme), addon.install);
return;
}
}
// We should theoretically be able to obtain (and use) addon.install if
// it is available. However, the addon.sourceURI rewriting won't be
// reflected in the AddonInstall, so we can't use it. If we ever get rid
@ -69,6 +80,8 @@ AddonUtilsInternal.prototype = {
* syncGUID - Sync GUID to use for the new add-on.
* enabled - Boolean indicating whether the add-on should be enabled upon
* install.
* requireSecureURI - Boolean indicating whether to require a secure
* URI to install from. This defaults to true.
*
* When complete it calls a callback with 2 arguments, error and result.
*
@ -92,6 +105,10 @@ AddonUtilsInternal.prototype = {
function installAddonFromSearchResult(addon, options, cb) {
this._log.info("Trying to install add-on from search result: " + addon.id);
if (options.requireSecureURI === undefined) {
options.requireSecureURI = true;
}
this.getInstallFromSearchResult(addon, function onResult(error, install) {
if (error) {
cb(error, null);
@ -147,10 +164,10 @@ AddonUtilsInternal.prototype = {
install.install();
}
catch (ex) {
this._log.error("Error installing add-on", ex);
this._log.error("Error installing add-on: " + Utils.exceptionstr(ex));
cb(ex, null);
}
}.bind(this));
}.bind(this), options.requireSecureURI);
},
/**
@ -231,7 +248,7 @@ AddonUtilsInternal.prototype = {
}
let ids = [];
for (let addon of installs) {
for each (let addon in installs) {
ids.push(addon.id);
}
@ -244,7 +261,6 @@ AddonUtilsInternal.prototype = {
installedIDs: [],
installs: [],
addons: [],
skipped: [],
errors: []
};
@ -282,21 +298,15 @@ AddonUtilsInternal.prototype = {
// server-side metrics aren't skewed (bug 708134). The server should
// ideally send proper URLs, but this solution was deemed too
// complicated at the time the functionality was implemented.
for (let addon of addons) {
// Find the specified options for this addon.
let options;
for (let install of installs) {
if (install.id == addon.id) {
options = install;
break;
}
}
if (!this.canInstallAddon(addon, options)) {
ourResult.skipped.push(addon.id);
for each (let addon in addons) {
// sourceURI presence isn't enforced by AddonRepository. So, we skip
// add-ons without a sourceURI.
if (!addon.sourceURI) {
this._log.info("Skipping install of add-on because missing " +
"sourceURI: " + addon.id);
continue;
}
// We can go ahead and attempt to install it.
toInstall.push(addon);
// We should always be able to QI the nsIURI to nsIURL. If not, we
@ -332,9 +342,9 @@ AddonUtilsInternal.prototype = {
// Start all the installs asynchronously. They will report back to us
// as they finish, eventually triggering the global callback.
for (let addon of toInstall) {
for each (let addon in toInstall) {
let options = {};
for (let install of installs) {
for each (let install in installs) {
if (install.id == addon.id) {
options = install;
break;
@ -352,52 +362,10 @@ AddonUtilsInternal.prototype = {
});
},
/**
* Returns true if we are able to install the specified addon, false
* otherwise. It is expected that this will log the reason if it returns
* false.
*
* @param addon
* (Addon) Add-on instance to check.
* @param options
* (object) The options specified for this addon. See installAddons()
* for the valid elements.
*/
canInstallAddon(addon, options) {
// sourceURI presence isn't enforced by AddonRepository. So, we skip
// add-ons without a sourceURI.
if (!addon.sourceURI) {
this._log.info("Skipping install of add-on because missing " +
"sourceURI: " + addon.id);
return false;
}
// Verify that the source URI uses TLS. We don't allow installs from
// insecure sources for security reasons. The Addon Manager ensures
// that cert validation etc is performed.
// (We should also consider just dropping this entirely and calling
// XPIProvider.isInstallAllowed, but that has additional semantics we might
// need to think through...)
let requireSecureURI = true;
if (options && options.requireSecureURI !== undefined) {
requireSecureURI = options.requireSecureURI;
}
if (requireSecureURI) {
let scheme = addon.sourceURI.scheme;
if (scheme != "https") {
this._log.info(`Skipping install of add-on "${addon.id}" because sourceURI's scheme of "${scheme}" is not trusted`);
return false;
}
}
this._log.info(`Add-on "${addon.id}" is able to be installed`);
return true;
},
/**
* Update the user disabled flag for an add-on.
*
* The supplied callback will be called when the operation is
* The supplied callback will ba called when the operation is
* complete. If the new flag matches the existing or if the add-on
* isn't currently active, the function will fire the callback
* immediately. Else, the callback is invoked when the AddonManager

784
services/sync/modules/bookmark_validator.js
View file

@ -1,784 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const Cu = Components.utils;
Cu.import("resource://gre/modules/PlacesUtils.jsm");
Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
Cu.import("resource://gre/modules/Task.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
this.EXPORTED_SYMBOLS = ["BookmarkValidator", "BookmarkProblemData"];
const LEFT_PANE_ROOT_ANNO = "PlacesOrganizer/OrganizerFolder";
const LEFT_PANE_QUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
// Indicates if a local bookmark tree node should be excluded from syncing.
function isNodeIgnored(treeNode) {
return treeNode.annos && treeNode.annos.some(anno => anno.name == LEFT_PANE_ROOT_ANNO ||
anno.name == LEFT_PANE_QUERY_ANNO);
}
const BOOKMARK_VALIDATOR_VERSION = 1;
/**
* Result of bookmark validation. Contains the following fields which describe
* server-side problems unless otherwise specified.
*
* - missingIDs (number): # of objects with missing ids
* - duplicates (array of ids): ids seen more than once
* - parentChildMismatches (array of {parent: parentid, child: childid}):
* instances where the child's parentid and the parent's children array
* do not match
* - cycles (array of array of ids). List of cycles found in the server-side tree.
* - clientCycles (array of array of ids). List of cycles found in the client-side tree.
* - orphans (array of {id: string, parent: string}): List of nodes with
* either no parentid, or where the parent could not be found.
* - missingChildren (array of {parent: id, child: id}):
* List of parent/children where the child id couldn't be found
* - deletedChildren (array of { parent: id, child: id }):
* List of parent/children where child id was a deleted item (but still showed up
* in the children array)
* - multipleParents (array of {child: id, parents: array of ids}):
* List of children that were part of multiple parent arrays
* - deletedParents (array of ids) : List of records that aren't deleted but
* had deleted parents
* - childrenOnNonFolder (array of ids): list of non-folders that still have
* children arrays
* - duplicateChildren (array of ids): list of records who have the same
* child listed multiple times in their children array
* - parentNotFolder (array of ids): list of records that have parents that
* aren't folders
* - rootOnServer (boolean): true if the root came from the server
* - badClientRoots (array of ids): Contains any client-side root ids where
* the root is missing or isn't a (direct) child of the places root.
*
* - clientMissing: Array of ids on the server missing from the client
* - serverMissing: Array of ids on the client missing from the server
* - serverDeleted: Array of ids on the client that the server had marked as deleted.
* - serverUnexpected: Array of ids that appear on the server but shouldn't
* because the client attempts to never upload them.
* - differences: Array of {id: string, differences: string array} recording
* the non-structural properties that are differente between the client and server
* - structuralDifferences: As above, but contains the items where the differences were
* structural, that is, they contained childGUIDs or parentid
*/
class BookmarkProblemData {
constructor() {
this.rootOnServer = false;
this.missingIDs = 0;
this.duplicates = [];
this.parentChildMismatches = [];
this.cycles = [];
this.clientCycles = [];
this.orphans = [];
this.missingChildren = [];
this.deletedChildren = [];
this.multipleParents = [];
this.deletedParents = [];
this.childrenOnNonFolder = [];
this.duplicateChildren = [];
this.parentNotFolder = [];
this.badClientRoots = [];
this.clientMissing = [];
this.serverMissing = [];
this.serverDeleted = [];
this.serverUnexpected = [];
this.differences = [];
this.structuralDifferences = [];
}
/**
* Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into
* [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc.
*/
_summarizeDifferences(prefix, diffs) {
let diffCounts = new Map();
for (let { differences } of diffs) {
for (let type of differences) {
let name = prefix + ":" + type;
let count = diffCounts.get(name) || 0;
diffCounts.set(name, count + 1);
}
}
return [...diffCounts].map(([name, count]) => ({ name, count }));
}
/**
* Produce a list summarizing problems found. Each entry contains {name, count},
* where name is the field name for the problem, and count is the number of times
* the problem was encountered.
*
* Validation has failed if all counts are not 0.
*
* If the `full` argument is truthy, we also include information about which
* properties we saw structural differences in. Currently, this means either
* "sdiff:parentid" and "sdiff:childGUIDS" may be present.
*/
getSummary(full) {
let result = [
{ name: "clientMissing", count: this.clientMissing.length },
{ name: "serverMissing", count: this.serverMissing.length },
{ name: "serverDeleted", count: this.serverDeleted.length },
{ name: "serverUnexpected", count: this.serverUnexpected.length },
{ name: "structuralDifferences", count: this.structuralDifferences.length },
{ name: "differences", count: this.differences.length },
{ name: "missingIDs", count: this.missingIDs },
{ name: "rootOnServer", count: this.rootOnServer ? 1 : 0 },
{ name: "duplicates", count: this.duplicates.length },
{ name: "parentChildMismatches", count: this.parentChildMismatches.length },
{ name: "cycles", count: this.cycles.length },
{ name: "clientCycles", count: this.clientCycles.length },
{ name: "badClientRoots", count: this.badClientRoots.length },
{ name: "orphans", count: this.orphans.length },
{ name: "missingChildren", count: this.missingChildren.length },
{ name: "deletedChildren", count: this.deletedChildren.length },
{ name: "multipleParents", count: this.multipleParents.length },
{ name: "deletedParents", count: this.deletedParents.length },
{ name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length },
{ name: "duplicateChildren", count: this.duplicateChildren.length },
{ name: "parentNotFolder", count: this.parentNotFolder.length },
];
if (full) {
let structural = this._summarizeDifferences("sdiff", this.structuralDifferences);
result.push.apply(result, structural);
}
return result;
}
}
// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon.
XPCOMUtils.defineLazyGetter(this, "SYNCED_ROOTS", () => [
PlacesUtils.bookmarks.menuGuid,
PlacesUtils.bookmarks.toolbarGuid,
PlacesUtils.bookmarks.unfiledGuid,
PlacesUtils.bookmarks.mobileGuid,
]);
class BookmarkValidator {
_followQueries(recordMap) {
for (let [guid, entry] of recordMap) {
if (entry.type !== "query" && (!entry.bmkUri || !entry.bmkUri.startsWith("place:"))) {
continue;
}
// Might be worth trying to parse the place: query instead so that this
// works "automatically" with things like aboutsync.
let queryNodeParent = PlacesUtils.getFolderContents(entry, false, true);
if (!queryNodeParent || !queryNodeParent.root.hasChildren) {
continue;
}
queryNodeParent = queryNodeParent.root;
let queryNode = null;
let numSiblings = 0;
let containerWasOpen = queryNodeParent.containerOpen;
queryNodeParent.containerOpen = true;
try {
try {
numSiblings = queryNodeParent.childCount;
} catch (e) {
// This throws when we can't actually get the children. This is the
// case for history containers, tag queries, ...
continue;
}
for (let i = 0; i < numSiblings && !queryNode; ++i) {
let child = queryNodeParent.getChild(i);
if (child && child.bookmarkGuid && child.bookmarkGuid === guid) {
queryNode = child;
}
}
} finally {
queryNodeParent.containerOpen = containerWasOpen;
}
if (!queryNode) {
continue;
}
let concreteId = PlacesUtils.getConcreteItemGuid(queryNode);
if (!concreteId) {
continue;
}
let concreteItem = recordMap.get(concreteId);
if (!concreteItem) {
continue;
}
entry.concrete = concreteItem;
}
}
createClientRecordsFromTree(clientTree) {
// Iterate over the treeNode, converting it to something more similar to what
// the server stores.
let records = [];
let recordsByGuid = new Map();
let syncedRoots = SYNCED_ROOTS;
function traverse(treeNode, synced) {
if (!synced) {
synced = syncedRoots.includes(treeNode.guid);
} else if (isNodeIgnored(treeNode)) {
synced = false;
}
let guid = PlacesSyncUtils.bookmarks.guidToSyncId(treeNode.guid);
let itemType = 'item';
treeNode.ignored = !synced;
treeNode.id = guid;
switch (treeNode.type) {
case PlacesUtils.TYPE_X_MOZ_PLACE:
let query = null;
if (treeNode.annos && treeNode.uri.startsWith("place:")) {
query = treeNode.annos.find(({name}) =>
name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO);
}
if (query && query.value) {
itemType = 'query';
} else {
itemType = 'bookmark';
}
break;
case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
let isLivemark = false;
if (treeNode.annos) {
for (let anno of treeNode.annos) {
if (anno.name === PlacesUtils.LMANNO_FEEDURI) {
isLivemark = true;
treeNode.feedUri = anno.value;
} else if (anno.name === PlacesUtils.LMANNO_SITEURI) {
isLivemark = true;
treeNode.siteUri = anno.value;
}
}
}
itemType = isLivemark ? "livemark" : "folder";
break;
case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
itemType = 'separator';
break;
}
if (treeNode.tags) {
treeNode.tags = treeNode.tags.split(",");
} else {
treeNode.tags = [];
}
treeNode.type = itemType;
treeNode.pos = treeNode.index;
treeNode.bmkUri = treeNode.uri;
records.push(treeNode);
// We want to use the "real" guid here.
recordsByGuid.set(treeNode.guid, treeNode);
if (treeNode.type === 'folder') {
treeNode.childGUIDs = [];
if (!treeNode.children) {
treeNode.children = [];
}
for (let child of treeNode.children) {
traverse(child, synced);
child.parent = treeNode;
child.parentid = guid;
treeNode.childGUIDs.push(child.guid);
}
}
}
traverse(clientTree, false);
clientTree.id = 'places';
this._followQueries(recordsByGuid);
return records;
}
/**
* Process the server-side list. Mainly this builds the records into a tree,
* but it also records information about problems, and produces arrays of the
* deleted and non-deleted nodes.
*
* Returns an object containing:
* - records:Array of non-deleted records. Each record contains the following
* properties
* - childGUIDs (array of strings, only present if type is 'folder'): the
* list of child GUIDs stored on the server.
* - children (array of records, only present if type is 'folder'):
* each record has these same properties. This may differ in content
* from what you may expect from the childGUIDs list, as it won't
* contain any records that could not be found.
* - parent (record): The parent to this record.
* - Unchanged properties send down from the server: id, title, type,
* parentName, parentid, bmkURI, keyword, tags, pos, queryId, loadInSidebar
* - root: Root of the server-side bookmark tree. Has the same properties as
* above.
* - deletedRecords: As above, but only contains items that the server sent
* where it also sent indication that the item should be deleted.
* - problemData: a BookmarkProblemData object, with the caveat that
* the fields describing client/server relationship will not have been filled
* out yet.
*/
inspectServerRecords(serverRecords) {
let deletedItemIds = new Set();
let idToRecord = new Map();
let deletedRecords = [];
let folders = [];
let problems = [];
let problemData = new BookmarkProblemData();
let resultRecords = [];
for (let record of serverRecords) {
if (!record.id) {
++problemData.missingIDs;
continue;
}
if (record.deleted) {
deletedItemIds.add(record.id);
} else {
if (idToRecord.has(record.id)) {
problemData.duplicates.push(record.id);
continue;
}
}
idToRecord.set(record.id, record);
if (record.children) {
if (record.type !== "folder") {
// Due to implementation details in engines/bookmarks.js, (Livemark
// subclassing BookmarkFolder) Livemarks will have a children array,
// but it should still be empty.
if (!record.children.length) {
continue;
}
// Otherwise we mark it as an error and still try to resolve the children
problemData.childrenOnNonFolder.push(record.id);
}
folders.push(record);
if (new Set(record.children).size !== record.children.length) {
problemData.duplicateChildren.push(record.id)
}
// The children array stores special guids as their local guid values,
// e.g. 'menu________' instead of 'menu', but all other parts of the
// serverside bookmark info stores it as the special value ('menu').
record.childGUIDs = record.children;
record.children = record.children.map(childID => {
return PlacesSyncUtils.bookmarks.guidToSyncId(childID);
});
}
}
for (let deletedId of deletedItemIds) {
let record = idToRecord.get(deletedId);
if (record && !record.isDeleted) {
deletedRecords.push(record);
record.isDeleted = true;
}
}
let root = idToRecord.get('places');
if (!root) {
// Fabricate a root. We want to remember that it's fake so that we can
// avoid complaining about stuff like it missing it's childGUIDs later.
root = { id: 'places', children: [], type: 'folder', title: '', fake: true };
resultRecords.push(root);
idToRecord.set('places', root);
} else {
problemData.rootOnServer = true;
}
// Build the tree, find orphans, and record most problems having to do with
// the tree structure.
for (let [id, record] of idToRecord) {
if (record === root) {
continue;
}
if (record.isDeleted) {
continue;
}
let parentID = record.parentid;
if (!parentID) {
problemData.orphans.push({id: record.id, parent: parentID});
continue;
}
let parent = idToRecord.get(parentID);
if (!parent) {
problemData.orphans.push({id: record.id, parent: parentID});
continue;
}
if (parent.type !== 'folder') {
problemData.parentNotFolder.push(record.id);
if (!parent.children) {
parent.children = [];
}
if (!parent.childGUIDs) {
parent.childGUIDs = [];
}
}
if (!record.isDeleted) {
resultRecords.push(record);
}
record.parent = parent;
if (parent !== root || problemData.rootOnServer) {
let childIndex = parent.children.indexOf(id);
if (childIndex < 0) {
problemData.parentChildMismatches.push({parent: parent.id, child: record.id});
} else {
parent.children[childIndex] = record;
}
} else {
parent.children.push(record);
}
if (parent.isDeleted && !record.isDeleted) {
problemData.deletedParents.push(record.id);
}
// We used to check if the parentName on the server matches the actual
// local parent name, but given this is used only for de-duping a record
// the first time it is seen and expensive to keep up-to-date, we decided
// to just stop recording it. See bug 1276969 for more.
}
// Check that we aren't missing any children.
for (let folder of folders) {
folder.unfilteredChildren = folder.children;
folder.children = [];
for (let ci = 0; ci < folder.unfilteredChildren.length; ++ci) {
let child = folder.unfilteredChildren[ci];
let childObject;
if (typeof child == "string") {
// This can happen the parent refers to a child that has a different
// parentid, or if it refers to a missing or deleted child. It shouldn't
// be possible with totally valid bookmarks.
childObject = idToRecord.get(child);
if (!childObject) {
problemData.missingChildren.push({parent: folder.id, child});
} else {
folder.unfilteredChildren[ci] = childObject;
if (childObject.isDeleted) {
problemData.deletedChildren.push({ parent: folder.id, child });
}
}
} else {
childObject = child;
}
if (!childObject) {
continue;
}
if (childObject.parentid === folder.id) {
folder.children.push(childObject);
continue;
}
// The child is very probably in multiple `children` arrays --
// see if we already have a problem record about it.
let currentProblemRecord = problemData.multipleParents.find(pr =>
pr.child === child);
if (currentProblemRecord) {
currentProblemRecord.parents.push(folder.id);
continue;
}
let otherParent = idToRecord.get(childObject.parentid);
// it's really an ... orphan ... sort of.
if (!otherParent) {
// if we never end up adding to this parent's list, we filter it out after this loop.
problemData.multipleParents.push({
child,
parents: [folder.id]
});
if (!problemData.orphans.some(r => r.id === child)) {
problemData.orphans.push({
id: child,
parent: childObject.parentid
});
}
continue;
}
if (otherParent.isDeleted) {
if (!problemData.deletedParents.includes(child)) {
problemData.deletedParents.push(child);
}
continue;
}
if (otherParent.childGUIDs && !otherParent.childGUIDs.includes(child)) {
if (!problemData.parentChildMismatches.some(r => r.child === child)) {
// Might not be possible to get here.
problemData.parentChildMismatches.push({ child, parent: folder.id });
}
}
problemData.multipleParents.push({
child,
parents: [childObject.parentid, folder.id]
});
}
}
problemData.multipleParents = problemData.multipleParents.filter(record =>
record.parents.length >= 2);
problemData.cycles = this._detectCycles(resultRecords);
return {
deletedRecords,
records: resultRecords,
problemData,
root,
};
}
// helper for inspectServerRecords
_detectCycles(records) {
// currentPath and pathLookup contain the same data. pathLookup is faster to
// query, but currentPath gives is the order of traversal that we need in
// order to report the members of the cycles.
let pathLookup = new Set();
let currentPath = [];
let cycles = [];
let seenEver = new Set();
const traverse = node => {
if (pathLookup.has(node)) {
let cycleStart = currentPath.lastIndexOf(node);
let cyclePath = currentPath.slice(cycleStart).map(n => n.id);
cycles.push(cyclePath);
return;
} else if (seenEver.has(node)) {
// If we're checking the server, this is a problem, but it should already be reported.
// On the client, this could happen due to including `node.concrete` in the child list.
return;
}
seenEver.add(node);
let children = node.children || [];
if (node.concrete) {
children.push(node.concrete);
}
if (children) {
pathLookup.add(node);
currentPath.push(node);
for (let child of children) {
traverse(child);
}
currentPath.pop();
pathLookup.delete(node);
}
};
for (let record of records) {
if (!seenEver.has(record)) {
traverse(record);
}
}
return cycles;
}
// Perform client-side sanity checking that doesn't involve server data
_validateClient(problemData, clientRecords) {
problemData.clientCycles = this._detectCycles(clientRecords);
for (let rootGUID of SYNCED_ROOTS) {
let record = clientRecords.find(record =>
record.guid === rootGUID);
if (!record || record.parentid !== "places") {
problemData.badClientRoots.push(rootGUID);
}
}
}
/**
* Compare the list of server records with the client tree.
*
* Returns the same data as described in the inspectServerRecords comment,
* with the following additional fields.
* - clientRecords: an array of client records in a similar format to
* the .records (ie, server records) entry.
* - problemData is the same as for inspectServerRecords, except all properties
* will be filled out.
*/
compareServerWithClient(serverRecords, clientTree) {
let clientRecords = this.createClientRecordsFromTree(clientTree);
let inspectionInfo = this.inspectServerRecords(serverRecords);
inspectionInfo.clientRecords = clientRecords;
// Mainly do this to remove deleted items and normalize child guids.
serverRecords = inspectionInfo.records;
let problemData = inspectionInfo.problemData;
this._validateClient(problemData, clientRecords);
let matches = [];
let allRecords = new Map();
let serverDeletedLookup = new Set(inspectionInfo.deletedRecords.map(r => r.id));
for (let sr of serverRecords) {
if (sr.fake) {
continue;
}
allRecords.set(sr.id, {client: null, server: sr});
}
for (let cr of clientRecords) {
let unified = allRecords.get(cr.id);
if (!unified) {
allRecords.set(cr.id, {client: cr, server: null});
} else {
unified.client = cr;
}
}
for (let [id, {client, server}] of allRecords) {
if (!client && server) {
problemData.clientMissing.push(id);
continue;
}
if (!server && client) {
if (serverDeletedLookup.has(id)) {
problemData.serverDeleted.push(id);
} else if (!client.ignored && client.id != "places") {
problemData.serverMissing.push(id);
}
continue;
}
if (server && client && client.ignored) {
problemData.serverUnexpected.push(id);
}
let differences = [];
let structuralDifferences = [];
// Don't bother comparing titles of roots. It's okay if locally it's
// "Mobile Bookmarks", but the server thinks it's "mobile".
// TODO: We probably should be handing other localized bookmarks (e.g.
// default bookmarks) here as well, see bug 1316041.
if (!SYNCED_ROOTS.includes(client.guid)) {
// We want to treat undefined, null and an empty string as identical
if ((client.title || "") !== (server.title || "")) {
differences.push("title");
}
}
if (client.parentid || server.parentid) {
if (client.parentid !== server.parentid) {
structuralDifferences.push('parentid');
}
}
if (client.tags || server.tags) {
let cl = client.tags || [];
let sl = server.tags || [];
if (cl.length !== sl.length || !cl.every((tag, i) => sl.indexOf(tag) >= 0)) {
differences.push('tags');
}
}
let sameType = client.type === server.type;
if (!sameType) {
if (server.type === "query" && client.type === "bookmark" && client.bmkUri.startsWith("place:")) {
sameType = true;
}
}
if (!sameType) {
differences.push('type');
} else {
switch (server.type) {
case 'bookmark':
case 'query':
if (server.bmkUri !== client.bmkUri) {
differences.push('bmkUri');
}
break;
case "livemark":
if (server.feedUri != client.feedUri) {
differences.push("feedUri");
}
if (server.siteUri != client.siteUri) {
differences.push("siteUri");
}
break;
case 'folder':
if (server.id === 'places' && !problemData.rootOnServer) {
// It's the fabricated places root. It won't have the GUIDs, but
// it doesn't matter.
break;
}
if (client.childGUIDs || server.childGUIDs) {
let cl = client.childGUIDs || [];
let sl = server.childGUIDs || [];
if (cl.length !== sl.length || !cl.every((id, i) => sl[i] === id)) {
structuralDifferences.push('childGUIDs');
}
}
break;
}
}
if (differences.length) {
problemData.differences.push({id, differences});
}
if (structuralDifferences.length) {
problemData.structuralDifferences.push({ id, differences: structuralDifferences });
}
}
return inspectionInfo;
}
_getServerState(engine) {
let collection = engine.itemSource();
let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name);
collection.full = true;
let items = [];
collection.recordHandler = function(item) {
item.decrypt(collectionKey);
items.push(item.cleartext);
};
let resp = collection.getBatched();
if (!resp.success) {
throw resp;
}
return items;
}
validate(engine) {
let self = this;
return Task.spawn(function*() {
let start = Date.now();
let clientTree = yield PlacesUtils.promiseBookmarksTree("", {
includeItemIds: true
});
let serverState = self._getServerState(engine);
let serverRecordCount = serverState.length;
let result = self.compareServerWithClient(serverState, clientTree);
let end = Date.now();
let duration = end-start;
return {
duration,
version: self.version,
problems: result.problemData,
recordCount: serverRecordCount
};
});
}
};
BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION;

151
services/sync/modules/browserid_identity.js
View file

@ -4,9 +4,9 @@
"use strict";
this.EXPORTED_SYMBOLS = ["BrowserIDManager", "AuthenticationError"];
this.EXPORTED_SYMBOLS = ["BrowserIDManager"];
var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/async.js");
@ -39,13 +39,12 @@ XPCOMUtils.defineLazyGetter(this, 'log', function() {
});
// FxAccountsCommon.js doesn't use a "namespace", so create one here.
var fxAccountsCommon = {};
let fxAccountsCommon = {};
Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
const OBSERVER_TOPICS = [
fxAccountsCommon.ONLOGIN_NOTIFICATION,
fxAccountsCommon.ONLOGOUT_NOTIFICATION,
fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION,
];
const PREF_SYNC_SHOW_CUSTOMIZATION = "services.sync-setup.ui.showCustomizationDialog";
@ -66,9 +65,8 @@ function deriveKeyBundle(kB) {
some other error object (which should do the right thing when toString() is
called on it)
*/
function AuthenticationError(details, source) {
function AuthenticationError(details) {
this.details = details;
this.source = source;
}
AuthenticationError.prototype = {
@ -106,6 +104,12 @@ this.BrowserIDManager.prototype = {
// we don't consider the lack of a keybundle as a failure state.
_shouldHaveSyncKeyBundle: false,
get readyToAuthenticate() {
// We are finished initializing when we *should* have a sync key bundle,
// although we might not actually have one due to auth failures etc.
return this._shouldHaveSyncKeyBundle;
},
get needsCustomization() {
try {
return Services.prefs.getBoolPref(PREF_SYNC_SHOW_CUSTOMIZATION);
@ -114,34 +118,11 @@ this.BrowserIDManager.prototype = {
}
},
hashedUID() {
if (!this._token) {
throw new Error("hashedUID: Don't have token");
}
return this._token.hashed_fxa_uid
},
deviceID() {
return this._signedInUser && this._signedInUser.deviceId;
},
initialize: function() {
for (let topic of OBSERVER_TOPICS) {
Services.obs.addObserver(this, topic, false);
}
// and a background fetch of account data just so we can set this.account,
// so we have a username available before we've actually done a login.
// XXX - this is actually a hack just for tests and really shouldn't be
// necessary. Also, you'd think it would be safe to allow this.account to
// be set to null when there's no user logged in, but argue with the test
// suite, not with me :)
this._fxaService.getSignedInUser().then(accountData => {
if (accountData) {
this.account = accountData.email;
}
}).catch(err => {
// As above, this is only for tests so it is safe to ignore.
});
return this.initializeWithCurrentIdentity();
},
/**
@ -149,7 +130,7 @@ this.BrowserIDManager.prototype = {
* the user is logged in, or is rejected if the login attempt has failed.
*/
ensureLoggedIn: function() {
if (!this._shouldHaveSyncKeyBundle && this.whenReadyToAuthenticate) {
if (!this._shouldHaveSyncKeyBundle) {
// We are already in the process of logging in.
return this.whenReadyToAuthenticate.promise;
}
@ -163,7 +144,7 @@ this.BrowserIDManager.prototype = {
// re-entering of credentials by the user is necessary we don't take any
// further action - an observer will fire when the user does that.
if (Weave.Status.login == LOGIN_FAILED_LOGIN_REJECTED) {
return Promise.reject(new Error("User needs to re-authenticate"));
return Promise.reject();
}
// So - we've a previous auth problem and aren't currently attempting to
@ -179,6 +160,7 @@ this.BrowserIDManager.prototype = {
}
this.resetCredentials();
this._signedInUser = null;
return Promise.resolve();
},
offerSyncOptions: function () {
@ -202,7 +184,7 @@ this.BrowserIDManager.prototype = {
// Reset the world before we do anything async.
this.whenReadyToAuthenticate = Promise.defer();
this.whenReadyToAuthenticate.promise.catch(err => {
this.whenReadyToAuthenticate.promise.then(null, (err) => {
this._log.error("Could not authenticate", err);
});
@ -258,14 +240,14 @@ this.BrowserIDManager.prototype = {
Services.obs.notifyObservers(null, "weave:service:setup-complete", null);
Weave.Utils.nextTick(Weave.Service.sync, Weave.Service);
}
}).catch(authErr => {
// report what failed...
this._log.error("Background fetch for key bundle failed", authErr);
}).then(null, err => {
this._shouldHaveSyncKeyBundle = true; // but we probably don't have one...
this.whenReadyToAuthenticate.reject(authErr);
this.whenReadyToAuthenticate.reject(err);
// report what failed...
this._log.error("Background fetch for key bundle failed", err);
});
// and we are done - the fetch continues on in the background...
}).catch(err => {
}).then(null, err => {
this._log.error("Processing logged in account", err);
});
},
@ -301,8 +283,7 @@ this.BrowserIDManager.prototype = {
// reauth with the server - in that case we will also get here, but
// should have the same identity.
// initializeWithCurrentIdentity will throw and log if these constraints
// aren't met (indirectly, via _updateSignedInUser()), so just go ahead
// and do the init.
// aren't met, so just go ahead and do the init.
this.initializeWithCurrentIdentity(true);
break;
@ -311,13 +292,6 @@ this.BrowserIDManager.prototype = {
// startOver will cause this instance to be thrown away, so there's
// nothing else to do.
break;
case fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION:
// throw away token and fetch a new one
this.resetCredentials();
this._ensureValidToken().catch(err =>
this._log.error("Error while fetching a new token", err));
break;
}
},
@ -413,9 +387,6 @@ this.BrowserIDManager.prototype = {
resetCredentials: function() {
this.resetSyncKey();
this._token = null;
// The cluster URL comes from the token, so resetting it to empty will
// force Sync to not accidentally use a value from an earlier token.
Weave.Service.clusterURL = null;
},
/**
@ -503,12 +474,7 @@ this.BrowserIDManager.prototype = {
// If we still can't get keys it probably means the user authenticated
// without unlocking the MP or cleared the saved logins, so we've now
// lost them - the user will need to reauth before continuing.
let result;
if (this._canFetchKeys()) {
result = STATUS_OK;
} else {
result = LOGIN_FAILED_LOGIN_REJECTED;
}
let result = this._canFetchKeys() ? STATUS_OK : LOGIN_FAILED_LOGIN_REJECTED;
log.debug("unlockAndVerifyAuthState re-fetched credentials and is returning", result);
return result;
}
@ -540,27 +506,14 @@ this.BrowserIDManager.prototype = {
return true;
},
// Get our tokenServerURL - a private helper. Returns a string.
get _tokenServerUrl() {
// We used to support services.sync.tokenServerURI but this was a
// pain-point for people using non-default servers as Sync may auto-reset
// all services.sync prefs. So if that still exists, it wins.
let url = Svc.Prefs.get("tokenServerURI"); // Svc.Prefs "root" is services.sync
if (!url) {
url = Services.prefs.getCharPref("identity.sync.tokenserver.uri");
}
while (url.endsWith("/")) { // trailing slashes cause problems...
url = url.slice(0, -1);
}
return url;
},
// Refresh the sync token for our user. Returns a promise that resolves
// with a token (which may be null in one sad edge-case), or rejects with an
// error.
_fetchTokenForUser: function() {
// tokenServerURI is mis-named - convention is uri means nsISomething...
let tokenServerURI = this._tokenServerUrl;
let tokenServerURI = Svc.Prefs.get("tokenServerURI");
if (tokenServerURI.endsWith("/")) { // trailing slashes cause problems...
tokenServerURI = tokenServerURI.slice(0, -1);
}
let log = this._log;
let client = this._tokenServerClient;
let fxa = this._fxaService;
@ -589,7 +542,7 @@ this.BrowserIDManager.prototype = {
);
}
let getToken = assertion => {
let getToken = (tokenServerURI, assertion) => {
log.debug("Getting a token");
let deferred = Promise.defer();
let cb = function (err, token) {
@ -617,18 +570,7 @@ this.BrowserIDManager.prototype = {
return fxa.whenVerified(this._signedInUser)
.then(() => maybeFetchKeys())
.then(() => getAssertion())
.then(assertion => getToken(assertion))
.catch(err => {
// If we get a 401 fetching the token it may be that our certificate
// needs to be regenerated.
if (!err.response || err.response.status !== 401) {
return Promise.reject(err);
}
log.warn("Token server returned 401, refreshing certificate and retrying token fetch");
return fxa.invalidateCertificate()
.then(() => getAssertion())
.then(assertion => getToken(assertion))
})
.then(assertion => getToken(tokenServerURI, assertion))
.then(token => {
// TODO: Make it be only 80% of the duration, so refresh the token
// before it actually expires. This is to avoid sync storage errors
@ -640,18 +582,15 @@ this.BrowserIDManager.prototype = {
}
return token;
})
.catch(err => {
.then(null, err => {
// TODO: unify these errors - we need to handle errors thrown by
// both tokenserverclient and hawkclient.
// A tokenserver error thrown based on a bad response.
if (err.response && err.response.status === 401) {
err = new AuthenticationError(err, "tokenserver");
err = new AuthenticationError(err);
// A hawkclient error.
} else if (err.code && err.code === 401) {
err = new AuthenticationError(err, "hawkclient");
// An FxAccounts.jsm error.
} else if (err.message == fxAccountsCommon.ERROR_AUTH_ERROR) {
err = new AuthenticationError(err, "fxaccounts");
err = new AuthenticationError(err);
}
// TODO: write tests to make sure that different auth error cases are handled here
@ -673,6 +612,7 @@ this.BrowserIDManager.prototype = {
// that there is no authentication dance still under way.
this._shouldHaveSyncKeyBundle = true;
Weave.Status.login = this._authFailureReason;
Services.obs.notifyObservers(null, "weave:ui:login:error", null);
throw err;
});
},
@ -684,19 +624,12 @@ this.BrowserIDManager.prototype = {
this._log.debug("_ensureValidToken already has one");
return Promise.resolve();
}
const notifyStateChanged =
() => Services.obs.notifyObservers(null, "weave:service:login:change", null);
// reset this._token as a safety net to reduce the possibility of us
// repeatedly attempting to use an invalid token if _fetchTokenForUser throws.
this._token = null;
return this._fetchTokenForUser().then(
token => {
this._token = token;
notifyStateChanged();
},
error => {
notifyStateChanged();
throw error
}
);
},
@ -719,16 +652,9 @@ this.BrowserIDManager.prototype = {
_getAuthenticationHeader: function(httpObject, method) {
let cb = Async.makeSpinningCallback();
this._ensureValidToken().then(cb, cb);
// Note that in failure states we return null, causing the request to be
// made without authorization headers, thereby presumably causing a 401,
// which causes Sync to log out. If we throw, this may not happen as
// expected.
try {
cb.wait();
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.error("Failed to fetch a token for authentication", ex);
return null;
}
@ -764,17 +690,8 @@ this.BrowserIDManager.prototype = {
createClusterManager: function(service) {
return new BrowserIDClusterManager(service);
},
}
// Tell Sync what the login status should be if it saw a 401 fetching
// info/collections as part of login verification (typically immediately
// after login.)
// In our case, it almost certainly means a transient error fetching a token
// (and hitting this will cause us to logout, which will correctly handle an
// authoritative login issue.)
loginStatusFromVerification404() {
return LOGIN_FAILED_NETWORK_ERROR;
},
};
/* An implementation of the ClusterManager for this identity
@ -820,7 +737,7 @@ BrowserIDClusterManager.prototype = {
// it's likely a 401 was received using the existing token - in which
// case we just discard the existing token and fetch a new one.
if (this.service.clusterURL) {
log.debug("_findCluster has a pre-existing clusterURL, so discarding the current token");
log.debug("_findCluster found existing clusterURL, so discarding the current token");
this.identity._token = null;
}
return this.identity._ensureValidToken();

204
services/sync/modules/collection_validator.js
View file

@ -1,204 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const Cu = Components.utils;
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/main.js");
this.EXPORTED_SYMBOLS = ["CollectionValidator", "CollectionProblemData"];
class CollectionProblemData {
constructor() {
this.missingIDs = 0;
this.duplicates = [];
this.clientMissing = [];
this.serverMissing = [];
this.serverDeleted = [];
this.serverUnexpected = [];
this.differences = [];
}
/**
* Produce a list summarizing problems found. Each entry contains {name, count},
* where name is the field name for the problem, and count is the number of times
* the problem was encountered.
*
* Validation has failed if all counts are not 0.
*/
getSummary() {
return [
{ name: "clientMissing", count: this.clientMissing.length },
{ name: "serverMissing", count: this.serverMissing.length },
{ name: "serverDeleted", count: this.serverDeleted.length },
{ name: "serverUnexpected", count: this.serverUnexpected.length },
{ name: "differences", count: this.differences.length },
{ name: "missingIDs", count: this.missingIDs },
{ name: "duplicates", count: this.duplicates.length }
];
}
}
class CollectionValidator {
// Construct a generic collection validator. This is intended to be called by
// subclasses.
// - name: Name of the engine
// - idProp: Property that identifies a record. That is, if a client and server
// record have the same value for the idProp property, they should be
// compared against eachother.
// - props: Array of properties that should be compared
constructor(name, idProp, props) {
this.name = name;
this.props = props;
this.idProp = idProp;
}
// Should a custom ProblemData type be needed, return it here.
emptyProblemData() {
return new CollectionProblemData();
}
getServerItems(engine) {
let collection = engine.itemSource();
let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name);
collection.full = true;
let items = [];
collection.recordHandler = function(item) {
item.decrypt(collectionKey);
items.push(item.cleartext);
};
let resp = collection.getBatched();
if (!resp.success) {
throw resp;
}
return items;
}
// Should return a promise that resolves to an array of client items.
getClientItems() {
return Promise.reject("Must implement");
}
// Turn the client item into something that can be compared with the server item,
// and is also safe to mutate.
normalizeClientItem(item) {
return Cu.cloneInto(item, {});
}
// Turn the server item into something that can be easily compared with the client
// items.
normalizeServerItem(item) {
return item;
}
// Return whether or not a server item should be present on the client. Expected
// to be overridden.
clientUnderstands(item) {
return true;
}
// Return whether or not a client item should be present on the server. Expected
// to be overridden
syncedByClient(item) {
return true;
}
// Compare the server item and the client item, and return a list of property
// names that are different. Can be overridden if needed.
getDifferences(client, server) {
let differences = [];
for (let prop of this.props) {
let clientProp = client[prop];
let serverProp = server[prop];
if ((clientProp || "") !== (serverProp || "")) {
differences.push(prop);
}
}
return differences;
}
// Returns an object containing
// problemData: an instance of the class returned by emptyProblemData(),
// clientRecords: Normalized client records
// records: Normalized server records,
// deletedRecords: Array of ids that were marked as deleted by the server.
compareClientWithServer(clientItems, serverItems) {
clientItems = clientItems.map(item => this.normalizeClientItem(item));
serverItems = serverItems.map(item => this.normalizeServerItem(item));
let problems = this.emptyProblemData();
let seenServer = new Map();
let serverDeleted = new Set();
let allRecords = new Map();
for (let record of serverItems) {
let id = record[this.idProp];
if (!id) {
++problems.missingIDs;
continue;
}
if (record.deleted) {
serverDeleted.add(record);
} else {
let possibleDupe = seenServer.get(id);
if (possibleDupe) {
problems.duplicates.push(id);
} else {
seenServer.set(id, record);
allRecords.set(id, { server: record, client: null, });
}
record.understood = this.clientUnderstands(record);
}
}
let recordPairs = [];
let seenClient = new Map();
for (let record of clientItems) {
let id = record[this.idProp];
record.shouldSync = this.syncedByClient(record);
seenClient.set(id, record);
let combined = allRecords.get(id);
if (combined) {
combined.client = record;
} else {
allRecords.set(id, { client: record, server: null });
}
}
for (let [id, { server, client }] of allRecords) {
if (!client && !server) {
throw new Error("Impossible: no client or server record for " + id);
} else if (server && !client) {
if (server.understood) {
problems.clientMissing.push(id);
}
} else if (client && !server) {
if (client.shouldSync) {
problems.serverMissing.push(id);
}
} else {
if (!client.shouldSync) {
if (!problems.serverUnexpected.includes(id)) {
problems.serverUnexpected.push(id);
}
continue;
}
let differences = this.getDifferences(client, server);
if (differences && differences.length) {
problems.differences.push({ id, differences });
}
}
}
return {
problemData: problems,
clientRecords: clientItems,
records: serverItems,
deletedRecords: [...serverDeleted]
};
}
}
// Default to 0, some engines may override.
CollectionValidator.prototype.version = 0;

31
services/sync/modules/constants.js
View file

@ -4,8 +4,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Process each item in the "constants hash" to add to "global" and give a name
this.EXPORTED_SYMBOLS = [];
for (let [key, val] of Object.entries({
this.EXPORTED_SYMBOLS = [((this[key] = val), key) for ([key, val] in Iterator({
WEAVE_VERSION: "@weave_version@",
@ -45,7 +44,7 @@ MAX_IGNORE_ERROR_COUNT: 5,
// Backoff intervals
MINIMUM_BACKOFF_INTERVAL: 15 * 60 * 1000, // 15 minutes
MAXIMUM_BACKOFF_INTERVAL: 8 * 60 * 60 * 1000, // 8 hours
MAXIMUM_BACKOFF_INTERVAL: 8 * 60 * 60 * 1000, // 8 hours
// HMAC event handling timeout.
// 10 minutes: a compromise between the multi-desktop sync interval
@ -76,10 +75,6 @@ PASSWORDS_STORE_BATCH_SIZE: 50, // same as MOBILE_BATCH_SIZE
ADDONS_STORE_BATCH_SIZE: 1000000, // process all addons at once
APPS_STORE_BATCH_SIZE: 50, // same as MOBILE_BATCH_SIZE
// Default batch size for download batching
// (how many records are fetched at a time from the server when batching is used).
DEFAULT_DOWNLOAD_BATCH_SIZE: 1000,
// score thresholds for early syncs
SINGLE_USER_THRESHOLD: 1000,
MULTI_DEVICE_THRESHOLD: 300,
@ -98,16 +93,13 @@ SCORE_UPDATE_DELAY: 100,
// observed spurious idle/back events and short enough to pre-empt user activity.
IDLE_OBSERVER_BACK_DELAY: 100,
// Max number of records or bytes to upload in a single POST - we'll do multiple POSTS if either
// MAX_UPLOAD_RECORDS or MAX_UPLOAD_BYTES is hit)
// Number of records to upload in a single POST (multiple POSTS if exceeded)
// FIXME: Record size limit is 256k (new cluster), so this can be quite large!
// (Bug 569295)
MAX_UPLOAD_RECORDS: 100,
MAX_UPLOAD_BYTES: 1024 * 1023, // just under 1MB
MAX_HISTORY_UPLOAD: 5000,
MAX_HISTORY_DOWNLOAD: 5000,
// TTL of the message sent to another device when sending a tab
NOTIFY_TAB_SENT_TTL_SECS: 1 * 3600, // 1 hour
// Top-level statuses:
STATUS_OK: "success.status_ok",
SYNC_FAILED: "error.sync.failed",
@ -130,6 +122,7 @@ LOGIN_FAILED_NETWORK_ERROR: "error.login.reason.network",
LOGIN_FAILED_SERVER_ERROR: "error.login.reason.server",
LOGIN_FAILED_INVALID_PASSPHRASE: "error.login.reason.recoverykey",
LOGIN_FAILED_LOGIN_REJECTED: "error.login.reason.account",
LOGIN_FAILED_NOT_READY: "error.login.reason.initializing",
// sync failure status codes
METARECORD_DOWNLOAD_FAIL: "error.sync.reason.metarecord_download_fail",
@ -152,8 +145,6 @@ ENGINE_UNKNOWN_FAIL: "error.engine.reason.unknown_fail",
ENGINE_APPLY_FAIL: "error.engine.reason.apply_fail",
ENGINE_METARECORD_DOWNLOAD_FAIL: "error.engine.reason.metarecord_download_fail",
ENGINE_METARECORD_UPLOAD_FAIL: "error.engine.reason.metarecord_upload_fail",
// an upload failure where the batch was interrupted with a 412
ENGINE_BATCH_INTERRUPTED: "error.engine.reason.batch_interrupted",
JPAKE_ERROR_CHANNEL: "jpake.error.channel",
JPAKE_ERROR_NETWORK: "jpake.error.network",
@ -181,7 +172,7 @@ kSyncBackoffNotMet: "Trying to sync before the server said it
kFirstSyncChoiceNotMade: "User has not selected an action for first sync",
// Application IDs
FIREFOX_ID: "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
FIREFOX_ID: "{8de7fcbb-c55c-4fbe-bfc5-fc555c87dbc4}",
FENNEC_ID: "{a23983c0-fd0e-11dc-95ff-0800200c9a66}",
SEAMONKEY_ID: "{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}",
TEST_HARNESS_ID: "xuth@mozilla.org",
@ -189,10 +180,6 @@ TEST_HARNESS_ID: "xuth@mozilla.org",
MIN_PP_LENGTH: 12,
MIN_PASS_LENGTH: 8,
DEVICE_TYPE_DESKTOP: "desktop",
DEVICE_TYPE_MOBILE: "mobile",
LOG_DATE_FORMAT: "%Y-%m-%d %H:%M:%S",
})) {
this[key] = val;
this.EXPORTED_SYMBOLS.push(key);
}
}))];

476
services/sync/modules/engines.js
View file

@ -7,24 +7,21 @@ this.EXPORTED_SYMBOLS = [
"Engine",
"SyncEngine",
"Tracker",
"Store",
"Changeset"
"Store"
];
var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/observers.js");
Cu.import("resource://services-common/utils.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/identity.js");
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/resource.js");
Cu.import("resource://services-sync/util.js");
XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
"resource://gre/modules/FxAccounts.jsm");
/*
* Trackers are associated with a single engine and deal with
* listening for changes to their particular data type.
@ -107,7 +104,7 @@ Tracker.prototype = {
Utils.jsonLoad("changes/" + this.file, this, function(json) {
if (json && (typeof(json) == "object")) {
this.changedIDs = json;
} else if (json !== null) {
} else {
this._log.warn("Changed IDs file " + this.file + " contains non-object value.");
json = null;
}
@ -132,30 +129,26 @@ Tracker.prototype = {
this._ignored.splice(index, 1);
},
_saveChangedID(id, when) {
this._log.trace(`Adding changed ID: ${id}, ${JSON.stringify(when)}`);
this.changedIDs[id] = when;
this.saveChangedIDs(this.onSavedChangedIDs);
},
addChangedID: function (id, when) {
if (!id) {
this._log.warn("Attempted to add undefined ID to tracker");
return false;
}
if (this.ignoreAll || this._ignored.includes(id)) {
if (this.ignoreAll || (id in this._ignored)) {
return false;
}
// Default to the current time in seconds if no time is provided.
if (when == null) {
when = this._now();
when = Math.floor(Date.now() / 1000);
}
// Add/update the entry if we have a newer time.
if ((this.changedIDs[id] || -Infinity) < when) {
this._saveChangedID(id, when);
this._log.trace("Adding changed ID: " + id + ", " + when);
this.changedIDs[id] = when;
this.saveChangedIDs(this.onSavedChangedIDs);
}
return true;
@ -166,9 +159,8 @@ Tracker.prototype = {
this._log.warn("Attempted to remove undefined ID to tracker");
return false;
}
if (this.ignoreAll || this._ignored.includes(id)) {
if (this.ignoreAll || (id in this._ignored))
return false;
}
if (this.changedIDs[id] != null) {
this._log.trace("Removing changed ID " + id);
delete this.changedIDs[id];
@ -183,10 +175,6 @@ Tracker.prototype = {
this.saveChangedIDs();
},
_now() {
return Date.now() / 1000;
},
_isTracking: false,
// Override these in your subclasses.
@ -311,21 +299,17 @@ Store.prototype = {
*/
applyIncomingBatch: function (records) {
let failed = [];
for (let record of records) {
for each (let record in records) {
try {
this.applyIncoming(record);
} catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
// This kind of exception should have a 'cause' attribute, which is an
// originating exception.
// ex.cause will carry its stack with it when rethrown.
throw ex.cause;
} catch (ex) {
if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) {
// This kind of exception should have a 'cause' attribute, which is an
// originating exception.
// ex.cause will carry its stack with it when rethrown.
throw ex.cause;
}
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("Failed to apply incoming record " + record.id, ex);
this.engine._noteApplyFailure();
this._log.warn("Failed to apply incoming record " + record.id);
this._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
failed.push(record.id);
}
};
@ -499,11 +483,7 @@ EngineManager.prototype = {
},
getAll: function () {
let engines = [];
for (let [, engine] of Object.entries(this._engines)) {
engines.push(engine);
}
return engines;
return [engine for ([name, engine] in Iterator(this._engines))];
},
/**
@ -516,7 +496,7 @@ EngineManager.prototype = {
},
get enabledEngineNames() {
return this.getEnabled().map(e => e.name);
return [e.name for each (e in this.getEnabled())];
},
persistDeclined: function () {
@ -593,11 +573,16 @@ EngineManager.prototype = {
this._engines[name] = engine;
}
} catch (ex) {
this._log.error(CommonUtils.exceptionStr(ex));
let mesg = ex.message ? ex.message : ex;
let name = engineObject || "";
name = name.prototype || "";
name = name.name || "";
this._log.error(`Could not initialize engine ${name}`, ex);
let out = "Could not initialize engine '" + name + "': " + mesg;
this._log.error(out);
return engineObject;
}
},
@ -643,17 +628,16 @@ Engine.prototype = {
// Signal to the engine that processing further records is pointless.
eEngineAbortApplyIncoming: "error.engine.abort.applyincoming",
// Should we keep syncing if we find a record that cannot be uploaded (ever)?
// If this is false, we'll throw, otherwise, we'll ignore the record and
// continue. This currently can only happen due to the record being larger
// than the record upload limit.
allowSkippedRecord: true,
get prefName() {
return this.name;
},
get enabled() {
// XXX: Disable non-functional add-ons syncing for the time being
// This check can go away when add-on syncing is addressed
if (this.prefName == "addons")
return false;
return Svc.Prefs.get("engine." + this.prefName, false);
},
@ -711,15 +695,6 @@ Engine.prototype = {
wipeClient: function () {
this._notify("wipe-client", this.name, this._wipeClient)();
},
/**
* If one exists, initialize and return a validator for this engine (which
* must have a `validate(engine)` method that returns a promise to an object
* with a getSummary method). Otherwise return null.
*/
getValidator: function () {
return null;
}
};
@ -813,11 +788,7 @@ SyncEngine.prototype = {
return this._toFetch;
},
set toFetch(val) {
let cb = (error) => {
if (error) {
this._log.error("Failed to read JSON records to fetch", error);
}
}
let cb = (error) => this._log.error(Utils.exceptionStr(error));
// Coerce the array to a string for more efficient comparison.
if (val + "" == this._toFetch) {
return;
@ -842,13 +813,7 @@ SyncEngine.prototype = {
return this._previousFailed;
},
set previousFailed(val) {
let cb = (error) => {
if (error) {
this._log.error("Failed to set previousFailed", error);
} else {
this._log.debug("Successfully wrote previousFailed.");
}
}
let cb = (error) => this._log.error(Utils.exceptionStr(error));
// Coerce the array to a string for more efficient comparison.
if (val + "" == this._previousFailed) {
return;
@ -881,8 +846,9 @@ SyncEngine.prototype = {
},
/*
* Returns a changeset for this sync. Engine implementations can override this
* method to bypass the tracker for certain or all changed items.
* Returns a mapping of IDs -> changed timestamp. Engine implementations
* can override this method to bypass the tracker for certain or all
* changed items.
*/
getChangedIDs: function () {
return this._tracker.changedIDs;
@ -950,16 +916,20 @@ SyncEngine.prototype = {
// this._modified to the tracker.
this.lastSyncLocal = Date.now();
if (this.lastSync) {
this._modified = this.pullNewChanges();
this._modified = this.getChangedIDs();
} else {
// Mark all items to be uploaded, but treat them as changed from long ago
this._log.debug("First sync, uploading all items");
this._modified = this.pullAllChanges();
this._modified = {};
for (let id in this._store.getAllIDs()) {
this._modified[id] = 0;
}
}
// Clear the tracker now. If the sync fails we'll add the ones we failed
// to upload back.
this._tracker.clearChangedIDs();
this._log.info(this._modified.count() +
this._log.info(Object.keys(this._modified).length +
" outgoing items pre-reconciliation");
// Keep track of what to delete at the end of sync
@ -970,7 +940,7 @@ SyncEngine.prototype = {
* A tiny abstraction to make it easier to test incoming record
* application.
*/
itemSource: function () {
_itemSource: function () {
return new Collection(this.engineURL, this._recordObj, this.service);
},
@ -987,7 +957,7 @@ SyncEngine.prototype = {
let isMobile = (Svc.Prefs.get("client.type") == "mobile");
if (!newitems) {
newitems = this.itemSource();
newitems = this._itemSource();
}
if (this._defaultSort) {
@ -1024,12 +994,10 @@ SyncEngine.prototype = {
try {
failed = failed.concat(this._store.applyIncomingBatch(applyBatch));
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
// Catch any error that escapes from applyIncomingBatch. At present
// those will all be abort events.
this._log.warn("Got exception, aborting processIncoming", ex);
this._log.warn("Got exception " + Utils.exceptionStr(ex) +
", aborting processIncoming.");
aborting = ex;
}
this._tracker.ignoreAll = false;
@ -1074,10 +1042,7 @@ SyncEngine.prototype = {
try {
try {
item.decrypt(key);
} catch (ex) {
if (!Utils.isHMACMismatch(ex)) {
throw ex;
}
} catch (ex if Utils.isHMACMismatch(ex)) {
let strategy = self.handleHMACMismatch(item, true);
if (strategy == SyncEngine.kRecoveryStrategy.retry) {
// You only get one retry.
@ -1087,10 +1052,7 @@ SyncEngine.prototype = {
key = self.service.collectionKeys.keyForCollection(self.name);
item.decrypt(key);
strategy = null;
} catch (ex) {
if (!Utils.isHMACMismatch(ex)) {
throw ex;
}
} catch (ex if Utils.isHMACMismatch(ex)) {
strategy = self.handleHMACMismatch(item, false);
}
}
@ -1103,8 +1065,7 @@ SyncEngine.prototype = {
self._log.debug("Ignoring second retry suggestion.");
// Fall through to error case.
case SyncEngine.kRecoveryStrategy.error:
self._log.warn("Error decrypting record", ex);
self._noteApplyFailure();
self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
failed.push(item.id);
return;
case SyncEngine.kRecoveryStrategy.ignore:
@ -1114,11 +1075,7 @@ SyncEngine.prototype = {
}
}
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
self._log.warn("Error decrypting record", ex);
self._noteApplyFailure();
self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
failed.push(item.id);
return;
}
@ -1126,20 +1083,15 @@ SyncEngine.prototype = {
let shouldApply;
try {
shouldApply = self._reconcile(item);
} catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
self._log.warn("Reconciliation failed: aborting incoming processing.");
failed.push(item.id);
aborting = ex.cause;
} catch (ex) {
if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) {
self._log.warn("Reconciliation failed: aborting incoming processing.");
self._noteApplyFailure();
failed.push(item.id);
aborting = ex.cause;
} else if (!Async.isShutdownException(ex)) {
self._log.warn("Failed to reconcile incoming record " + item.id, ex);
self._noteApplyFailure();
failed.push(item.id);
return;
} else {
throw ex;
}
self._log.warn("Failed to reconcile incoming record " + item.id);
self._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
failed.push(item.id);
return;
}
if (shouldApply) {
@ -1158,7 +1110,7 @@ SyncEngine.prototype = {
// Only bother getting data from the server if there's new things
if (this.lastModified == null || this.lastModified > this.lastSync) {
let resp = newitems.getBatched();
let resp = newitems.get();
doApplyBatchAndPersistFailed.call(this);
if (!resp.success) {
resp.failureCode = ENGINE_DOWNLOAD_FAIL;
@ -1243,13 +1195,7 @@ SyncEngine.prototype = {
// Apply remaining items.
doApplyBatchAndPersistFailed.call(this);
count.newFailed = this.previousFailed.reduce((count, engine) => {
if (failedInPreviousSync.indexOf(engine) == -1) {
count++;
this._noteApplyNewFailure();
}
return count;
}, 0);
count.newFailed = Utils.arraySub(this.previousFailed, failedInPreviousSync).length;
count.succeeded = Math.max(0, count.applied - count.failed);
this._log.info(["Records:",
count.applied, "applied,",
@ -1260,14 +1206,6 @@ SyncEngine.prototype = {
Observers.notify("weave:engine:sync:applied", count, this.name);
},
_noteApplyFailure: function () {
// here would be a good place to record telemetry...
},
_noteApplyNewFailure: function () {
// here would be a good place to record telemetry...
},
/**
* Find a GUID of an item that is a duplicate of the incoming item but happens
* to have a different GUID
@ -1278,16 +1216,6 @@ SyncEngine.prototype = {
// By default, assume there's no dupe items for the engine
},
// Called when the server has a record marked as deleted, but locally we've
// changed it more recently than the deletion. If we return false, the
// record will be deleted locally. If we return true, we'll reupload the
// record to the server -- any extra work that's needed as part of this
// process should be done at this point (such as mark the record's parent
// for reuploading in the case of bookmarks).
_shouldReviveRemotelyDeletedRecord(remoteItem) {
return true;
},
_deleteId: function (id) {
this._tracker.removeChangedID(id);
@ -1298,18 +1226,6 @@ SyncEngine.prototype = {
this._delete.ids.push(id);
},
_switchItemToDupe(localDupeGUID, incomingItem) {
// The local, duplicate ID is always deleted on the server.
this._deleteId(localDupeGUID);
// We unconditionally change the item's ID in case the engine knows of
// an item but doesn't expose it through itemExists. If the API
// contract were stronger, this could be changed.
this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " +
incomingItem.id);
this._store.changeItemID(localDupeGUID, incomingItem.id);
},
/**
* Reconcile incoming record with local state.
*
@ -1329,12 +1245,12 @@ SyncEngine.prototype = {
// because some state may change during the course of this function and we
// need to operate on the original values.
let existsLocally = this._store.itemExists(item.id);
let locallyModified = this._modified.has(item.id);
let locallyModified = item.id in this._modified;
// TODO Handle clock drift better. Tracked in bug 721181.
let remoteAge = AsyncResource.serverTime - item.modified;
let localAge = locallyModified ?
(Date.now() / 1000 - this._modified.getModifiedTimestamp(item.id)) : null;
(Date.now() / 1000 - this._modified[item.id]) : null;
let remoteIsNewer = remoteAge < localAge;
this._log.trace("Reconciling " + item.id + ". exists=" +
@ -1363,18 +1279,15 @@ SyncEngine.prototype = {
"exists and isn't modified.");
return true;
}
this._log.trace("Incoming record is deleted but we had local changes.");
if (remoteIsNewer) {
this._log.trace("Remote record is newer -- deleting local record.");
return true;
}
// If the local record is newer, we defer to individual engines for
// how to handle this. By default, we revive the record.
let willRevive = this._shouldReviveRemotelyDeletedRecord(item);
this._log.trace("Local record is newer -- reviving? " + willRevive);
return !willRevive;
// TODO As part of bug 720592, determine whether we should do more here.
// In the case where the local changes are newer, it is quite possible
// that the local client will restore data a remote client had tried to
// delete. There might be a good reason for that delete and it might be
// enexpected for this client to restore that data.
this._log.trace("Incoming record is deleted but we had local changes. " +
"Applying the youngest record.");
return remoteIsNewer;
}
// At this point the incoming record is not for a deletion and must have
@ -1386,32 +1299,40 @@ SyncEngine.prototype = {
// refresh the metadata collected above. See bug 710448 for the history
// of this logic.
if (!existsLocally) {
let localDupeGUID = this._findDupe(item);
if (localDupeGUID) {
this._log.trace("Local item " + localDupeGUID + " is a duplicate for " +
let dupeID = this._findDupe(item);
if (dupeID) {
this._log.trace("Local item " + dupeID + " is a duplicate for " +
"incoming item " + item.id);
// The local, duplicate ID is always deleted on the server.
this._deleteId(dupeID);
// The current API contract does not mandate that the ID returned by
// _findDupe() actually exists. Therefore, we have to perform this
// check.
existsLocally = this._store.itemExists(localDupeGUID);
existsLocally = this._store.itemExists(dupeID);
// We unconditionally change the item's ID in case the engine knows of
// an item but doesn't expose it through itemExists. If the API
// contract were stronger, this could be changed.
this._log.debug("Switching local ID to incoming: " + dupeID + " -> " +
item.id);
this._store.changeItemID(dupeID, item.id);
// If the local item was modified, we carry its metadata forward so
// appropriate reconciling can be performed.
if (this._modified.has(localDupeGUID)) {
if (dupeID in this._modified) {
locallyModified = true;
localAge = this._tracker._now() - this._modified.getModifiedTimestamp(localDupeGUID);
localAge = Date.now() / 1000 - this._modified[dupeID];
remoteIsNewer = remoteAge < localAge;
this._modified.swap(localDupeGUID, item.id);
this._modified[item.id] = this._modified[dupeID];
delete this._modified[dupeID];
} else {
locallyModified = false;
localAge = null;
}
// Tell the engine to do whatever it needs to switch the items.
this._switchItemToDupe(localDupeGUID, item);
this._log.debug("Local item after duplication: age=" + localAge +
"; modified=" + locallyModified + "; exists=" +
existsLocally);
@ -1440,7 +1361,7 @@ SyncEngine.prototype = {
if (remoteIsNewer) {
this._log.trace("Applying incoming because local item was deleted " +
"before the incoming item was changed.");
this._modified.delete(item.id);
delete this._modified[item.id];
return true;
}
@ -1466,7 +1387,7 @@ SyncEngine.prototype = {
this._log.trace("Ignoring incoming item because the local item is " +
"identical.");
this._modified.delete(item.id);
delete this._modified[item.id];
return false;
}
@ -1491,97 +1412,69 @@ SyncEngine.prototype = {
_uploadOutgoing: function () {
this._log.trace("Uploading local changes to server.");
let modifiedIDs = this._modified.ids();
let modifiedIDs = Object.keys(this._modified);
if (modifiedIDs.length) {
this._log.trace("Preparing " + modifiedIDs.length +
" outgoing records");
let counts = { sent: modifiedIDs.length, failed: 0 };
// collection we'll upload
let up = new Collection(this.engineURL, null, this.service);
let count = 0;
let failed = [];
let successful = [];
let handleResponse = (resp, batchOngoing = false) => {
// Note: We don't want to update this.lastSync, or this._modified until
// the batch is complete, however we want to remember success/failure
// indicators for when that happens.
// Upload what we've got so far in the collection
let doUpload = Utils.bind2(this, function(desc) {
this._log.info("Uploading " + desc + " of " + modifiedIDs.length +
" records");
let resp = up.post();
if (!resp.success) {
this._log.debug("Uploading records failed: " + resp);
resp.failureCode = resp.status == 412 ? ENGINE_BATCH_INTERRUPTED : ENGINE_UPLOAD_FAIL;
resp.failureCode = ENGINE_UPLOAD_FAIL;
throw resp;
}
// Update server timestamp from the upload.
failed = failed.concat(Object.keys(resp.obj.failed));
successful = successful.concat(resp.obj.success);
if (batchOngoing) {
// Nothing to do yet
return;
}
// Advance lastSync since we've finished the batch.
let modified = resp.headers["x-weave-timestamp"];
if (modified > this.lastSync) {
if (modified > this.lastSync)
this.lastSync = modified;
}
if (failed.length && this._log.level <= Log.Level.Debug) {
let failed_ids = Object.keys(resp.obj.failed);
if (failed_ids.length)
this._log.debug("Records that will be uploaded again because "
+ "the server couldn't store them: "
+ failed.join(", "));
+ failed_ids.join(", "));
// Clear successfully uploaded objects.
for each (let id in resp.obj.success) {
delete this._modified[id];
}
counts.failed += failed.length;
up.clearRecords();
});
for (let id of successful) {
this._modified.delete(id);
}
this._onRecordsWritten(successful, failed);
// clear for next batch
failed.length = 0;
successful.length = 0;
};
let postQueue = up.newPostQueue(this._log, this.lastSync, handleResponse);
for (let id of modifiedIDs) {
let out;
let ok = false;
for each (let id in modifiedIDs) {
try {
out = this._createRecord(id);
let out = this._createRecord(id);
if (this._log.level <= Log.Level.Trace)
this._log.trace("Outgoing: " + out);
out.encrypt(this.service.collectionKeys.keyForCollection(this.name));
ok = true;
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("Error creating record", ex);
up.pushData(out);
}
if (ok) {
let { enqueued, error } = postQueue.enqueue(out);
if (!enqueued) {
++counts.failed;
if (!this.allowSkippedRecord) {
throw error;
}
}
catch(ex) {
this._log.warn("Error creating record: " + Utils.exceptionStr(ex));
}
// Partial upload
if ((++count % MAX_UPLOAD_RECORDS) == 0)
doUpload((count - MAX_UPLOAD_RECORDS) + " - " + count + " out");
this._store._sleep(0);
}
postQueue.flush(true);
Observers.notify("weave:engine:sync:uploaded", counts, this.name);
}
},
_onRecordsWritten(succeeded, failed) {
// Implement this method to take specific actions against successfully
// uploaded records and failed records.
// Final upload
if (count % MAX_UPLOAD_RECORDS > 0)
doUpload(count >= MAX_UPLOAD_RECORDS ? "last batch" : "all");
}
},
// Any cleanup necessary.
@ -1596,7 +1489,7 @@ SyncEngine.prototype = {
coll.delete();
});
for (let [key, val] of Object.entries(this._delete)) {
for (let [key, val] in Iterator(this._delete)) {
// Remove the key for future uses
delete this._delete[key];
@ -1619,8 +1512,10 @@ SyncEngine.prototype = {
}
// Mark failed WBOs as changed again so they are reuploaded next time.
this.trackRemainingChanges();
this._modified.clear();
for (let [id, when] in Iterator(this._modified)) {
this._tracker.addChangedID(id, when);
}
this._modified = {};
},
_sync: function () {
@ -1656,11 +1551,9 @@ SyncEngine.prototype = {
try {
this._log.trace("Trying to decrypt a record from the server..");
test.get();
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.debug("Failed test decrypt", ex);
}
catch(ex) {
this._log.debug("Failed test decrypt: " + Utils.exceptionStr(ex));
}
return canDecrypt;
@ -1706,108 +1599,5 @@ SyncEngine.prototype = {
return (this.service.handleHMACEvent() && mayRetry) ?
SyncEngine.kRecoveryStrategy.retry :
SyncEngine.kRecoveryStrategy.error;
},
/**
* Returns a changeset containing all items in the store. The default
* implementation returns a changeset with timestamps from long ago, to
* ensure we always use the remote version if one exists.
*
* This function is only called for the first sync. Subsequent syncs call
* `pullNewChanges`.
*
* @return A `Changeset` object.
*/
pullAllChanges() {
let changeset = new Changeset();
for (let id in this._store.getAllIDs()) {
changeset.set(id, 0);
}
return changeset;
},
/*
* Returns a changeset containing entries for all currently tracked items.
* The default implementation returns a changeset with timestamps indicating
* when the item was added to the tracker.
*
* @return A `Changeset` object.
*/
pullNewChanges() {
return new Changeset(this.getChangedIDs());
},
/**
* Adds all remaining changeset entries back to the tracker, typically for
* items that failed to upload. This method is called at the end of each sync.
*
*/
trackRemainingChanges() {
for (let [id, change] of this._modified.entries()) {
this._tracker.addChangedID(id, change);
}
},
}
};
/**
* A changeset is created for each sync in `Engine::get{Changed, All}IDs`,
* and stores opaque change data for tracked IDs. The default implementation
* only records timestamps, though engines can extend this to store additional
* data for each entry.
*/
class Changeset {
// Creates a changeset with an initial set of tracked entries.
constructor(changes = {}) {
this.changes = changes;
}
// Returns the last modified time, in seconds, for an entry in the changeset.
// `id` is guaranteed to be in the set.
getModifiedTimestamp(id) {
return this.changes[id];
}
// Adds a change for a tracked ID to the changeset.
set(id, change) {
this.changes[id] = change;
}
// Indicates whether an entry is in the changeset.
has(id) {
return id in this.changes;
}
// Deletes an entry from the changeset. Used to clean up entries for
// reconciled and successfully uploaded records.
delete(id) {
delete this.changes[id];
}
// Swaps two entries in the changeset. Used when reconciling duplicates that
// have local changes.
swap(oldID, newID) {
this.changes[newID] = this.changes[oldID];
delete this.changes[oldID];
}
// Returns an array of all tracked IDs in this changeset.
ids() {
return Object.keys(this.changes);
}
// Returns an array of `[id, change]` tuples. Used to repopulate the tracker
// with entries for failed uploads at the end of a sync.
entries() {
return Object.entries(this.changes);
}
// Returns the number of entries in this changeset.
count() {
return this.ids().length;
}
// Clears the changeset.
clear() {
this.changes = {};
}
}

140
services/sync/modules/engines/addons.js
View file

@ -25,18 +25,15 @@
*
* Synchronization is influenced by the following preferences:
*
* - services.sync.addons.ignoreRepositoryChecking
* - services.sync.addons.ignoreUserEnabledChanges
* - services.sync.addons.trustedSourceHostnames
*
* and also influenced by whether addons have repository caching enabled and
* whether they allow installation of addons from insecure options (both of
* which are themselves influenced by the "extensions." pref branch)
*
* See the documentation in services-sync.js for the behavior of these prefs.
*/
"use strict";
var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://services-sync/addonutils.js");
Cu.import("resource://services-sync/addonsreconciler.js");
@ -44,7 +41,6 @@ Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/collection_validator.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Preferences.jsm");
@ -54,7 +50,7 @@ XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository",
"resource://gre/modules/addons/AddonRepository.jsm");
this.EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"];
this.EXPORTED_SYMBOLS = ["AddonsEngine"];
// 7 days in milliseconds.
const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000;
@ -154,7 +150,7 @@ AddonsEngine.prototype = {
*/
getChangedIDs: function getChangedIDs() {
let changes = {};
for (let [id, modified] of Object.entries(this._tracker.changedIDs)) {
for (let [id, modified] in Iterator(this._tracker.changedIDs)) {
changes[id] = modified;
}
@ -164,7 +160,7 @@ AddonsEngine.prototype = {
// we assume this function is only called from within a sync.
let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate);
let addons = this._reconciler.addons;
for (let change of reconcilerChanges) {
for each (let change in reconcilerChanges) {
let changeTime = change[0];
let id = change[2];
@ -177,7 +173,7 @@ AddonsEngine.prototype = {
continue;
}
if (!this.isAddonSyncable(addons[id])) {
if (!this._store.isAddonSyncable(addons[id])) {
continue;
}
@ -235,10 +231,6 @@ AddonsEngine.prototype = {
let cb = Async.makeSpinningCallback();
this._reconciler.refreshGlobalState(cb);
cb.wait();
},
isAddonSyncable(addon, ignoreRepoCheck) {
return this._store.isAddonSyncable(addon, ignoreRepoCheck);
}
};
@ -286,14 +278,6 @@ AddonsStore.prototype = {
}
}
// Ignore incoming records for which an existing non-syncable addon
// exists.
let existingMeta = this.reconciler.addons[record.addonID];
if (existingMeta && !this.isAddonSyncable(existingMeta)) {
this._log.info("Ignoring incoming record for an existing but non-syncable addon", record.addonID);
return;
}
Store.prototype.applyIncoming.call(this, record);
},
@ -307,23 +291,15 @@ AddonsStore.prototype = {
id: record.addonID,
syncGUID: record.id,
enabled: record.enabled,
requireSecureURI: this._extensionsPrefs.get("install.requireSecureOrigin", true),
requireSecureURI: !Svc.Prefs.get("addons.ignoreRepositoryChecking", false),
}], cb);
// This will throw if there was an error. This will get caught by the sync
// engine and the record will try to be applied later.
let results = cb.wait();
if (results.skipped.includes(record.addonID)) {
this._log.info("Add-on skipped: " + record.addonID);
// Just early-return for skipped addons - we don't want to arrange to
// try again next time because the condition that caused up to skip
// will remain true for this addon forever.
return;
}
let addon;
for (let a of results.addons) {
for each (let a in results.addons) {
if (a.id == record.addonID) {
addon = a;
break;
@ -467,8 +443,7 @@ AddonsStore.prototype = {
let ids = {};
let addons = this.reconciler.addons;
for (let id in addons) {
let addon = addons[id];
for each (let addon in addons) {
if (this.isAddonSyncable(addon)) {
ids[addon.guid] = true;
}
@ -499,7 +474,7 @@ AddonsStore.prototype = {
}
this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
Utils.catch.call(this, () => addon.uninstall())();
Utils.catch(addon.uninstall)();
}
},
@ -538,22 +513,16 @@ AddonsStore.prototype = {
*
* @param addon
* Addon instance
* @param ignoreRepoCheck
* Should we skip checking the Addons repository (primarially useful
* for testing and validation).
* @return Boolean indicating whether it is appropriate for Sync
*/
isAddonSyncable: function isAddonSyncable(addon, ignoreRepoCheck = false) {
isAddonSyncable: function isAddonSyncable(addon) {
// Currently, we limit syncable add-ons to those that are:
// 1) In a well-defined set of types
// 2) Installed in the current profile
// 3) Not installed by a foreign entity (i.e. installed by the app)
// since they act like global extensions.
// 4) Is not a hotfix.
// 5) The addons XPIProvider doesn't veto it (i.e not being installed in
// the profile directory, or any other reasons it says the addon can't
// be synced)
// 6) Are installed from AMO
// 5) Are installed from AMO
// We could represent the test as a complex boolean expression. We go the
// verbose route so the failure reason is logged.
@ -573,12 +542,6 @@ AddonsStore.prototype = {
return false;
}
// If the addon manager says it's not syncable, we skip it.
if (!addon.isSyncable) {
this._log.debug(addon.id + " not syncable: vetoed by the addon manager.");
return false;
}
// This may be too aggressive. If an add-on is downloaded from AMO and
// manually placed in the profile directory, foreignInstall will be set.
// Arguably, that add-on should be syncable.
@ -589,20 +552,15 @@ AddonsStore.prototype = {
}
// Ignore hotfix extensions (bug 741670). The pref may not be defined.
// XXX - note that addon.isSyncable will be false for hotfix addons, so
// this check isn't strictly necessary - except for Sync tests which aren't
// setup to create a "real" hotfix addon. This can be removed once those
// tests are fixed (but keeping it doesn't hurt either)
if (this._extensionsPrefs.get("hotfix.id", null) == addon.id) {
this._log.debug(addon.id + " not syncable: is a hotfix.");
return false;
}
// If the AddonRepository's cache isn't enabled (which it typically isn't
// in tests), getCachedAddonByID always returns null - so skip the check
// in that case. We also provide a way to specifically opt-out of the check
// even if the cache is enabled, which is used by the validators.
if (ignoreRepoCheck || !AddonRepository.cacheEnabled) {
// We provide a back door to skip the repository checking of an add-on.
// This is utilized by the tests to make testing easier. Users could enable
// this, but it would sacrifice security.
if (Svc.Prefs.get("addons.ignoreRepositoryChecking", false)) {
return true;
}
@ -745,69 +703,3 @@ AddonsTracker.prototype = {
this.reconciler.stopListening();
},
};
class AddonValidator extends CollectionValidator {
constructor(engine = null) {
super("addons", "id", [
"addonID",
"enabled",
"applicationID",
"source"
]);
this.engine = engine;
}
getClientItems() {
return Promise.all([
new Promise(resolve =>
AddonManager.getAllAddons(resolve)),
new Promise(resolve =>
AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"], resolve)),
]).then(([installed, addonsWithPendingOperation]) => {
// Addons pending install won't be in the first list, but addons pending
// uninstall/enable/disable will be in both lists.
let all = new Map(installed.map(addon => [addon.id, addon]));
for (let addon of addonsWithPendingOperation) {
all.set(addon.id, addon);
}
// Convert to an array since Map.prototype.values returns an iterable
return [...all.values()];
});
}
normalizeClientItem(item) {
let enabled = !item.userDisabled;
if (item.pendingOperations & AddonManager.PENDING_ENABLE) {
enabled = true;
} else if (item.pendingOperations & AddonManager.PENDING_DISABLE) {
enabled = false;
}
return {
enabled,
id: item.syncGUID,
addonID: item.id,
applicationID: Services.appinfo.ID,
source: "amo", // check item.foreignInstall?
original: item
};
}
normalizeServerItem(item) {
let guid = this.engine._findDupe(item);
if (guid) {
item.id = guid;
}
return item;
}
clientUnderstands(item) {
return item.applicationID === Services.appinfo.ID;
}
syncedByClient(item) {
return !item.original.hidden &&
!item.original.isSystem &&
!(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) &&
this.engine.isAddonSyncable(item.original, true);
}
}

1779
services/sync/modules/engines/bookmarks.js
View file

File diff suppressed because it is too large Load diff

461
services/sync/modules/engines/clients.js
View file

@ -2,57 +2,24 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
/**
* How does the clients engine work?
*
* - We use 2 files - commands.json and commands-syncing.json.
*
* - At sync upload time, we attempt a rename of commands.json to
* commands-syncing.json, and ignore errors (helps for crash during sync!).
* - We load commands-syncing.json and stash the contents in
* _currentlySyncingCommands which lives for the duration of the upload process.
* - We use _currentlySyncingCommands to build the outgoing records
* - Immediately after successful upload, we delete commands-syncing.json from
* disk (and clear _currentlySyncingCommands). We reconcile our local records
* with what we just wrote in the server, and add failed IDs commands
* back in commands.json
* - Any time we need to "save" a command for future syncs, we load
* commands.json, update it, and write it back out.
*/
this.EXPORTED_SYMBOLS = [
"ClientEngine",
"ClientsRec"
];
var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://services-common/async.js");
Cu.import("resource://services-common/stringbundle.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/resource.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://gre/modules/Services.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
"resource://gre/modules/FxAccounts.jsm");
const CLIENTS_TTL = 1814400; // 21 days
const CLIENTS_TTL_REFRESH = 604800; // 7 days
const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days
const SUPPORTED_PROTOCOL_VERSIONS = ["1.1", "1.5"];
function hasDupeCommand(commands, action) {
if (!commands) {
return false;
}
return commands.some(other => other.command == action.command &&
Utils.deepEquals(other.args, action.args));
}
this.ClientsRec = function ClientsRec(collection, id) {
CryptoWrapper.call(this, collection, id);
}
@ -66,27 +33,23 @@ Utils.deferGetSet(ClientsRec,
"cleartext",
["name", "type", "commands",
"version", "protocols",
"formfactor", "os", "appPackage", "application", "device",
"fxaDeviceId"]);
"formfactor", "os", "appPackage", "application", "device"]);
this.ClientEngine = function ClientEngine(service) {
SyncEngine.call(this, "Clients", service);
// Reset the last sync timestamp on every startup so that we fetch all clients
this.resetLastSync();
// Reset the client on every startup so that we fetch recent clients
this._resetClient();
}
ClientEngine.prototype = {
__proto__: SyncEngine.prototype,
_storeObj: ClientStore,
_recordObj: ClientsRec,
_trackerObj: ClientsTracker,
allowSkippedRecord: false,
// Always sync client data as it controls other sync behavior
get enabled() {
return true;
},
get enabled() true,
get lastRecordUpload() {
return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
@ -95,31 +58,18 @@ ClientEngine.prototype = {
Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
},
get remoteClients() {
// return all non-stale clients for external consumption.
return Object.values(this._store._remoteClients).filter(v => !v.stale);
},
remoteClientExists(id) {
let client = this._store._remoteClients[id];
return !!(client && !client.stale);
},
// Aggregate some stats on the composition of clients on this account
get stats() {
let stats = {
hasMobile: this.localType == DEVICE_TYPE_MOBILE,
hasMobile: this.localType == "mobile",
names: [this.localName],
numClients: 1,
};
for (let id in this._store._remoteClients) {
let {name, type, stale} = this._store._remoteClients[id];
if (!stale) {
stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE;
stats.names.push(name);
stats.numClients++;
}
for each (let {name, type} in this._store._remoteClients) {
stats.hasMobile = stats.hasMobile || type == "mobile";
stats.names.push(name);
stats.numClients++;
}
return stats;
@ -135,11 +85,7 @@ ClientEngine.prototype = {
counts.set(this.localType, 1);
for (let id in this._store._remoteClients) {
let record = this._store._remoteClients[id];
if (record.stale) {
continue; // pretend "stale" records don't exist.
}
for each (let record in this._store._remoteClients) {
let type = record.type;
if (!counts.has(type)) {
counts.set(type, 0);
@ -156,9 +102,7 @@ ClientEngine.prototype = {
let localID = Svc.Prefs.get("client.GUID", "");
return localID == "" ? this.localID = Utils.makeGUID() : localID;
},
set localID(value) {
Svc.Prefs.set("client.GUID", value);
},
set localID(value) Svc.Prefs.set("client.GUID", value),
get brandName() {
let brand = new StringBundle("chrome://branding/locale/brand.properties");
@ -166,97 +110,23 @@ ClientEngine.prototype = {
},
get localName() {
let name = Utils.getDeviceName();
// If `getDeviceName` returns the default name, set the pref. FxA registers
// the device before syncing, so we don't need to update the registration
// in this case.
Svc.Prefs.set("client.name", name);
return name;
},
set localName(value) {
Svc.Prefs.set("client.name", value);
// Update the registration in the background.
fxAccounts.updateDeviceRegistration().catch(error => {
this._log.warn("failed to update fxa device registration", error);
});
},
let localName = Svc.Prefs.get("client.name", "");
if (localName != "")
return localName;
get localType() {
return Utils.getDeviceType();
},
set localType(value) {
Svc.Prefs.set("client.type", value);
return this.localName = Utils.getDefaultDeviceName();
},
set localName(value) Svc.Prefs.set("client.name", value),
getClientName(id) {
if (id == this.localID) {
return this.localName;
}
let client = this._store._remoteClients[id];
return client ? client.name : "";
},
getClientFxaDeviceId(id) {
if (this._store._remoteClients[id]) {
return this._store._remoteClients[id].fxaDeviceId;
}
return null;
},
get localType() Svc.Prefs.get("client.type", "desktop"),
set localType(value) Svc.Prefs.set("client.type", value),
isMobile: function isMobile(id) {
if (this._store._remoteClients[id])
return this._store._remoteClients[id].type == DEVICE_TYPE_MOBILE;
return this._store._remoteClients[id].type == "mobile";
return false;
},
_readCommands() {
let cb = Async.makeSpinningCallback();
Utils.jsonLoad("commands", this, commands => cb(null, commands));
return cb.wait() || {};
},
/**
* Low level function, do not use directly (use _addClientCommand instead).
*/
_saveCommands(commands) {
let cb = Async.makeSpinningCallback();
Utils.jsonSave("commands", this, commands, error => {
if (error) {
this._log.error("Failed to save JSON outgoing commands", error);
}
cb();
});
cb.wait();
},
_prepareCommandsForUpload() {
let cb = Async.makeSpinningCallback();
Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors
.then(() => {
Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands));
});
return cb.wait() || {};
},
_deleteUploadedCommands() {
delete this._currentlySyncingCommands;
Async.promiseSpinningly(
Utils.jsonRemove("commands-syncing", this).catch(err => {
this._log.error("Failed to delete syncing-commands file", err);
})
);
},
_addClientCommand(clientId, command) {
const allCommands = this._readCommands();
const clientCommands = allCommands[clientId] || [];
if (hasDupeCommand(clientCommands, command)) {
return;
}
allCommands[clientId] = clientCommands.concat(command);
this._saveCommands(allCommands);
},
_syncStartup: function _syncStartup() {
// Reupload new client record periodically.
if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
@ -266,157 +136,9 @@ ClientEngine.prototype = {
SyncEngine.prototype._syncStartup.call(this);
},
_processIncoming() {
// Fetch all records from the server.
this.lastSync = 0;
this._incomingClients = {};
try {
SyncEngine.prototype._processIncoming.call(this);
// Since clients are synced unconditionally, any records in the local store
// that don't exist on the server must be for disconnected clients. Remove
// them, so that we don't upload records with commands for clients that will
// never see them. We also do this to filter out stale clients from the
// tabs collection, since showing their list of tabs is confusing.
for (let id in this._store._remoteClients) {
if (!this._incomingClients[id]) {
this._log.info(`Removing local state for deleted client ${id}`);
this._removeRemoteClient(id);
}
}
// Bug 1264498: Mobile clients don't remove themselves from the clients
// collection when the user disconnects Sync, so we mark as stale clients
// with the same name that haven't synced in over a week.
// (Note we can't simply delete them, or we re-apply them next sync - see
// bug 1287687)
delete this._incomingClients[this.localID];
let names = new Set([this.localName]);
for (let id in this._incomingClients) {
let record = this._store._remoteClients[id];
if (!names.has(record.name)) {
names.add(record.name);
continue;
}
let remoteAge = AsyncResource.serverTime - this._incomingClients[id];
if (remoteAge > STALE_CLIENT_REMOTE_AGE) {
this._log.info(`Hiding stale client ${id} with age ${remoteAge}`);
record.stale = true;
}
}
} finally {
this._incomingClients = null;
}
},
_uploadOutgoing() {
this._currentlySyncingCommands = this._prepareCommandsForUpload();
const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands);
for (let clientId of clientWithPendingCommands) {
if (this._store._remoteClients[clientId] || this.localID == clientId) {
this._modified.set(clientId, 0);
}
}
SyncEngine.prototype._uploadOutgoing.call(this);
},
_onRecordsWritten(succeeded, failed) {
// Reconcile the status of the local records with what we just wrote on the
// server
for (let id of succeeded) {
const commandChanges = this._currentlySyncingCommands[id];
if (id == this.localID) {
if (this.localCommands) {
this.localCommands = this.localCommands.filter(command => !hasDupeCommand(commandChanges, command));
}
} else {
const clientRecord = this._store._remoteClients[id];
if (!commandChanges || !clientRecord) {
// should be impossible, else we wouldn't have been writing it.
this._log.warn("No command/No record changes for a client we uploaded");
continue;
}
// fixup the client record, so our copy of _remoteClients matches what we uploaded.
clientRecord.commands = this._store.createRecord(id);
// we could do better and pass the reference to the record we just uploaded,
// but this will do for now
}
}
// Re-add failed commands
for (let id of failed) {
const commandChanges = this._currentlySyncingCommands[id];
if (!commandChanges) {
continue;
}
this._addClientCommand(id, commandChanges);
}
this._deleteUploadedCommands();
// Notify other devices that their own client collection changed
const idsToNotify = succeeded.reduce((acc, id) => {
if (id == this.localID) {
return acc;
}
const fxaDeviceId = this.getClientFxaDeviceId(id);
return fxaDeviceId ? acc.concat(fxaDeviceId) : acc;
}, []);
if (idsToNotify.length > 0) {
this._notifyCollectionChanged(idsToNotify);
}
},
_notifyCollectionChanged(ids) {
const message = {
version: 1,
command: "sync:collection_changed",
data: {
collections: ["clients"]
}
};
fxAccounts.notifyDevices(ids, message, NOTIFY_TAB_SENT_TTL_SECS);
},
_syncFinish() {
// Record histograms for our device types, and also write them to a pref
// so non-histogram telemetry (eg, UITelemetry) has easy access to them.
for (let [deviceType, count] of this.deviceTypes) {
let hid;
let prefName = this.name + ".devices.";
switch (deviceType) {
case "desktop":
hid = "WEAVE_DEVICE_COUNT_DESKTOP";
prefName += "desktop";
break;
case "mobile":
hid = "WEAVE_DEVICE_COUNT_MOBILE";
prefName += "mobile";
break;
default:
this._log.warn(`Unexpected deviceType "${deviceType}" recording device telemetry.`);
continue;
}
Services.telemetry.getHistogramById(hid).add(count);
Svc.Prefs.set(prefName, count);
}
SyncEngine.prototype._syncFinish.call(this);
},
_reconcile: function _reconcile(item) {
// Every incoming record is reconciled, so we use this to track the
// contents of the collection on the server.
this._incomingClients[item.id] = item.modified;
if (!this._store.itemExists(item.id)) {
return true;
}
// Clients are synced unconditionally, so we'll always have new records.
// Unfortunately, this will cause the scheduler to use the immediate sync
// interval for the multi-device case, instead of the active interval. We
// work around this by updating the record during reconciliation, and
// returning false to indicate that the record doesn't need to be applied
// later.
this._store.update(item);
return false;
// Always process incoming items because they might have commands
_reconcile: function _reconcile() {
return true;
},
// Treat reset the same as wiping for locally cached clients
@ -426,13 +148,7 @@ ClientEngine.prototype = {
_wipeClient: function _wipeClient() {
SyncEngine.prototype._resetClient.call(this);
delete this.localCommands;
this._store.wipe();
const logRemoveError = err => this._log.warn("Could not delete json file", err);
Async.promiseSpinningly(
Utils.jsonRemove("commands", this).catch(logRemoveError)
.then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError))
);
},
removeClientData: function removeClientData() {
@ -470,6 +186,14 @@ ClientEngine.prototype = {
displayURI: { args: 3, desc: "Instruct a client to display a URI" },
},
/**
* Remove any commands for the local client and mark it for upload.
*/
clearCommands: function clearCommands() {
delete this.localCommands;
this._tracker.addChangedID(this.localID);
},
/**
* Sends a command+args pair to a specific client.
*
@ -484,17 +208,30 @@ ClientEngine.prototype = {
if (!client) {
throw new Error("Unknown remote client ID: '" + clientId + "'.");
}
if (client.stale) {
throw new Error("Stale remote client ID: '" + clientId + "'.");
}
// notDupe compares two commands and returns if they are not equal.
let notDupe = function(other) {
return other.command != command || !Utils.deepEquals(other.args, args);
};
let action = {
command: command,
args: args,
};
if (!client.commands) {
client.commands = [action];
}
// Add the new action if there are no duplicates.
else if (client.commands.every(notDupe)) {
client.commands.push(action);
}
// It must be a dupe. Skip.
else {
return;
}
this._log.trace("Client " + clientId + " got a new action: " + [command, args]);
this._addClientCommand(clientId, action);
this._tracker.addChangedID(clientId);
},
@ -505,17 +242,13 @@ ClientEngine.prototype = {
*/
processIncomingCommands: function processIncomingCommands() {
return this._notify("clients:process-commands", "", function() {
if (!this.localCommands) {
return true;
}
let commands = this.localCommands;
const clearedCommands = this._readCommands()[this.localID];
const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command));
// Immediately clear out the commands as we've got them locally.
this.clearCommands();
let URIsToDisplay = [];
// Process each command in order.
for (let rawCommand of commands) {
let {command, args} = rawCommand;
for each (let {command, args} in commands) {
this._log.debug("Processing command: " + command + "(" + args + ")");
let engines = [args[0]];
@ -536,20 +269,12 @@ ClientEngine.prototype = {
this.service.logout();
return false;
case "displayURI":
let [uri, clientId, title] = args;
URIsToDisplay.push({ uri, clientId, title });
this._handleDisplayURI.apply(this, args);
break;
default:
this._log.debug("Received an unknown command: " + command);
break;
}
// Add the command to the "cleared" commands list
this._addClientCommand(this.localID, rawCommand)
}
this._tracker.addChangedID(this.localID);
if (URIsToDisplay.length) {
this._handleDisplayURIs(URIsToDisplay);
}
return true;
@ -588,10 +313,8 @@ ClientEngine.prototype = {
if (clientId) {
this._sendCommandToClient(command, args, clientId);
} else {
for (let [id, record] of Object.entries(this._store._remoteClients)) {
if (!record.stale) {
this._sendCommandToClient(command, args, id);
}
for (let id in this._store._remoteClients) {
this._sendCommandToClient(command, args, id);
}
}
},
@ -622,11 +345,11 @@ ClientEngine.prototype = {
},
/**
* Handle a bunch of received 'displayURI' commands.
* Handle a single received 'displayURI' command.
*
* Interested parties should observe the "weave:engine:clients:display-uris"
* topic. The callback will receive an array as the subject parameter
* containing objects with the following keys:
* Interested parties should observe the "weave:engine:clients:display-uri"
* topic. The callback will receive an object as the subject parameter with
* the following keys:
*
* uri URI (string) that is requested for display.
* clientId ID of client that sent the command.
@ -634,24 +357,21 @@ ClientEngine.prototype = {
*
* The 'data' parameter to the callback will not be defined.
*
* @param uris
* An array containing URI objects to display
* @param uris[].uri
* @param uri
* String URI that was received
* @param uris[].clientId
* @param clientId
* ID of client that sent URI
* @param uris[].title
* @param title
* String title of page that URI corresponds to. Older clients may not
* send this.
*/
_handleDisplayURIs: function _handleDisplayURIs(uris) {
Svc.Obs.notify("weave:engine:clients:display-uris", uris);
},
_handleDisplayURI: function _handleDisplayURI(uri, clientId, title) {
this._log.info("Received a URI for display: " + uri + " (" + title +
") from " + clientId);
_removeRemoteClient(id) {
delete this._store._remoteClients[id];
this._tracker.removeChangedID(id);
},
let subject = {uri: uri, client: clientId, title: title};
Svc.Obs.notify("weave:engine:clients:display-uri", subject);
}
};
function ClientStore(name, engine) {
@ -660,48 +380,29 @@ function ClientStore(name, engine) {
ClientStore.prototype = {
__proto__: Store.prototype,
_remoteClients: {},
create(record) {
this.update(record);
this.update(record)
},
update: function update(record) {
if (record.id == this.engine.localID) {
// Only grab commands from the server; local name/type always wins
// Only grab commands from the server; local name/type always wins
if (record.id == this.engine.localID)
this.engine.localCommands = record.commands;
} else {
else
this._remoteClients[record.id] = record.cleartext;
}
},
createRecord: function createRecord(id, collection) {
let record = new ClientsRec(collection, id);
const commandsChanges = this.engine._currentlySyncingCommands ?
this.engine._currentlySyncingCommands[id] :
[];
// Package the individual components into a record for the local client
if (id == this.engine.localID) {
let cb = Async.makeSpinningCallback();
fxAccounts.getDeviceId().then(id => cb(null, id), cb);
try {
record.fxaDeviceId = cb.wait();
} catch(error) {
this._log.warn("failed to get fxa device id", error);
}
record.name = this.engine.localName;
record.type = this.engine.localType;
record.commands = this.engine.localCommands;
record.version = Services.appinfo.version;
record.protocols = SUPPORTED_PROTOCOL_VERSIONS;
// Substract the commands we recorded that we've already executed
if (commandsChanges && commandsChanges.length &&
this.engine.localCommands && this.engine.localCommands.length) {
record.commands = this.engine.localCommands.filter(command => !hasDupeCommand(commandsChanges, command));
}
// Optional fields.
record.os = Services.appinfo.OS; // "Darwin"
record.appPackage = Services.appinfo.ID;
@ -712,20 +413,6 @@ ClientStore.prototype = {
// record.formfactor = ""; // Bug 1100722
} else {
record.cleartext = this._remoteClients[id];
// Add the commands we have to send
if (commandsChanges && commandsChanges.length) {
const recordCommands = record.cleartext.commands || [];
const newCommands = commandsChanges.filter(command => !hasDupeCommand(recordCommands, command));
record.cleartext.commands = recordCommands.concat(newCommands);
}
if (record.cleartext.stale) {
// It's almost certainly a logic error for us to upload a record we
// consider stale, so make log noise, but still remove the flag.
this._log.error(`Preparing to upload record ${id} that we consider stale`);
delete record.cleartext.stale;
}
}
return record;
@ -768,7 +455,7 @@ ClientsTracker.prototype = {
break;
case "weave:engine:stop-tracking":
if (this._enabled) {
Svc.Prefs.ignore("client.name", this);
Svc.Prefs.ignore("clients.name", this);
this._enabled = false;
}
break;

102
services/sync/modules/engines/forms.js
View file

@ -2,11 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec', 'FormValidator'];
this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec'];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cu = Components.utils;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://services-sync/engines.js");
@ -14,10 +14,9 @@ Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/collection_validator.js");
Cu.import("resource://gre/modules/Log.jsm");
const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds.
const FORMS_TTL = 5184000; // 60 days
this.FormRec = function FormRec(collection, id) {
CryptoWrapper.call(this, collection, id);
@ -31,30 +30,26 @@ FormRec.prototype = {
Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
var FormWrapper = {
let FormWrapper = {
_log: Log.repository.getLogger("Sync.Engine.Forms"),
_getEntryCols: ["fieldname", "value"],
_guidCols: ["guid"],
_promiseSearch: function(terms, searchData) {
return new Promise(resolve => {
let results = [];
let callbacks = {
handleResult(result) {
results.push(result);
},
handleCompletion(reason) {
resolve(results);
}
};
Svc.FormHistory.search(terms, searchData, callbacks);
})
},
// Do a "sync" search by spinning the event loop until it completes.
_searchSpinningly: function(terms, searchData) {
return Async.promiseSpinningly(this._promiseSearch(terms, searchData));
let results = [];
let cb = Async.makeSpinningCallback();
let callbacks = {
handleResult: function(result) {
results.push(result);
},
handleCompletion: function(reason) {
cb(null, results);
}
};
Svc.FormHistory.search(terms, searchData, callbacks);
return cb.wait();
},
_updateSpinningly: function(changes) {
@ -114,9 +109,7 @@ FormEngine.prototype = {
syncPriority: 6,
get prefName() {
return "history";
},
get prefName() "history",
_findDupe: function _findDupe(item) {
return FormWrapper.getGUID(item.name, item.value);
@ -232,9 +225,7 @@ FormTracker.prototype = {
observe: function (subject, topic, data) {
Tracker.prototype.observe.call(this, subject, topic, data);
if (this.ignoreAll) {
return;
}
switch (topic) {
case "satchel-storage-changed":
if (data == "formhistory-add" || data == "formhistory-remove") {
@ -250,56 +241,3 @@ FormTracker.prototype = {
this.score += SCORE_INCREMENT_MEDIUM;
},
};
class FormsProblemData extends CollectionProblemData {
getSummary() {
// We don't support syncing deleted form data, so "clientMissing" isn't a problem
return super.getSummary().filter(entry =>
entry.name !== "clientMissing");
}
}
class FormValidator extends CollectionValidator {
constructor() {
super("forms", "id", ["name", "value"]);
}
emptyProblemData() {
return new FormsProblemData();
}
getClientItems() {
return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {});
}
normalizeClientItem(item) {
return {
id: item.guid,
guid: item.guid,
name: item.fieldname,
fieldname: item.fieldname,
value: item.value,
original: item,
};
}
normalizeServerItem(item) {
let res = Object.assign({
guid: item.id,
fieldname: item.name,
original: item,
}, item);
// Missing `name` or `value` causes the getGUID call to throw
if (item.name !== undefined && item.value !== undefined) {
let guid = FormWrapper.getGUID(item.name, item.value);
if (guid) {
res.guid = guid;
res.id = guid;
res.duped = true;
}
}
return res;
}
}

68
services/sync/modules/engines/history.js
View file

@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ['HistoryEngine', 'HistoryRec'];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cu = Components.utils;
var Cr = Components.results;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cu = Components.utils;
const Cr = Components.results;
const HISTORY_TTL = 5184000; // 60 days
@ -44,25 +44,6 @@ HistoryEngine.prototype = {
applyIncomingBatchSize: HISTORY_STORE_BATCH_SIZE,
syncPriority: 7,
_processIncoming: function (newitems) {
// We want to notify history observers that a batch operation is underway
// so they don't do lots of work for each incoming record.
let observers = PlacesUtils.history.getObservers();
function notifyHistoryObservers(notification) {
for (let observer of observers) {
try {
observer[notification]();
} catch (ex) { }
}
}
notifyHistoryObservers("onBeginUpdateBatch");
try {
return SyncEngine.prototype._processIncoming.call(this, newitems);
} finally {
notifyHistoryObservers("onEndUpdateBatch");
}
},
};
function HistoryStore(name, engine) {
@ -70,8 +51,7 @@ function HistoryStore(name, engine) {
// Explicitly nullify our references to our cached services so we don't leak
Svc.Obs.add("places-shutdown", function() {
for (let query in this._stmts) {
let stmt = this._stmts;
for each ([query, stmt] in Iterator(this._stmts)) {
stmt.finalize();
}
this._stmts = {};
@ -105,7 +85,7 @@ HistoryStore.prototype = {
return this._getStmt(
"UPDATE moz_places " +
"SET guid = :guid " +
"WHERE url_hash = hash(:page_url) AND url = :page_url");
"WHERE url = :page_url");
},
// Some helper functions to handle GUIDs
@ -127,7 +107,7 @@ HistoryStore.prototype = {
return this._getStmt(
"SELECT guid " +
"FROM moz_places " +
"WHERE url_hash = hash(:page_url) AND url = :page_url");
"WHERE url = :page_url");
},
_guidCols: ["guid"],
@ -146,12 +126,12 @@ HistoryStore.prototype = {
},
get _visitStm() {
return this._getStmt(`/* do not warn (bug 599936) */
SELECT visit_type type, visit_date date
FROM moz_historyvisits
JOIN moz_places h ON h.id = place_id
WHERE url_hash = hash(:url) AND url = :url
ORDER BY date DESC LIMIT 20`);
return this._getStmt(
"/* do not warn (bug 599936) */ " +
"SELECT visit_type type, visit_date date " +
"FROM moz_historyvisits " +
"WHERE place_id = (SELECT id FROM moz_places WHERE url = :url) " +
"ORDER BY date DESC LIMIT 10");
},
_visitCols: ["date", "type"],
@ -223,10 +203,7 @@ HistoryStore.prototype = {
} else {
shouldApply = this._recordToPlaceInfo(record);
}
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
} catch(ex) {
failed.push(record.id);
shouldApply = false;
}
@ -299,14 +276,14 @@ HistoryStore.prototype = {
if (!visit.date || typeof visit.date != "number") {
this._log.warn("Encountered record with invalid visit date: "
+ visit.date);
continue;
throw "Visit has no date!";
}
if (!visit.type ||
!Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)) {
this._log.warn("Encountered record with invalid visit type: " +
visit.type + "; ignoring.");
continue;
if (!visit.type || !(visit.type >= PlacesUtils.history.TRANSITION_LINK &&
visit.type <= PlacesUtils.history.TRANSITION_FRAMED_LINK)) {
this._log.warn("Encountered record with invalid visit type: "
+ visit.type);
throw "Invalid visit type!";
}
// Dates need to be integers.
@ -317,7 +294,6 @@ HistoryStore.prototype = {
// overwritten.
continue;
}
visit.visitDate = visit.date;
visit.transitionType = visit.type;
k += 1;
@ -369,9 +345,7 @@ HistoryStore.prototype = {
},
wipe: function HistStore_wipe() {
let cb = Async.makeSyncCallback();
PlacesUtils.history.clear().then(result => {cb(null, result)}, err => {cb(err)});
return Async.waitForSyncCallback(cb);
PlacesUtils.history.removeAllPages();
}
};

89
services/sync/modules/engines/passwords.js
View file

@ -2,16 +2,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec', 'PasswordValidator'];
this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec'];
var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/collection_validator.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/async.js");
this.LoginRec = function LoginRec(collection, id) {
CryptoWrapper.call(this, collection, id);
@ -24,7 +22,6 @@ LoginRec.prototype = {
Utils.deferGetSet(LoginRec, "cleartext", [
"hostname", "formSubmitURL",
"httpRealm", "username", "password", "usernameField", "passwordField",
"timeCreated", "timePasswordChanged",
]);
@ -70,10 +67,7 @@ PasswordEngine.prototype = {
Svc.Prefs.set("deletePwdFxA", true);
Svc.Prefs.reset("deletePwd"); // The old prefname we previously used.
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.debug("Password deletes failed", ex);
this._log.debug("Password deletes failed: " + Utils.exceptionStr(ex));
}
}
},
@ -89,7 +83,7 @@ PasswordEngine.prototype = {
this._store._sleep(0); // Yield back to main thread after synchronous operation.
// Look for existing logins that match the hostname, but ignore the password.
for (let local of logins) {
for each (let local in logins) {
if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) {
return local.guid;
}
@ -104,13 +98,6 @@ function PasswordStore(name, engine) {
PasswordStore.prototype = {
__proto__: Store.prototype,
_newPropertyBag: function () {
return Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2);
},
/**
* Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo).
*/
_nsLoginInfoFromRecord: function (record) {
function nullUndefined(x) {
return (x == undefined) ? null : x;
@ -131,21 +118,13 @@ PasswordStore.prototype = {
record.password,
record.usernameField,
record.passwordField);
info.QueryInterface(Ci.nsILoginMetaInfo);
info.guid = record.id;
if (record.timeCreated) {
info.timeCreated = record.timeCreated;
}
if (record.timePasswordChanged) {
info.timePasswordChanged = record.timePasswordChanged;
}
return info;
},
_getLoginFromGUID: function (id) {
let prop = this._newPropertyBag();
let prop = Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2);
prop.setPropertyAsAUTF8String("guid", id);
let logins = Services.logins.searchLogins({}, prop);
@ -190,7 +169,8 @@ PasswordStore.prototype = {
return;
}
let prop = this._newPropertyBag();
let prop = Cc["@mozilla.org/hash-property-bag;1"]
.createInstance(Ci.nsIWritablePropertyBag2);
prop.setPropertyAsAUTF8String("guid", newID);
Services.logins.modifyLogin(oldLogin, prop);
@ -217,11 +197,6 @@ PasswordStore.prototype = {
record.usernameField = login.usernameField;
record.passwordField = login.passwordField;
// Optional fields.
login.QueryInterface(Ci.nsILoginMetaInfo);
record.timeCreated = login.timeCreated;
record.timePasswordChanged = login.timePasswordChanged;
return record;
},
@ -237,7 +212,8 @@ PasswordStore.prototype = {
try {
Services.logins.addLogin(login);
} catch(ex) {
this._log.debug(`Adding record ${record.id} resulted in exception`, ex);
this._log.debug("Adding record " + record.id +
" resulted in exception " + Utils.exceptionStr(ex));
}
},
@ -269,7 +245,9 @@ PasswordStore.prototype = {
try {
Services.logins.modifyLogin(loginItem, newinfo);
} catch(ex) {
this._log.debug(`Modifying record ${record.id} resulted in exception; not modifying`, ex);
this._log.debug("Modifying record " + record.id +
" resulted in exception " + Utils.exceptionStr(ex) +
". Not modifying.");
}
},
@ -326,46 +304,3 @@ PasswordTracker.prototype = {
}
},
};
class PasswordValidator extends CollectionValidator {
constructor() {
super("passwords", "id", [
"hostname",
"formSubmitURL",
"httpRealm",
"password",
"passwordField",
"username",
"usernameField",
]);
}
getClientItems() {
let logins = Services.logins.getAllLogins({});
let syncHosts = Utils.getSyncCredentialsHosts()
let result = logins.map(l => l.QueryInterface(Ci.nsILoginMetaInfo))
.filter(l => !syncHosts.has(l.hostname));
return Promise.resolve(result);
}
normalizeClientItem(item) {
return {
id: item.guid,
guid: item.guid,
hostname: item.hostname,
formSubmitURL: item.formSubmitURL,
httpRealm: item.httpRealm,
password: item.password,
passwordField: item.passwordField,
username: item.username,
usernameField: item.usernameField,
original: item,
}
}
normalizeServerItem(item) {
return Object.assign({ guid: item.id }, item);
}
}

81
services/sync/modules/engines/prefs.js
View file

@ -4,11 +4,11 @@
this.EXPORTED_SYMBOLS = ['PrefsEngine', 'PrefRec'];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cu = Components.utils;
const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
const SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
@ -42,7 +42,6 @@ PrefsEngine.prototype = {
version: 2,
syncPriority: 1,
allowSkippedRecord: false,
getChangedIDs: function () {
// No need for a proper timestamp (no conflict resolution needed).
@ -88,45 +87,37 @@ PrefStore.prototype = {
_getSyncPrefs: function () {
let syncPrefs = Cc["@mozilla.org/preferences-service;1"]
.getService(Ci.nsIPrefService)
.getBranch(PREF_SYNC_PREFS_PREFIX)
.getBranch(SYNC_PREFS_PREFIX)
.getChildList("", {});
// Also sync preferences that determine which prefs get synced.
let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref);
let controlPrefs = syncPrefs.map(pref => SYNC_PREFS_PREFIX + pref);
return controlPrefs.concat(syncPrefs);
},
_isSynced: function (pref) {
return pref.startsWith(PREF_SYNC_PREFS_PREFIX) ||
this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false);
return pref.startsWith(SYNC_PREFS_PREFIX) ||
this._prefs.get(SYNC_PREFS_PREFIX + pref, false);
},
_getAllPrefs: function () {
let values = {};
for (let pref of this._getSyncPrefs()) {
for each (let pref in this._getSyncPrefs()) {
if (this._isSynced(pref)) {
// Missing and default prefs get the null value.
values[pref] = this._prefs.isSet(pref) ? this._prefs.get(pref, null) : null;
// Missing prefs get the null value.
values[pref] = this._prefs.get(pref, null);
}
}
return values;
},
_updateLightWeightTheme (themeID) {
let themeObject = null;
if (themeID) {
themeObject = LightweightThemeManager.getUsedTheme(themeID);
}
LightweightThemeManager.currentTheme = themeObject;
},
_setAllPrefs: function (values) {
let selectedThemeIDPref = "lightweightThemes.selectedThemeID";
let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null);
let selectedThemeIDAfter = selectedThemeIDBefore;
let enabledPref = "lightweightThemes.isThemeSelected";
let enabledBefore = this._prefs.get(enabledPref, false);
let prevTheme = LightweightThemeManager.currentTheme;
// Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise
// _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device).
let prefs = Object.keys(values).sort(a => -a.indexOf(PREF_SYNC_PREFS_PREFIX));
let prefs = Object.keys(values).sort(a => -a.indexOf(SYNC_PREFS_PREFIX));
for (let pref of prefs) {
if (!this._isSynced(pref)) {
continue;
@ -134,30 +125,26 @@ PrefStore.prototype = {
let value = values[pref];
switch (pref) {
// Some special prefs we don't want to set directly.
case selectedThemeIDPref:
selectedThemeIDAfter = value;
break;
// default is to just set the pref
default:
if (value == null) {
// Pref has gone missing. The best we can do is reset it.
this._prefs.reset(pref);
} else {
try {
this._prefs.set(pref, value);
} catch(ex) {
this._log.trace("Failed to set pref: " + pref + ": " + ex);
}
}
// Pref has gone missing. The best we can do is reset it.
if (value == null) {
this._prefs.reset(pref);
continue;
}
try {
this._prefs.set(pref, value);
} catch(ex) {
this._log.trace("Failed to set pref: " + pref + ": " + ex);
}
}
// Notify the lightweight theme manager if the selected theme has changed.
if (selectedThemeIDBefore != selectedThemeIDAfter) {
this._updateLightWeightTheme(selectedThemeIDAfter);
// Notify the lightweight theme manager of all the new values
let enabledNow = this._prefs.get(enabledPref, false);
if (enabledBefore && !enabledNow) {
LightweightThemeManager.currentTheme = null;
} else if (enabledNow && LightweightThemeManager.usedThemes[0] != prevTheme) {
LightweightThemeManager.currentTheme = null;
LightweightThemeManager.currentTheme = LightweightThemeManager.usedThemes[0];
}
},
@ -261,8 +248,8 @@ PrefTracker.prototype = {
case "nsPref:changed":
// Trigger a sync for MULTI-DEVICE for a change that determines
// which prefs are synced or a regular pref change.
if (data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 ||
this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)) {
if (data.indexOf(SYNC_PREFS_PREFIX) == 0 ||
this._prefs.get(SYNC_PREFS_PREFIX + data, false)) {
this.score += SCORE_INCREMENT_XLARGE;
this.modified = true;
this._log.trace("Preference " + data + " changed");

71
services/sync/modules/engines/tabs.js
View file

@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["TabEngine", "TabSetRecord"];
var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
const TABS_TTL = 604800; // 7 days.
const TAB_ENTRIES_LIMIT = 25; // How many URLs to include in tab history.
@ -43,11 +43,6 @@ TabEngine.prototype = {
_storeObj: TabStore,
_trackerObj: TabTracker,
_recordObj: TabSetRecord,
// A flag to indicate if we have synced in this session. This is to help
// consumers of remote tabs that may want to differentiate between "I've an
// empty tab list as I haven't yet synced" vs "I've an empty tab list
// as there really are no tabs"
hasSyncedThisSession: false,
syncPriority: 3,
@ -72,7 +67,6 @@ TabEngine.prototype = {
SyncEngine.prototype._resetClient.call(this);
this._store.wipe();
this._tracker.modified = true;
this.hasSyncedThisSession = false;
},
removeClientData: function () {
@ -100,12 +94,7 @@ TabEngine.prototype = {
}
return SyncEngine.prototype._reconcile.call(this, item);
},
_syncFinish() {
this.hasSyncedThisSession = true;
return SyncEngine.prototype._syncFinish.call(this);
},
}
};
@ -145,7 +134,7 @@ TabStore.prototype = {
}
for (let tab of win.gBrowser.tabs) {
let tabState = this.getTabState(tab);
tabState = this.getTabState(tab);
// Make sure there are history entries to look at.
if (!tabState || !tabState.entries.length) {
@ -165,11 +154,6 @@ TabStore.prototype = {
continue;
}
if (current.url.length >= (MAX_UPLOAD_BYTES - 1000)) {
this._log.trace("Skipping over-long URL.");
continue;
}
// The element at `index` is the current page. Previous URLs were
// previously visited URLs; subsequent URLs are in the 'forward' stack,
// which we can't represent in Sync, so we truncate here.
@ -189,9 +173,7 @@ TabStore.prototype = {
allTabs.push({
title: current.title || "",
urlHistory: urls,
icon: tabState.image ||
(tabState.attributes && tabState.attributes.image) ||
"",
icon: tabState.attributes && tabState.attributes.image || "",
lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000),
});
}
@ -265,9 +247,27 @@ TabStore.prototype = {
create: function (record) {
this._log.debug("Adding remote tabs from " + record.clientName);
this._remoteClients[record.id] = Object.assign({}, record.cleartext, {
lastModified: record.modified
});
this._remoteClients[record.id] = record.cleartext;
// Lose some precision, but that's good enough (seconds).
let roundModify = Math.floor(record.modified / 1000);
let notifyState = Svc.Prefs.get("notifyTabState");
// If there's no existing pref, save this first modified time.
if (notifyState == null) {
Svc.Prefs.set("notifyTabState", roundModify);
return;
}
// Don't change notifyState if it's already 0 (don't notify).
if (notifyState == 0) {
return;
}
// We must have gotten a new tab that isn't the same as last time.
if (notifyState != roundModify) {
Svc.Prefs.set("notifyTabState", 0);
}
},
update: function (record) {
@ -302,14 +302,10 @@ TabTracker.prototype = {
_registerListenersForWindow: function (window) {
this._log.trace("Registering tab listeners in window");
for (let topic of this._topics) {
for each (let topic in this._topics) {
window.addEventListener(topic, this.onTab, false);
}
window.addEventListener("unload", this._unregisterListeners, false);
// If it's got a tab browser we can listen for things like navigation.
if (window.gBrowser) {
window.gBrowser.addProgressListener(this);
}
},
_unregisterListeners: function (event) {
@ -319,12 +315,9 @@ TabTracker.prototype = {
_unregisterListenersForWindow: function (window) {
this._log.trace("Removing tab listeners in window");
window.removeEventListener("unload", this._unregisterListeners, false);
for (let topic of this._topics) {
for each (let topic in this._topics) {
window.removeEventListener(topic, this.onTab, false);
}
if (window.gBrowser) {
window.gBrowser.removeProgressListener(this);
}
},
startTracking: function () {
@ -380,14 +373,4 @@ TabTracker.prototype = {
this.score += SCORE_INCREMENT_SMALL;
}
},
// web progress listeners.
onLocationChange: function (webProgress, request, location, flags) {
// We only care about top-level location changes which are not in the same
// document.
if (webProgress.isTopLevel &&
((flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0)) {
this.modified = true;
}
},
};

262
services/sync/modules/healthreport.jsm Normal file
View file

@ -0,0 +1,262 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = [
"SyncProvider",
];
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://gre/modules/Metrics.jsm", this);
Cu.import("resource://gre/modules/Promise.jsm", this);
Cu.import("resource://gre/modules/Services.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
const DAILY_LAST_NUMERIC_FIELD = {type: Metrics.Storage.FIELD_DAILY_LAST_NUMERIC};
const DAILY_LAST_TEXT_FIELD = {type: Metrics.Storage.FIELD_DAILY_LAST_TEXT};
const DAILY_COUNTER_FIELD = {type: Metrics.Storage.FIELD_DAILY_COUNTER};
XPCOMUtils.defineLazyModuleGetter(this, "Weave",
"resource://services-sync/main.js");
function SyncMeasurement1() {
Metrics.Measurement.call(this);
}
SyncMeasurement1.prototype = Object.freeze({
__proto__: Metrics.Measurement.prototype,
name: "sync",
version: 1,
fields: {
enabled: DAILY_LAST_NUMERIC_FIELD,
preferredProtocol: DAILY_LAST_TEXT_FIELD,
activeProtocol: DAILY_LAST_TEXT_FIELD,
syncStart: DAILY_COUNTER_FIELD,
syncSuccess: DAILY_COUNTER_FIELD,
syncError: DAILY_COUNTER_FIELD,
},
});
function SyncDevicesMeasurement1() {
Metrics.Measurement.call(this);
}
SyncDevicesMeasurement1.prototype = Object.freeze({
__proto__: Metrics.Measurement.prototype,
name: "devices",
version: 1,
fields: {},
shouldIncludeField: function (name) {
return true;
},
fieldType: function (name) {
return Metrics.Storage.FIELD_DAILY_COUNTER;
},
});
function SyncMigrationMeasurement1() {
Metrics.Measurement.call(this);
}
SyncMigrationMeasurement1.prototype = Object.freeze({
__proto__: Metrics.Measurement.prototype,
name: "migration",
version: 1,
fields: {
state: DAILY_LAST_TEXT_FIELD, // last "user" or "internal" state we saw for the day
accepted: DAILY_COUNTER_FIELD, // number of times user tried to start migration
declined: DAILY_COUNTER_FIELD, // number of times user closed nagging infobar
unlinked: DAILY_LAST_NUMERIC_FIELD, // did the user decline and unlink
},
});
this.SyncProvider = function () {
Metrics.Provider.call(this);
};
SyncProvider.prototype = Object.freeze({
__proto__: Metrics.Provider.prototype,
name: "org.mozilla.sync",
measurementTypes: [
SyncDevicesMeasurement1,
SyncMeasurement1,
SyncMigrationMeasurement1,
],
_OBSERVERS: [
"weave:service:sync:start",
"weave:service:sync:finish",
"weave:service:sync:error",
"fxa-migration:state-changed",
"fxa-migration:internal-state-changed",
"fxa-migration:internal-telemetry",
],
postInit: function () {
for (let o of this._OBSERVERS) {
Services.obs.addObserver(this, o, false);
}
return Promise.resolve();
},
onShutdown: function () {
for (let o of this._OBSERVERS) {
Services.obs.removeObserver(this, o);
}
return Promise.resolve();
},
observe: function (subject, topic, data) {
switch (topic) {
case "weave:service:sync:start":
case "weave:service:sync:finish":
case "weave:service:sync:error":
return this._observeSync(subject, topic, data);
case "fxa-migration:state-changed":
case "fxa-migration:internal-state-changed":
case "fxa-migration:internal-telemetry":
return this._observeMigration(subject, topic, data);
}
Cu.reportError("unexpected topic in sync healthreport provider: " + topic);
},
_observeSync: function (subject, topic, data) {
let field;
switch (topic) {
case "weave:service:sync:start":
field = "syncStart";
break;
case "weave:service:sync:finish":
field = "syncSuccess";
break;
case "weave:service:sync:error":
field = "syncError";
break;
default:
Cu.reportError("unexpected sync topic in sync healthreport provider: " + topic);
return;
}
let m = this.getMeasurement(SyncMeasurement1.prototype.name,
SyncMeasurement1.prototype.version);
return this.enqueueStorageOperation(function recordSyncEvent() {
return m.incrementDailyCounter(field);
});
},
_observeMigration: function(subject, topic, data) {
switch (topic) {
case "fxa-migration:state-changed":
case "fxa-migration:internal-state-changed": {
// We record both "user" and "internal" states in the same field. This
// works for us as user state is always null when there is an internal
// state.
if (!data) {
return; // we don't count the |null| state
}
let m = this.getMeasurement(SyncMigrationMeasurement1.prototype.name,
SyncMigrationMeasurement1.prototype.version);
return this.enqueueStorageOperation(function() {
return m.setDailyLastText("state", data);
});
}
case "fxa-migration:internal-telemetry": {
// |data| is our field name.
let m = this.getMeasurement(SyncMigrationMeasurement1.prototype.name,
SyncMigrationMeasurement1.prototype.version);
return this.enqueueStorageOperation(function() {
switch (data) {
case "accepted":
case "declined":
return m.incrementDailyCounter(data);
case "unlinked":
return m.setDailyLastNumeric(data, 1);
default:
Cu.reportError("Unexpected migration field in sync healthreport provider: " + data);
return Promise.resolve();
}
});
}
default:
Cu.reportError("unexpected migration topic in sync healthreport provider: " + topic);
return;
}
},
collectDailyData: function () {
return this.storage.enqueueTransaction(this._populateDailyData.bind(this));
},
_populateDailyData: function* () {
let m = this.getMeasurement(SyncMeasurement1.prototype.name,
SyncMeasurement1.prototype.version);
let svc = Cc["@mozilla.org/weave/service;1"]
.getService(Ci.nsISupports)
.wrappedJSObject;
let enabled = svc.enabled;
yield m.setDailyLastNumeric("enabled", enabled ? 1 : 0);
// preferredProtocol is constant and only changes as the client
// evolves.
yield m.setDailyLastText("preferredProtocol", "1.5");
let protocol = svc.fxAccountsEnabled ? "1.5" : "1.1";
yield m.setDailyLastText("activeProtocol", protocol);
if (!enabled) {
return;
}
// Before grabbing more information, be sure the Sync service
// is fully initialized. This has the potential to initialize
// Sync on the spot. This may be undesired if Sync appears to
// be enabled but it really isn't. That responsibility should
// be up to svc.enabled to not return false positives, however.
yield svc.whenLoaded();
if (Weave.Status.service != Weave.STATUS_OK) {
return;
}
// Device types are dynamic. So we need to dynamically create fields if
// they don't exist.
let dm = this.getMeasurement(SyncDevicesMeasurement1.prototype.name,
SyncDevicesMeasurement1.prototype.version);
let devices = Weave.Service.clientsEngine.deviceTypes;
for (let [field, count] of devices) {
let hasField = this.storage.hasFieldFromMeasurement(dm.id, field,
this.storage.FIELD_DAILY_LAST_NUMERIC);
let fieldID;
if (hasField) {
fieldID = this.storage.fieldIDFromMeasurement(dm.id, field);
} else {
fieldID = yield this.storage.registerField(dm.id, field,
this.storage.FIELD_DAILY_LAST_NUMERIC);
}
yield this.storage.setDailyLastNumericFromFieldID(fieldID, count);
}
},
});

43
services/sync/modules/identity.js
View file

@ -6,14 +6,13 @@
this.EXPORTED_SYMBOLS = ["IdentityManager"];
var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Promise.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/async.js");
// Lazy import to prevent unnecessary load on startup.
for (let symbol of ["BulkKeyBundle", "SyncKeyBundle"]) {
@ -85,14 +84,18 @@ IdentityManager.prototype = {
_syncKeyBundle: null,
/**
* Initialize the identity provider.
* Initialize the identity provider. Returns a promise that is resolved
* when initialization is complete and the provider can be queried for
* its state
*/
initialize: function() {
// Nothing to do for this identity provider.
return Promise.resolve();
},
finalize: function() {
// Nothing to do for this identity provider.
return Promise.resolve();
},
/**
@ -111,6 +114,14 @@ IdentityManager.prototype = {
return Promise.resolve();
},
/**
* Indicates if the identity manager is still initializing
*/
get readyToAuthenticate() {
// We initialize in a fully sync manner, so we are always finished.
return true;
},
get account() {
return Svc.Prefs.get("account", this.username);
},
@ -195,7 +206,7 @@ IdentityManager.prototype = {
return null;
}
for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) {
for each (let login in this._getLogins(PWDMGR_PASSWORD_REALM)) {
if (login.username.toLowerCase() == username) {
// It should already be UTF-8 encoded, but we don't take any chances.
this._basicPassword = Utils.encodeUTF8(login.password);
@ -249,7 +260,7 @@ IdentityManager.prototype = {
return null;
}
for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
for each (let login in this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
if (login.username.toLowerCase() == username) {
this._syncKey = login.password;
}
@ -326,7 +337,7 @@ IdentityManager.prototype = {
try {
this._syncKeyBundle = new SyncKeyBundle(this.username, this.syncKey);
} catch (ex) {
this._log.warn("Failed to create sync bundle", ex);
this._log.warn(Utils.exceptionStr(ex));
return null;
}
}
@ -400,7 +411,7 @@ IdentityManager.prototype = {
this._setLogin(PWDMGR_PASSWORD_REALM, this.username,
this._basicPassword);
} else {
for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) {
for each (let login in this._getLogins(PWDMGR_PASSWORD_REALM)) {
Services.logins.removeLogin(login);
}
}
@ -412,7 +423,7 @@ IdentityManager.prototype = {
if (this._syncKey) {
this._setLogin(PWDMGR_PASSPHRASE_REALM, this.username, this._syncKey);
} else {
for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
for each (let login in this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
Services.logins.removeLogin(login);
}
}
@ -447,9 +458,6 @@ IdentityManager.prototype = {
try {
service.recordManager.get(service.storageURL + "meta/fxa_credentials");
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("Failed to pre-fetch the migration sentinel", ex);
}
},
@ -469,7 +477,7 @@ IdentityManager.prototype = {
*/
_setLogin: function _setLogin(realm, username, password) {
let exists = false;
for (let login of this._getLogins(realm)) {
for each (let login in this._getLogins(realm)) {
if (login.username == username && login.password == password) {
exists = true;
} else {
@ -505,7 +513,7 @@ IdentityManager.prototype = {
deleteSyncCredentials: function deleteSyncCredentials() {
for (let host of this._getSyncCredentialsHosts()) {
let logins = Services.logins.findLogins({}, host, "", "");
for (let login of logins) {
for each (let login in logins) {
Services.logins.removeLogin(login);
}
}
@ -593,13 +601,4 @@ IdentityManager.prototype = {
// Do nothing for Sync 1.1.
return {accepted: true};
},
// Tell Sync what the login status should be if it saw a 401 fetching
// info/collections as part of login verification (typically immediately
// after login.)
// In our case it means an authoritative "password is incorrect".
loginStatusFromVerification404() {
return LOGIN_FAILED_LOGIN_REJECTED;
}
};

8
services/sync/modules/jpakeclient.js
View file

@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["JPAKEClient", "SendCredentialsController"];
var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");
@ -281,7 +281,8 @@ JPAKEClient.prototype = {
let rng = Cc["@mozilla.org/security/random-generator;1"]
.createInstance(Ci.nsIRandomGenerator);
let bytes = rng.generateRandomBytes(JPAKE_LENGTH_CLIENTID / 2);
this._clientID = bytes.map(byte => ("0" + byte.toString(16)).slice(-2)).join("");
this._clientID = [("0" + byte.toString(16)).slice(-2)
for each (byte in bytes)].join("");
},
_createSecret: function _createSecret() {
@ -290,7 +291,8 @@ JPAKEClient.prototype = {
let rng = Cc["@mozilla.org/security/random-generator;1"]
.createInstance(Ci.nsIRandomGenerator);
let bytes = rng.generateRandomBytes(JPAKE_LENGTH_SECRET);
return bytes.map(byte => key[Math.floor(byte * key.length / 256)]).join("");
return [key[Math.floor(byte * key.length / 256)]
for each (byte in bytes)].join("");
},
_newRequest: function _newRequest(uri) {

2
services/sync/modules/keys.js
View file

@ -9,7 +9,7 @@ this.EXPORTED_SYMBOLS = [
"SyncKeyBundle"
];
var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");

16
services/sync/modules/main.js
View file

@ -6,7 +6,7 @@ this.EXPORTED_SYMBOLS = ['Weave'];
this.Weave = {};
Components.utils.import("resource://services-sync/constants.js", Weave);
var lazies = {
let lazies = {
"jpakeclient.js": ["JPAKEClient", "SendCredentialsController"],
"notifications.js": ["Notifications", "Notification", "NotificationButton"],
"service.js": ["Service"],
@ -15,14 +15,12 @@ var lazies = {
};
function lazyImport(module, dest, props) {
function getter(prop) {
return function() {
let ns = {};
Components.utils.import(module, ns);
delete dest[prop];
return dest[prop] = ns[prop];
};
}
function getter(prop) function() {
let ns = {};
Components.utils.import(module, ns);
delete dest[prop];
return dest[prop] = ns[prop];
};
props.forEach(function (prop) { dest.__defineGetter__(prop, getter(prop)); });
}

8
services/sync/modules/notifications.js
View file

@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ["Notifications", "Notification", "NotificationButton"];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
Cu.import("resource://services-common/observers.js");
Cu.import("resource://gre/modules/Log.jsm");

211
services/sync/modules/policies.js
View file

@ -7,26 +7,16 @@ this.EXPORTED_SYMBOLS = [
"SyncScheduler",
];
var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/logmanager.js");
Cu.import("resource://services-common/async.js");
XPCOMUtils.defineLazyModuleGetter(this, "Status",
"resource://services-sync/status.js");
XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
"resource://gre/modules/AddonManager.jsm");
// Get the value for an interval that's stored in preferences. To save users
// from themselves (and us from them!) the minimum time they can specify
// is 60s.
function getThrottledIntervalPreference(prefName) {
return Math.max(Svc.Prefs.get(prefName), 60) * 1000;
}
this.SyncScheduler = function SyncScheduler(service) {
this.service = service;
@ -55,12 +45,12 @@ SyncScheduler.prototype = {
let part = service.fxAccountsEnabled ? "fxa" : "sync11";
let prefSDInterval = "scheduler." + part + ".singleDeviceInterval";
this.singleDeviceInterval = getThrottledIntervalPreference(prefSDInterval);
this.singleDeviceInterval = Svc.Prefs.get(prefSDInterval) * 1000;
this.idleInterval = getThrottledIntervalPreference("scheduler.idleInterval");
this.activeInterval = getThrottledIntervalPreference("scheduler.activeInterval");
this.immediateInterval = getThrottledIntervalPreference("scheduler.immediateInterval");
this.eolInterval = getThrottledIntervalPreference("scheduler.eolInterval");
this.idleInterval = Svc.Prefs.get("scheduler.idleInterval") * 1000;
this.activeInterval = Svc.Prefs.get("scheduler.activeInterval") * 1000;
this.immediateInterval = Svc.Prefs.get("scheduler.immediateInterval") * 1000;
this.eolInterval = Svc.Prefs.get("scheduler.eolInterval") * 1000;
// A user is non-idle on startup by default.
this.idle = false;
@ -71,40 +61,20 @@ SyncScheduler.prototype = {
},
// nextSync is in milliseconds, but prefs can't hold that much
get nextSync() {
return Svc.Prefs.get("nextSync", 0) * 1000;
},
set nextSync(value) {
Svc.Prefs.set("nextSync", Math.floor(value / 1000));
},
get nextSync() Svc.Prefs.get("nextSync", 0) * 1000,
set nextSync(value) Svc.Prefs.set("nextSync", Math.floor(value / 1000)),
get syncInterval() {
return Svc.Prefs.get("syncInterval", this.singleDeviceInterval);
},
set syncInterval(value) {
Svc.Prefs.set("syncInterval", value);
},
get syncInterval() Svc.Prefs.get("syncInterval", this.singleDeviceInterval),
set syncInterval(value) Svc.Prefs.set("syncInterval", value),
get syncThreshold() {
return Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD);
},
set syncThreshold(value) {
Svc.Prefs.set("syncThreshold", value);
},
get syncThreshold() Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD),
set syncThreshold(value) Svc.Prefs.set("syncThreshold", value),
get globalScore() {
return Svc.Prefs.get("globalScore", 0);
},
set globalScore(value) {
Svc.Prefs.set("globalScore", value);
},
get globalScore() Svc.Prefs.get("globalScore", 0),
set globalScore(value) Svc.Prefs.set("globalScore", value),
get numClients() {
return Svc.Prefs.get("numClients", 0);
},
set numClients(value) {
Svc.Prefs.set("numClients", value);
},
get numClients() Svc.Prefs.get("numClients", 0),
set numClients(value) Svc.Prefs.set("numClients", value),
init: function init() {
this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")];
@ -249,10 +219,7 @@ SyncScheduler.prototype = {
this.setDefaults();
try {
Svc.Idle.removeIdleObserver(this, Svc.Prefs.get("scheduler.idleTime"));
} catch (ex) {
if (ex.result != Cr.NS_ERROR_FAILURE) {
throw ex;
}
} catch (ex if (ex.result == Cr.NS_ERROR_FAILURE)) {
// In all likelihood we didn't have an idle observer registered yet.
// It's all good.
}
@ -285,11 +252,10 @@ SyncScheduler.prototype = {
case "wake_notification":
this._log.debug("Woke from sleep.");
Utils.nextTick(() => {
// Trigger a sync if we have multiple clients. We give it 5 seconds
// incase the network is still in the process of coming back up.
// Trigger a sync if we have multiple clients.
if (this.numClients > 1) {
this._log.debug("More than 1 client. Will sync in 5s.");
this.scheduleNextSync(5000);
this._log.debug("More than 1 client. Syncing.");
this.scheduleNextSync(0);
}
});
break;
@ -531,6 +497,45 @@ SyncScheduler.prototype = {
this.syncTimer.clear();
},
/**
* Prevent new syncs from starting. This is used by the FxA migration code
* where we can't afford to have a sync start partway through the migration.
* To handle the edge-case of a sync starting and not stopping, we store
* this state in a pref, so on the next startup we remain blocked (and thus
* sync will never start) so the migration can complete.
*
* As a safety measure, we only block for some period of time, and after
* that it will automatically unblock. This ensures that if things go
* really pear-shaped and we never end up calling unblockSync() we haven't
* completely broken the world.
*/
blockSync: function(until = null) {
if (!until) {
until = Date.now() + DEFAULT_BLOCK_PERIOD;
}
// until is specified in ms, but Prefs can't hold that much
Svc.Prefs.set("scheduler.blocked-until", Math.floor(until / 1000));
},
unblockSync: function() {
Svc.Prefs.reset("scheduler.blocked-until");
// the migration code should be ready to roll, so resume normal operations.
this.checkSyncStatus();
},
get isBlocked() {
let until = Svc.Prefs.get("scheduler.blocked-until");
if (until === undefined) {
return false;
}
if (until <= Math.floor(Date.now() / 1000)) {
// we were previously blocked but the time has expired.
Svc.Prefs.reset("scheduler.blocked-until");
return false;
}
// we remain blocked.
return true;
},
};
this.ErrorHandler = function ErrorHandler(service) {
@ -570,10 +575,7 @@ ErrorHandler.prototype = {
root.level = Log.Level[Svc.Prefs.get("log.rootLogger")];
let logs = ["Sync", "FirefoxAccounts", "Hawk", "Common.TokenServerClient",
"Sync.SyncMigration", "browserwindow.syncui",
"Services.Common.RESTRequest", "Services.Common.RESTRequest",
"BookmarkSyncUtils"
];
"Sync.SyncMigration"];
this._logManager = new LogManager(Svc.Prefs, logs, "sync");
},
@ -590,25 +592,17 @@ ErrorHandler.prototype = {
this._log.debug(data + " failed to apply some records.");
}
break;
case "weave:engine:sync:error": {
case "weave:engine:sync:error":
let exception = subject; // exception thrown by engine's sync() method
let engine_name = data; // engine name that threw the exception
this.checkServerError(exception);
Status.engines = [engine_name, exception.failureCode || ENGINE_UNKNOWN_FAIL];
if (Async.isShutdownException(exception)) {
this._log.debug(engine_name + " was interrupted due to the application shutting down");
} else {
this._log.debug(engine_name + " failed", exception);
Services.telemetry.getKeyedHistogramById("WEAVE_ENGINE_SYNC_ERRORS")
.add(engine_name);
}
this._log.debug(engine_name + " failed: " + Utils.exceptionStr(exception));
break;
}
case "weave:service:login:error":
this._log.error("Sync encountered a login error");
this.resetFileLog();
this.resetFileLog(this._logManager.REASON_ERROR);
if (this.shouldReportError()) {
this.notifyOnNextTick("weave:ui:login:error");
@ -618,23 +612,12 @@ ErrorHandler.prototype = {
this.dontIgnoreErrors = false;
break;
case "weave:service:sync:error": {
case "weave:service:sync:error":
if (Status.sync == CREDENTIALS_CHANGED) {
this.service.logout();
}
let exception = subject;
if (Async.isShutdownException(exception)) {
// If we are shutting down we just log the fact, attempt to flush
// the log file and get out of here!
this._log.error("Sync was interrupted due to the application shutting down");
this.resetFileLog();
break;
}
// Not a shutdown related exception...
this._log.error("Sync encountered an error", exception);
this.resetFileLog();
this.resetFileLog(this._logManager.REASON_ERROR);
if (this.shouldReportError()) {
this.notifyOnNextTick("weave:ui:sync:error");
@ -644,7 +627,6 @@ ErrorHandler.prototype = {
this.dontIgnoreErrors = false;
break;
}
case "weave:service:sync:finish":
this._log.trace("Status.service is " + Status.service);
@ -660,8 +642,8 @@ ErrorHandler.prototype = {
}
if (Status.service == SYNC_FAILED_PARTIAL) {
this._log.error("Some engines did not sync correctly.");
this.resetFileLog();
this._log.debug("Some engines did not sync correctly.");
this.resetFileLog(this._logManager.REASON_ERROR);
if (this.shouldReportError()) {
this.dontIgnoreErrors = false;
@ -669,7 +651,7 @@ ErrorHandler.prototype = {
break;
}
} else {
this.resetFileLog();
this.resetFileLog(this._logManager.REASON_SUCCESS);
}
this.dontIgnoreErrors = false;
this.notifyOnNextTick("weave:ui:sync:finish");
@ -696,52 +678,22 @@ ErrorHandler.prototype = {
Utils.nextTick(this.service.sync, this.service);
},
_dumpAddons: function _dumpAddons() {
// Just dump the items that sync may be concerned with. Specifically,
// active extensions that are not hidden.
let addonPromise = new Promise(resolve => {
try {
AddonManager.getAddonsByTypes(["extension"], resolve);
} catch (e) {
this._log.warn("Failed to dump addons", e)
resolve([])
}
});
return addonPromise.then(addons => {
let relevantAddons = addons.filter(x => x.isActive && !x.hidden);
this._log.debug("Addons installed", relevantAddons.length);
for (let addon of relevantAddons) {
this._log.debug(" - ${name}, version ${version}, id ${id}", addon);
}
});
},
/**
* Generate a log file for the sync that just completed
* and refresh the input & output streams.
*
* @param reason
* A constant from the LogManager that indicates the reason for the
* reset.
*/
resetFileLog: function resetFileLog() {
let onComplete = logType => {
resetFileLog: function resetFileLog(reason) {
let onComplete = () => {
Svc.Obs.notify("weave:service:reset-file-log");
this._log.trace("Notified: " + Date.now());
if (logType == this._logManager.ERROR_LOG_WRITTEN) {
Cu.reportError("Sync encountered an error - see about:sync-log for the log file.");
}
};
// If we're writing an error log, dump extensions that may be causing problems.
let beforeResetLog;
if (this._logManager.sawError) {
beforeResetLog = this._dumpAddons();
} else {
beforeResetLog = Promise.resolve();
}
// Note we do not return the promise here - the caller doesn't need to wait
// for this to complete.
beforeResetLog
.then(() => this._logManager.resetFileLog())
.then(onComplete, onComplete);
this._logManager.resetFileLog(reason).then(onComplete, onComplete);
},
/**
@ -775,9 +727,6 @@ ErrorHandler.prototype = {
}
},
// A function to indicate if Sync errors should be "reported" - which in this
// context really means "should be notify observers of an error" - but note
// that since bug 1180587, no one is going to surface an error to the user.
shouldReportError: function shouldReportError() {
if (Status.login == MASTER_PASSWORD_LOCKED) {
this._log.trace("shouldReportError: false (master password locked).");
@ -817,12 +766,8 @@ ErrorHandler.prototype = {
return false;
}
let result = ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 &&
[Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1);
this._log.trace("shouldReportError: ${result} due to login=${login}, sync=${sync}",
{result, login: Status.login, sync: Status.sync});
return result;
return ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 &&
[Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1);
},
get currentAlertMode() {
@ -925,7 +870,7 @@ ErrorHandler.prototype = {
case 401:
this.service.logout();
this._log.info("Got 401 response; resetting clusterURL.");
this.service.clusterURL = null;
Svc.Prefs.reset("clusterURL");
let delay = 0;
if (Svc.Prefs.get("lastSyncReassigned")) {

481
services/sync/modules/record.js
View file

@ -10,10 +10,10 @@ this.EXPORTED_SYMBOLS = [
"Collection",
];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
const CRYPTO_COLLECTION = "crypto";
const KEYS_WBO = "keys";
@ -23,7 +23,6 @@ Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/keys.js");
Cu.import("resource://services-sync/resource.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/async.js");
this.WBORecord = function WBORecord(collection, id) {
this.data = {};
@ -86,7 +85,7 @@ WBORecord.prototype = {
toJSON: function toJSON() {
// Copy fields from data to be stringified, making sure payload is a string
let obj = {};
for (let [key, val] of Object.entries(this.data))
for (let [key, val] in Iterator(this.data))
obj[key] = key == "payload" ? JSON.stringify(val) : val;
if (this.ttl)
obj.ttl = this.ttl;
@ -196,9 +195,7 @@ CryptoWrapper.prototype = {
},
// The custom setter below masks the parent's getter, so explicitly call it :(
get id() {
return WBORecord.prototype.__lookupGetter__("id").call(this);
},
get id() WBORecord.prototype.__lookupGetter__("id").call(this),
// Keep both plaintext and encrypted versions of the id to verify integrity
set id(val) {
@ -238,11 +235,8 @@ RecordManager.prototype = {
record.deserialize(this.response);
return this.set(url, record);
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.debug("Failed to import record", ex);
} catch(ex) {
this._log.debug("Failed to import record: " + Utils.exceptionStr(ex));
return null;
}
},
@ -281,10 +275,10 @@ RecordManager.prototype = {
* You can update this thing simply by giving it /info/collections. It'll
* use the last modified time to bring itself up to date.
*/
this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_, collections) {
this.lastModified = lastModified || 0;
this._default = default_ || null;
this._collections = collections || {};
this.CollectionKeyManager = function CollectionKeyManager() {
this.lastModified = 0;
this._collections = {};
this._default = null;
this._log = Log.repository.getLogger("Sync.CollectionKeyManager");
}
@ -293,19 +287,6 @@ this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_
// Note that the last modified time needs to be preserved.
CollectionKeyManager.prototype = {
/**
* Generate a new CollectionKeyManager that has the same attributes
* as this one.
*/
clone() {
const newCollections = {};
for (let c in this._collections) {
newCollections[c] = this._collections[c];
}
return new CollectionKeyManager(this.lastModified, this._default, newCollections);
},
// Return information about old vs new keys:
// * same: true if two collections are equal
// * changed: an array of collection names that changed.
@ -328,7 +309,7 @@ CollectionKeyManager.prototype = {
// Return a sorted, unique array.
changed.sort();
let last;
changed = changed.filter(x => (x != last) && (last = x));
changed = [x for each (x in changed) if ((x != last) && (last = x))];
return {same: changed.length == 0,
changed: changed};
},
@ -374,15 +355,15 @@ CollectionKeyManager.prototype = {
/**
* Create a WBO for the current keys.
*/
asWBO: function(collection, id) {
return this._makeWBO(this._collections, this._default);
},
asWBO: function(collection, id)
this._makeWBO(this._collections, this._default),
/**
* Compute a new default key, and new keys for any specified collections.
*/
newKeys: function(collections) {
let newDefaultKeyBundle = this.newDefaultKeyBundle();
let newDefaultKey = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
newDefaultKey.generateRandom();
let newColls = {};
if (collections) {
@ -392,7 +373,7 @@ CollectionKeyManager.prototype = {
newColls[c] = b;
});
}
return [newDefaultKeyBundle, newColls];
return [newDefaultKey, newColls];
},
/**
@ -406,57 +387,6 @@ CollectionKeyManager.prototype = {
return this._makeWBO(newColls, newDefaultKey);
},
/**
* Create a new default key.
*
* @returns {BulkKeyBundle}
*/
newDefaultKeyBundle() {
const key = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
key.generateRandom();
return key;
},
/**
* Create a new default key and store it as this._default, since without one you cannot use setContents.
*/
generateDefaultKey() {
this._default = this.newDefaultKeyBundle();
},
/**
* Return true if keys are already present for each of the given
* collections.
*/
hasKeysFor(collections) {
// We can't use filter() here because sometimes collections is an iterator.
for (let collection of collections) {
if (!this._collections[collection]) {
return false;
}
}
return true;
},
/**
* Return a new CollectionKeyManager that has keys for each of the
* given collections (creating new ones for collections where we
* don't already have keys).
*/
ensureKeysFor(collections) {
const newKeys = Object.assign({}, this._collections);
for (let c of collections) {
if (newKeys[c]) {
continue; // don't replace existing keys
}
const b = new BulkKeyBundle(c);
b.generateRandom();
newKeys[c] = b;
}
return new CollectionKeyManager(this.lastModified, this._default, newKeys);
},
// Take the fetched info/collections WBO, checking the change
// time of the crypto collection.
updateNeeded: function(info_collections) {
@ -487,6 +417,9 @@ CollectionKeyManager.prototype = {
//
setContents: function setContents(payload, modified) {
if (!modified)
throw "No modified time provided to setContents.";
let self = this;
this._log.info("Setting collection keys contents. Our last modified: " +
@ -516,7 +449,9 @@ CollectionKeyManager.prototype = {
if (v) {
let keyObj = new BulkKeyBundle(k);
keyObj.keyPairB64 = v;
newCollections[k] = keyObj;
if (keyObj) {
newCollections[k] = keyObj;
}
}
}
}
@ -527,11 +462,8 @@ CollectionKeyManager.prototype = {
let sameColls = collComparison.same;
if (sameDefault && sameColls) {
self._log.info("New keys are the same as our old keys!");
if (modified) {
self._log.info("Bumped local modified time.");
self.lastModified = modified;
}
self._log.info("New keys are the same as our old keys! Bumped local modified time.");
self.lastModified = modified;
return false;
}
@ -543,10 +475,8 @@ CollectionKeyManager.prototype = {
this._collections = newCollections;
// Always trust the server.
if (modified) {
self._log.info("Bumping last modified to " + modified);
self.lastModified = modified;
}
self._log.info("Bumping last modified to " + modified);
self.lastModified = modified;
return sameDefault ? collComparison.changed : true;
},
@ -594,12 +524,6 @@ this.Collection = function Collection(uri, recordObj, service) {
this._older = 0;
this._newer = 0;
this._data = [];
// optional members used by batch upload operations.
this._batch = null;
this._commit = false;
// Used for batch download operations -- note that this is explicitly an
// opaque value and not (necessarily) a number.
this._offset = null;
}
Collection.prototype = {
__proto__: Resource.prototype,
@ -623,12 +547,6 @@ Collection.prototype = {
args.push("ids=" + this.ids);
if (this.limit > 0 && this.limit != Infinity)
args.push("limit=" + this.limit);
if (this._batch)
args.push("batch=" + encodeURIComponent(this._batch));
if (this._commit)
args.push("commit=true");
if (this._offset)
args.push("offset=" + encodeURIComponent(this._offset));
this.uri.query = (args.length > 0)? '?' + args.join('&') : '';
},
@ -641,14 +559,14 @@ Collection.prototype = {
},
// Apply the action to a certain set of ids
get ids() { return this._ids; },
get ids() this._ids,
set ids(value) {
this._ids = value;
this._rebuildURL();
},
// Limit how many records to get
get limit() { return this._limit; },
get limit() this._limit,
set limit(value) {
this._limit = value;
this._rebuildURL();
@ -678,100 +596,12 @@ Collection.prototype = {
this._rebuildURL();
},
get offset() { return this._offset; },
set offset(value) {
this._offset = value;
this._rebuildURL();
pushData: function Coll_pushData(data) {
this._data.push(data);
},
// Set information about the batch for this request.
get batch() { return this._batch; },
set batch(value) {
this._batch = value;
this._rebuildURL();
},
get commit() { return this._commit; },
set commit(value) {
this._commit = value && true;
this._rebuildURL();
},
// Similar to get(), but will page through the items `batchSize` at a time,
// deferring calling the record handler until we've gotten them all.
//
// Returns the last response processed, and doesn't run the record handler
// on any items if a non-success status is received while downloading the
// records (or if a network error occurs).
getBatched(batchSize = DEFAULT_DOWNLOAD_BATCH_SIZE) {
let totalLimit = Number(this.limit) || Infinity;
if (batchSize <= 0 || batchSize >= totalLimit) {
// Invalid batch sizes should arguably be an error, but they're easy to handle
return this.get();
}
if (!this.full) {
throw new Error("getBatched is unimplemented for guid-only GETs");
}
// _onComplete and _onProgress are reset after each `get` by AsyncResource.
// We overwrite _onRecord to something that stores the data in an array
// until the end.
let { _onComplete, _onProgress, _onRecord } = this;
let recordBuffer = [];
let resp;
try {
this._onRecord = r => recordBuffer.push(r);
let lastModifiedTime;
this.limit = batchSize;
do {
this._onProgress = _onProgress;
this._onComplete = _onComplete;
if (batchSize + recordBuffer.length > totalLimit) {
this.limit = totalLimit - recordBuffer.length;
}
this._log.trace("Performing batched GET", { limit: this.limit, offset: this.offset });
// Actually perform the request
resp = this.get();
if (!resp.success) {
break;
}
// Initialize last modified, or check that something broken isn't happening.
let lastModified = resp.headers["x-last-modified"];
if (!lastModifiedTime) {
lastModifiedTime = lastModified;
this.setHeader("X-If-Unmodified-Since", lastModified);
} else if (lastModified != lastModifiedTime) {
// Should be impossible -- We'd get a 412 in this case.
throw new Error("X-Last-Modified changed in the middle of a download batch! " +
`${lastModified} => ${lastModifiedTime}`)
}
// If this is missing, we're finished.
this.offset = resp.headers["x-weave-next-offset"];
} while (this.offset && totalLimit > recordBuffer.length);
} finally {
// Ensure we undo any temporary state so that subsequent calls to get()
// or getBatched() work properly. We do this before calling the record
// handler so that we can more convincingly pretend to be a normal get()
// call. Note: we're resetting these to the values they had before this
// function was called.
this._onRecord = _onRecord;
this._limit = totalLimit;
this._offset = null;
delete this._headers["x-if-unmodified-since"];
this._rebuildURL();
}
if (resp.success && Async.checkAppReady()) {
// call the original _onRecord (e.g. the user supplied record handler)
// for each record we've stored
for (let record of recordBuffer) {
this._onRecord(record);
}
}
return resp;
clearRecords: function Coll_clearRecords() {
this._data = [];
},
set recordHandler(onRecord) {
@ -781,8 +611,6 @@ Collection.prototype = {
// Switch to newline separated records for incremental parsing
coll.setHeader("Accept", "application/newlines");
this._onRecord = onRecord;
this._onProgress = function() {
let newline;
while ((newline = this._data.indexOf("\n")) > 0) {
@ -793,247 +621,8 @@ Collection.prototype = {
// Deserialize a record from json and give it to the callback
let record = new coll._recordObj();
record.deserialize(json);
coll._onRecord(record);
onRecord(record);
}
};
},
// This object only supports posting via the postQueue object.
post() {
throw new Error("Don't directly post to a collection - use newPostQueue instead");
},
newPostQueue(log, timestamp, postCallback) {
let poster = (data, headers, batch, commit) => {
this.batch = batch;
this.commit = commit;
for (let [header, value] of headers) {
this.setHeader(header, value);
}
return Resource.prototype.post.call(this, data);
}
let getConfig = (name, defaultVal) => {
if (this._service.serverConfiguration && this._service.serverConfiguration.hasOwnProperty(name)) {
return this._service.serverConfiguration[name];
}
return defaultVal;
}
let config = {
max_post_bytes: getConfig("max_post_bytes", MAX_UPLOAD_BYTES),
max_post_records: getConfig("max_post_records", MAX_UPLOAD_RECORDS),
max_batch_bytes: getConfig("max_total_bytes", Infinity),
max_batch_records: getConfig("max_total_records", Infinity),
}
// Handle config edge cases
if (config.max_post_records <= 0) { config.max_post_records = MAX_UPLOAD_RECORDS; }
if (config.max_batch_records <= 0) { config.max_batch_records = Infinity; }
if (config.max_post_bytes <= 0) { config.max_post_bytes = MAX_UPLOAD_BYTES; }
if (config.max_batch_bytes <= 0) { config.max_batch_bytes = Infinity; }
// Max size of BSO payload is 256k. This assumes at most 4k of overhead,
// which sounds like plenty. If the server says it can't handle this, we
// might have valid records we can't sync, so we give up on syncing.
let requiredMax = 260 * 1024;
if (config.max_post_bytes < requiredMax) {
this._log.error("Server configuration max_post_bytes is too low", config);
throw new Error("Server configuration max_post_bytes is too low");
}
return new PostQueue(poster, timestamp, config, log, postCallback);
},
};
/* A helper to manage the posting of records while respecting the various
size limits.
This supports the concept of a server-side "batch". The general idea is:
* We queue as many records as allowed in memory, then make a single POST.
* This first POST (optionally) gives us a batch ID, which we use for
all subsequent posts, until...
* At some point we hit a batch-maximum, and jump through a few hoops to
commit the current batch (ie, all previous POSTs) and start a new one.
* Eventually commit the final batch.
In most cases we expect there to be exactly 1 batch consisting of possibly
multiple POSTs.
*/
function PostQueue(poster, timestamp, config, log, postCallback) {
// The "post" function we should use when it comes time to do the post.
this.poster = poster;
this.log = log;
// The config we use. We expect it to have fields "max_post_records",
// "max_batch_records", "max_post_bytes", and "max_batch_bytes"
this.config = config;
// The callback we make with the response when we do get around to making the
// post (which could be during any of the enqueue() calls or the final flush())
// This callback may be called multiple times and must not add new items to
// the queue.
// The second argument passed to this callback is a boolean value that is true
// if we're in the middle of a batch, and false if either the batch is
// complete, or it's a post to a server that does not understand batching.
this.postCallback = postCallback;
// The string where we are capturing the stringified version of the records
// queued so far. It will always be invalid JSON as it is always missing the
// closing bracket.
this.queued = "";
// The number of records we've queued so far but are yet to POST.
this.numQueued = 0;
// The number of records/bytes we've processed in previous POSTs for our
// current batch. Does *not* include records currently queued for the next POST.
this.numAlreadyBatched = 0;
this.bytesAlreadyBatched = 0;
// The ID of our current batch. Can be undefined (meaning we are yet to make
// the first post of a patch, so don't know if we have a batch), null (meaning
// we've made the first post but the server response indicated no batching
// semantics), otherwise we have made the first post and it holds the batch ID
// returned from the server.
this.batchID = undefined;
// Time used for X-If-Unmodified-Since -- should be the timestamp from the last GET.
this.lastModified = timestamp;
}
PostQueue.prototype = {
enqueue(record) {
// We want to ensure the record has a .toJSON() method defined - even
// though JSON.stringify() would implicitly call it, the stringify might
// still work even if it isn't defined, which isn't what we want.
let jsonRepr = record.toJSON();
if (!jsonRepr) {
throw new Error("You must only call this with objects that explicitly support JSON");
}
let bytes = JSON.stringify(jsonRepr);
// Do a flush if we can't add this record without exceeding our single-request
// limits, or without exceeding the total limit for a single batch.
let newLength = this.queued.length + bytes.length + 2; // extras for leading "[" / "," and trailing "]"
let maxAllowedBytes = Math.min(256 * 1024, this.config.max_post_bytes);
let postSizeExceeded = this.numQueued >= this.config.max_post_records ||
newLength >= maxAllowedBytes;
let batchSizeExceeded = (this.numQueued + this.numAlreadyBatched) >= this.config.max_batch_records ||
(newLength + this.bytesAlreadyBatched) >= this.config.max_batch_bytes;
let singleRecordTooBig = bytes.length + 2 > maxAllowedBytes;
if (postSizeExceeded || batchSizeExceeded) {
this.log.trace(`PostQueue flushing due to postSizeExceeded=${postSizeExceeded}, batchSizeExceeded=${batchSizeExceeded}` +
`, max_batch_bytes: ${this.config.max_batch_bytes}, max_post_bytes: ${this.config.max_post_bytes}`);
if (singleRecordTooBig) {
return { enqueued: false, error: new Error("Single record too large to submit to server") };
}
// We need to write the queue out before handling this one, but we only
// commit the batch (and thus start a new one) if the batch is full.
// Note that if a single record is too big for the batch or post, then
// the batch may be empty, and so we don't flush in that case.
if (this.numQueued) {
this.flush(batchSizeExceeded || singleRecordTooBig);
}
}
// Either a ',' or a '[' depending on whether this is the first record.
this.queued += this.numQueued ? "," : "[";
this.queued += bytes;
this.numQueued++;
return { enqueued: true };
},
flush(finalBatchPost) {
if (!this.queued) {
// nothing queued - we can't be in a batch, and something has gone very
// bad if we think we are.
if (this.batchID) {
throw new Error(`Flush called when no queued records but we are in a batch ${this.batchID}`);
}
return;
}
// the batch query-param and headers we'll send.
let batch;
let headers = [];
if (this.batchID === undefined) {
// First commit in a (possible) batch.
batch = "true";
} else if (this.batchID) {
// We have an existing batch.
batch = this.batchID;
} else {
// Not the first post and we know we have no batch semantics.
batch = null;
}
headers.push(["x-if-unmodified-since", this.lastModified]);
this.log.info(`Posting ${this.numQueued} records of ${this.queued.length+1} bytes with batch=${batch}`);
let queued = this.queued + "]";
if (finalBatchPost) {
this.bytesAlreadyBatched = 0;
this.numAlreadyBatched = 0;
} else {
this.bytesAlreadyBatched += queued.length;
this.numAlreadyBatched += this.numQueued;
}
this.queued = "";
this.numQueued = 0;
let response = this.poster(queued, headers, batch, !!(finalBatchPost && this.batchID !== null));
if (!response.success) {
this.log.trace("Server error response during a batch", response);
// not clear what we should do here - we expect the consumer of this to
// abort by throwing in the postCallback below.
return this.postCallback(response, !finalBatchPost);
}
if (finalBatchPost) {
this.log.trace("Committed batch", this.batchID);
this.batchID = undefined; // we are now in "first post for the batch" state.
this.lastModified = response.headers["x-last-modified"];
return this.postCallback(response, false);
}
if (response.status != 202) {
if (this.batchID) {
throw new Error("Server responded non-202 success code while a batch was in progress");
}
this.batchID = null; // no batch semantics are in place.
this.lastModified = response.headers["x-last-modified"];
return this.postCallback(response, false);
}
// this response is saying the server has batch semantics - we should
// always have a batch ID in the response.
let responseBatchID = response.obj.batch;
this.log.trace("Server responsed 202 with batch", responseBatchID);
if (!responseBatchID) {
this.log.error("Invalid server response: 202 without a batch ID", response);
throw new Error("Invalid server response: 202 without a batch ID");
}
if (this.batchID === undefined) {
this.batchID = responseBatchID;
if (!this.lastModified) {
this.lastModified = response.headers["x-last-modified"];
if (!this.lastModified) {
throw new Error("Batch response without x-last-modified");
}
}
}
if (this.batchID != responseBatchID) {
throw new Error(`Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}`);
}
this.postCallback(response, true);
},
}

88
services/sync/modules/resource.js
View file

@ -7,13 +7,12 @@ this.EXPORTED_SYMBOLS = [
"Resource"
];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
Cu.import("resource://gre/modules/Preferences.jsm");
Cu.import("resource://gre/modules/NetUtil.jsm");
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/observers.js");
@ -74,6 +73,20 @@ AsyncResource.prototype = {
*/
authenticator: null,
// The string to use as the base User-Agent in Sync requests.
// These strings will look something like
//
// Firefox/4.0 FxSync/1.8.0.20100101.mobile
//
// or
//
// Firefox Aurora/5.0a1 FxSync/1.9.0.20110409.desktop
//
_userAgent:
Services.appinfo.name + "/" + Services.appinfo.version + // Product.
" FxSync/" + WEAVE_VERSION + "." + // Sync.
Services.appinfo.appBuildID + ".", // Build.
// Wait 5 minutes before killing a request.
ABORT_TIMEOUT: 300000,
@ -121,9 +134,7 @@ AsyncResource.prototype = {
//
// Get and set the data encapulated in the resource.
_data: null,
get data() {
return this._data;
},
get data() this._data,
set data(value) {
this._data = value;
},
@ -135,9 +146,16 @@ AsyncResource.prototype = {
// to obtain a request channel.
//
_createRequest: function Res__createRequest(method) {
let channel = NetUtil.newChannel({uri: this.spec, loadUsingSystemPrincipal: true})
.QueryInterface(Ci.nsIRequest)
.QueryInterface(Ci.nsIHttpChannel);
let channel = Services.io.newChannel2(this.spec,
null,
null,
null, // aLoadingNode
Services.scriptSecurityManager.getSystemPrincipal(),
null, // aTriggeringPrincipal
Ci.nsILoadInfo.SEC_NORMAL,
Ci.nsIContentPolicy.TYPE_OTHER)
.QueryInterface(Ci.nsIRequest)
.QueryInterface(Ci.nsIHttpChannel);
channel.loadFlags |= DEFAULT_LOAD_FLAGS;
@ -147,7 +165,8 @@ AsyncResource.prototype = {
// Compose a UA string fragment from the various available identifiers.
if (Svc.Prefs.get("sendVersionInfo", true)) {
channel.setRequestHeader("user-agent", Utils.userAgent, false);
let ua = this._userAgent + Svc.Prefs.get("client.type", "desktop");
channel.setRequestHeader("user-agent", ua, false);
}
let headers = this.headers;
@ -155,7 +174,7 @@ AsyncResource.prototype = {
if (this.authenticator) {
let result = this.authenticator(this, method);
if (result && result.headers) {
for (let [k, v] of Object.entries(result.headers)) {
for (let [k, v] in Iterator(result.headers)) {
headers[k.toLowerCase()] = v;
}
}
@ -163,7 +182,7 @@ AsyncResource.prototype = {
this._log.debug("No authenticator found.");
}
for (let [key, value] of Object.entries(headers)) {
for (let [key, value] in Iterator(headers)) {
if (key == 'authorization')
this._log.trace("HTTP Header " + key + ": ***** (suppressed)");
else
@ -209,10 +228,10 @@ AsyncResource.prototype = {
this._log, this.ABORT_TIMEOUT);
channel.requestMethod = action;
try {
channel.asyncOpen2(listener);
channel.asyncOpen(listener, null);
} catch (ex) {
// asyncOpen2 can throw in a bunch of cases -- e.g., a forbidden port.
this._log.warn("Caught an error in asyncOpen2", ex);
// asyncOpen can throw in a bunch of cases -- e.g., a forbidden port.
this._log.warn("Caught an error in asyncOpen: " + CommonUtils.exceptionStr(ex));
CommonUtils.nextTick(callback.bind(this, ex));
}
},
@ -259,7 +278,9 @@ AsyncResource.prototype = {
} catch(ex) {
// Got a response, but an exception occurred during processing.
// This shouldn't occur.
this._log.warn("Caught unexpected exception in _oncomplete", ex);
this._log.warn("Caught unexpected exception " + CommonUtils.exceptionStr(ex) +
" in _onComplete.");
this._log.debug(CommonUtils.stackTrace(ex));
}
// Process headers. They can be empty, or the call can otherwise fail, so
@ -297,18 +318,16 @@ AsyncResource.prototype = {
contentLength + ".");
}
} catch (ex) {
this._log.debug("Caught exception visiting headers in _onComplete", ex);
this._log.debug("Caught exception " + CommonUtils.exceptionStr(ex) +
" visiting headers in _onComplete.");
this._log.debug(CommonUtils.stackTrace(ex));
}
let ret = new String(data);
ret.url = channel.URI.spec;
ret.status = status;
ret.success = success;
ret.headers = headers;
if (!success) {
this._log.warn(`${action} request to ${ret.url} failed with status ${status}`);
}
// Make a lazy getter to convert the json response into an object.
// Note that this can cause a parse error to be thrown far away from the
// actual fetch, so be warned!
@ -316,7 +335,7 @@ AsyncResource.prototype = {
try {
return JSON.parse(ret);
} catch (ex) {
this._log.warn("Got exception parsing response body", ex);
this._log.warn("Got exception parsing response body: \"" + CommonUtils.exceptionStr(ex));
// Stringify to avoid possibly printing non-printable characters.
this._log.debug("Parse fail: Response body starts: \"" +
JSON.stringify((ret + "").slice(0, 100)) +
@ -384,12 +403,7 @@ Resource.prototype = {
try {
this._doRequest(action, data, callback);
return Async.waitForSyncCallback(cb);
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("${action} request to ${url} failed: ${ex}",
{ action, url: this.uri.spec, ex });
} catch(ex) {
// Combine the channel stack with this request stack. Need to create
// a new error object for that.
let error = Error(ex.message);
@ -527,7 +541,7 @@ ChannelListener.prototype = {
siStream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance(Ci.nsIScriptableInputStream);
siStream.init(stream);
} catch (ex) {
this._log.warn("Exception creating nsIScriptableInputStream", ex);
this._log.warn("Exception creating nsIScriptableInputStream." + CommonUtils.exceptionStr(ex));
this._log.debug("Parameters: " + req.URI.spec + ", " + stream + ", " + off + ", " + count);
// Cannot proceed, so rethrow and allow the channel to cancel itself.
throw ex;
@ -543,11 +557,9 @@ ChannelListener.prototype = {
try {
this._onProgress();
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("Got exception calling onProgress handler during fetch of "
+ req.URI.spec, ex);
+ req.URI.spec);
this._log.debug(CommonUtils.exceptionStr(ex));
this._log.trace("Rethrowing; expect a failure code from the HTTP channel.");
throw ex;
}
@ -562,7 +574,7 @@ ChannelListener.prototype = {
try {
CommonUtils.namedTimer(this.abortRequest, this._timeout, this, "abortTimer");
} catch (ex) {
this._log.warn("Got exception extending abort timer", ex);
this._log.warn("Got exception extending abort timer: " + CommonUtils.exceptionStr(ex));
}
},
@ -656,14 +668,14 @@ ChannelNotificationListener.prototype = {
}
}
} catch (ex) {
this._log.error("Error copying headers", ex);
this._log.error("Error copying headers: " + CommonUtils.exceptionStr(ex));
}
// We let all redirects proceed.
try {
callback.onRedirectVerifyCallback(Cr.NS_OK);
} catch (ex) {
this._log.error("onRedirectVerifyCallback threw!", ex);
this._log.error("onRedirectVerifyCallback threw!" + CommonUtils.exceptionStr(ex));
}
}
};

20
services/sync/modules/rest.js
View file

@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");
@ -27,6 +27,21 @@ SyncStorageRequest.prototype = {
_logName: "Sync.StorageRequest",
/**
* The string to use as the base User-Agent in Sync requests.
* These strings will look something like
*
* Firefox/4.0 FxSync/1.8.0.20100101.mobile
*
* or
*
* Firefox Aurora/5.0a1 FxSync/1.9.0.20110409.desktop
*/
userAgent:
Services.appinfo.name + "/" + Services.appinfo.version + // Product.
" FxSync/" + WEAVE_VERSION + "." + // Sync.
Services.appinfo.appBuildID + ".", // Build.
/**
* Wait 5 minutes before killing a request.
*/
@ -35,7 +50,8 @@ SyncStorageRequest.prototype = {
dispatch: function dispatch(method, data, onComplete, onProgress) {
// Compose a UA string fragment from the various available identifiers.
if (Svc.Prefs.get("sendVersionInfo", true)) {
this.setHeader("user-agent", Utils.userAgent);
let ua = this.userAgent + Svc.Prefs.get("client.type", "desktop");
this.setHeader("user-agent", ua);
}
if (this.authenticator) {

283
services/sync/modules/service.js
View file

@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ["Service"];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
// How long before refreshing the cluster
const CLUSTER_BACKOFF = 5 * 60 * 1000; // 5 minutes
@ -21,6 +21,7 @@ const KEYS_WBO = "keys";
Cu.import("resource://gre/modules/Preferences.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/utils.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/engines/clients.js");
@ -32,7 +33,6 @@ Cu.import("resource://services-sync/rest.js");
Cu.import("resource://services-sync/stages/enginesync.js");
Cu.import("resource://services-sync/stages/declined.js");
Cu.import("resource://services-sync/status.js");
Cu.import("resource://services-sync/telemetry.js");
Cu.import("resource://services-sync/userapi.js");
Cu.import("resource://services-sync/util.js");
@ -51,6 +51,15 @@ const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
INFO_COLLECTION_COUNTS,
INFO_QUOTA];
// A structure mapping a (boolean) telemetry probe name to a preference name.
// The probe will record true if the pref is modified, false otherwise.
const TELEMETRY_CUSTOM_SERVER_PREFS = {
WEAVE_CUSTOM_LEGACY_SERVER_CONFIGURATION: "services.sync.serverURL",
WEAVE_CUSTOM_FXA_SERVER_CONFIGURATION: "identity.fxaccounts.auth.uri",
WEAVE_CUSTOM_TOKEN_SERVER_CONFIGURATION: "services.sync.tokenServerURI",
};
function Sync11Service() {
this._notify = Utils.notify("weave:service:");
}
@ -64,13 +73,8 @@ Sync11Service.prototype = {
storageURL: null,
metaURL: null,
cryptoKeyURL: null,
// The cluster URL comes via the ClusterManager object, which in the FxA
// world is ebbedded in the token returned from the token server.
_clusterURL: null,
get serverURL() {
return Svc.Prefs.get("serverURL");
},
get serverURL() Svc.Prefs.get("serverURL"),
set serverURL(value) {
if (!value.endsWith("/")) {
value += "/";
@ -80,20 +84,14 @@ Sync11Service.prototype = {
if (value == this.serverURL)
return;
// A new server most likely uses a different cluster, so clear that
Svc.Prefs.set("serverURL", value);
// A new server most likely uses a different cluster, so clear that.
this._clusterURL = null;
Svc.Prefs.reset("clusterURL");
},
get clusterURL() {
return this._clusterURL || "";
},
get clusterURL() Svc.Prefs.get("clusterURL", ""),
set clusterURL(value) {
if (value != null && typeof value != "string") {
throw new Error("cluster must be a string, got " + (typeof value));
}
this._clusterURL = value;
Svc.Prefs.set("clusterURL", value);
this._updateCachedURLs();
},
@ -171,16 +169,8 @@ Sync11Service.prototype = {
_updateCachedURLs: function _updateCachedURLs() {
// Nothing to cache yet if we don't have the building blocks
if (!this.clusterURL || !this.identity.username) {
// Also reset all other URLs used by Sync to ensure we aren't accidentally
// using one cached earlier - if there's no cluster URL any cached ones
// are invalid.
this.infoURL = undefined;
this.storageURL = undefined;
this.metaURL = undefined;
this.cryptoKeysURL = undefined;
if (!this.clusterURL || !this.identity.username)
return;
}
this._log.debug("Caching URLs under storage user base: " + this.userBaseURL);
@ -315,6 +305,21 @@ Sync11Service.prototype = {
return false;
},
// The global "enabled" state comes from prefs, and will be set to false
// whenever the UI that exposes what to sync finds all Sync engines disabled.
get enabled() {
return Svc.Prefs.get("enabled");
},
set enabled(val) {
// There's no real reason to impose this other than to catch someone doing
// something we don't expect with bad consequences - all setting of this
// pref are in the UI code and external to this module.
if (val) {
throw new Error("Only disabling via this setter is supported");
}
Svc.Prefs.set("enabled", val);
},
/**
* Prepare to initialize the rest of Weave after waiting a little bit
*/
@ -344,8 +349,6 @@ Sync11Service.prototype = {
this._clusterManager = this.identity.createClusterManager(this);
this.recordManager = new RecordManager(this);
this.enabled = true;
this._registerEngines();
let ua = Cc["@mozilla.org/network/protocol;1?name=http"].
@ -359,7 +362,6 @@ Sync11Service.prototype = {
}
Svc.Obs.add("weave:service:setup-complete", this);
Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon
Svc.Prefs.observe("engine.", this);
this.scheduler = new SyncScheduler(this);
@ -375,6 +377,12 @@ Sync11Service.prototype = {
Svc.Obs.notify("weave:engine:start-tracking");
}
// Telemetry probes to indicate if the user is using custom servers.
for (let [probeName, prefName] of Iterator(TELEMETRY_CUSTOM_SERVER_PREFS)) {
let isCustomized = Services.prefs.prefHasUserValue(prefName);
Services.telemetry.getHistogramById(probeName).add(isCustomized);
}
// Send an event now that Weave service is ready. We don't do this
// synchronously so that observers can import this module before
// registering an observer.
@ -424,7 +432,7 @@ Sync11Service.prototype = {
// Map each old pref to the current pref branch
let oldPref = new Preferences(oldPrefBranch);
for (let pref of oldPrefNames)
for each (let pref in oldPrefNames)
Svc.Prefs.set(pref, oldPref.get(pref));
// Remove all the old prefs and remember that we've migrated
@ -472,7 +480,8 @@ Sync11Service.prototype = {
this.engineManager.register(ns[engineName]);
} catch (ex) {
this._log.warn("Could not register engine " + name, ex);
this._log.warn("Could not register engine " + name + ": " +
CommonUtils.exceptionStr(ex));
}
}
@ -486,13 +495,6 @@ Sync11Service.prototype = {
observe: function observe(subject, topic, data) {
switch (topic) {
// Ideally this observer should be in the SyncScheduler, but it would require
// some work to know about the sync specific engines. We should move this there once it does.
case "sync:collection_changed":
if (data.includes("clients")) {
this.sync([]); // [] = clients collection only
}
break;
case "weave:service:setup-complete":
let status = this._checkSetup();
if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED)
@ -557,8 +559,7 @@ Sync11Service.prototype = {
// Always check for errors; this is also where we look for X-Weave-Alert.
this.errorHandler.checkServerError(info);
if (!info.success) {
this._log.error("Aborting sync: failed to get collections.")
throw info;
throw "Aborting sync: failed to get collections.";
}
return info;
},
@ -675,13 +676,21 @@ Sync11Service.prototype = {
} catch (ex) {
// This means no keys are present, or there's a network error.
this._log.debug("Failed to fetch and verify keys", ex);
this._log.debug("Failed to fetch and verify keys: "
+ Utils.exceptionStr(ex));
this.errorHandler.checkServerError(ex);
return false;
}
},
verifyLogin: function verifyLogin(allow40XRecovery = true) {
// If the identity isn't ready it might not know the username...
if (!this.identity.readyToAuthenticate) {
this._log.info("Not ready to authenticate in verifyLogin.");
this.status.login = LOGIN_FAILED_NOT_READY;
return false;
}
if (!this.identity.username) {
this._log.warn("No username in verifyLogin.");
this.status.login = LOGIN_FAILED_NO_USERNAME;
@ -753,12 +762,8 @@ Sync11Service.prototype = {
return this.verifyLogin(false);
}
// We must have the right cluster, but the server doesn't expect us.
// The implications of this depend on the identity being used - for
// the legacy identity, it's an authoritatively "incorrect password",
// (ie, LOGIN_FAILED_LOGIN_REJECTED) but for FxA it probably means
// "transient error fetching auth token".
this.status.login = this.identity.loginStatusFromVerification404();
// We must have the right cluster, but the server doesn't expect us
this.status.login = LOGIN_FAILED_LOGIN_REJECTED;
return false;
default:
@ -769,7 +774,7 @@ Sync11Service.prototype = {
}
} catch (ex) {
// Must have failed on some network issue
this._log.debug("verifyLogin failed", ex);
this._log.debug("verifyLogin failed: " + Utils.exceptionStr(ex));
this.status.login = LOGIN_FAILED_NETWORK_ERROR;
this.errorHandler.checkServerError(ex);
return false;
@ -842,7 +847,8 @@ Sync11Service.prototype = {
try {
cb.wait();
} catch (ex) {
this._log.debug("Password change failed", ex);
this._log.debug("Password change failed: " +
CommonUtils.exceptionStr(ex));
return false;
}
@ -884,11 +890,12 @@ Sync11Service.prototype = {
// Deletion doesn't make sense if we aren't set up yet!
if (this.clusterURL != "") {
// Clear client-specific data from the server, including disabled engines.
for (let engine of [this.clientsEngine].concat(this.engineManager.getAll())) {
for each (let engine in [this.clientsEngine].concat(this.engineManager.getAll())) {
try {
engine.removeClientData();
} catch(ex) {
this._log.warn(`Deleting client data for ${engine.name} failed`, ex);
this._log.warn("Deleting client data for " + engine.name + " failed:"
+ Utils.exceptionStr(ex));
}
}
this._log.debug("Finished deleting client data.");
@ -914,7 +921,6 @@ Sync11Service.prototype = {
this._ignorePrefObserver = true;
Svc.Prefs.resetBranch("");
this._ignorePrefObserver = false;
this.clusterURL = null;
Svc.Prefs.set("lastversion", WEAVE_VERSION);
@ -931,22 +937,25 @@ Sync11Service.prototype = {
return;
}
try {
this.identity.finalize();
// an observer so the FxA migration code can take some action before
// the new identity is created.
Svc.Obs.notify("weave:service:start-over:init-identity");
this.identity.username = "";
this.status.__authManager = null;
this.identity = Status._authManager;
this._clusterManager = this.identity.createClusterManager(this);
Svc.Obs.notify("weave:service:start-over:finish");
} catch (err) {
this._log.error("startOver failed to re-initialize the identity manager: " + err);
// Still send the observer notification so the current state is
// reflected in the UI.
Svc.Obs.notify("weave:service:start-over:finish");
}
this.identity.finalize().then(
() => {
// an observer so the FxA migration code can take some action before
// the new identity is created.
Svc.Obs.notify("weave:service:start-over:init-identity");
this.identity.username = "";
this.status.__authManager = null;
this.identity = Status._authManager;
this._clusterManager = this.identity.createClusterManager(this);
Svc.Obs.notify("weave:service:start-over:finish");
}
).then(null,
err => {
this._log.error("startOver failed to re-initialize the identity manager: " + err);
// Still send the observer notification so the current state is
// reflected in the UI.
Svc.Obs.notify("weave:service:start-over:finish");
}
);
},
persistLogin: function persistLogin() {
@ -981,12 +990,8 @@ Sync11Service.prototype = {
}
// Ask the identity manager to explicitly login now.
this._log.info("Logging in the user.");
let cb = Async.makeSpinningCallback();
this.identity.ensureLoggedIn().then(
() => cb(null),
err => cb(err || "ensureLoggedIn failed")
);
this.identity.ensureLoggedIn().then(cb, cb);
// Just let any errors bubble up - they've more context than we do!
cb.wait();
@ -997,9 +1002,9 @@ Sync11Service.prototype = {
&& (username || password || passphrase)) {
Svc.Obs.notify("weave:service:setup-complete");
}
this._log.info("Logging in the user.");
this._updateCachedURLs();
this._log.info("User logged in successfully - verifying login.");
if (!this.verifyLogin()) {
// verifyLogin sets the failure states here.
throw "Login failed: " + this.status.login;
@ -1064,49 +1069,11 @@ Sync11Service.prototype = {
}
},
// Note: returns false if we failed for a reason other than the server not yet
// supporting the api.
_fetchServerConfiguration() {
if (Svc.Prefs.get("APILevel") >= 2) {
// This is similar to _fetchInfo, but with different error handling.
// Only supported by later sync implementations.
let infoURL = this.userBaseURL + "info/configuration";
this._log.debug("Fetching server configuration", infoURL);
let configResponse;
try {
configResponse = this.resource(infoURL).get();
} catch (ex) {
// This is probably a network or similar error.
this._log.warn("Failed to fetch info/configuration", ex);
this.errorHandler.checkServerError(ex);
return false;
}
if (configResponse.status == 404) {
// This server doesn't support the URL yet - that's OK.
this._log.debug("info/configuration returned 404 - using default upload semantics");
} else if (configResponse.status != 200) {
this._log.warn(`info/configuration returned ${configResponse.status} - using default configuration`);
this.errorHandler.checkServerError(configResponse);
return false;
} else {
this.serverConfiguration = configResponse.obj;
}
this._log.trace("info/configuration for this server", this.serverConfiguration);
}
return true;
},
// Stuff we need to do after login, before we can really do
// anything (e.g. key setup).
_remoteSetup: function _remoteSetup(infoResponse) {
let reset = false;
if (!this._fetchServerConfiguration()) {
return false;
}
this._log.debug("Fetching global metadata record");
let meta = this.recordManager.get(this.metaURL);
@ -1133,7 +1100,7 @@ Sync11Service.prototype = {
return false;
}
if (this.recordManager.response.status == 404) {
if (!this.recordManager.response.success || !newMeta) {
this._log.debug("No meta/global record on the server. Creating one.");
newMeta = new WBORecord("meta", "global");
newMeta.payload.syncID = this.syncID;
@ -1143,16 +1110,10 @@ Sync11Service.prototype = {
newMeta.isNew = true;
this.recordManager.set(this.metaURL, newMeta);
let uploadRes = newMeta.upload(this.resource(this.metaURL));
if (!uploadRes.success) {
if (!newMeta.upload(this.resource(this.metaURL)).success) {
this._log.warn("Unable to upload new meta/global. Failing remote setup.");
this.errorHandler.checkServerError(uploadRes);
return false;
}
} else if (!newMeta) {
this._log.warn("Unable to get meta/global. Failing remote setup.");
this.errorHandler.checkServerError(this.recordManager.response);
return false;
} else {
// If newMeta, then it stands to reason that meta != null.
newMeta.isNew = meta.isNew;
@ -1293,9 +1254,13 @@ Sync11Service.prototype = {
return reason;
},
sync: function sync(engineNamesToSync) {
let dateStr = Utils.formatTimestamp(new Date());
this._log.debug("User-Agent: " + Utils.userAgent);
sync: function sync() {
if (!this.enabled) {
this._log.debug("Not syncing as Sync is disabled.");
return;
}
let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
this._log.debug("User-Agent: " + SyncStorageRequest.prototype.userAgent);
this._log.info("Starting sync at " + dateStr);
this._catch(function () {
// Make sure we're logged in.
@ -1309,14 +1274,14 @@ Sync11Service.prototype = {
else {
this._log.trace("In sync: no need to login.");
}
return this._lockedSync(engineNamesToSync);
return this._lockedSync.apply(this, arguments);
})();
},
/**
* Sync up engines with the server.
*/
_lockedSync: function _lockedSync(engineNamesToSync) {
_lockedSync: function _lockedSync() {
return this._lock("service.js: sync",
this._notify("sync", "", function onNotify() {
@ -1327,7 +1292,7 @@ Sync11Service.prototype = {
let cb = Async.makeSpinningCallback();
synchronizer.onComplete = cb;
synchronizer.sync(engineNamesToSync);
synchronizer.sync();
// wait() throws if the first argument is truthy, which is exactly what
// we want.
let result = cb.wait();
@ -1338,31 +1303,27 @@ Sync11Service.prototype = {
// We successfully synchronized.
// Check if the identity wants to pre-fetch a migration sentinel from
// the server.
// Only supported by Sync server API level 2+
// If we have no clusterURL, we are probably doing a node reassignment
// so don't attempt to get it in that case.
if (Svc.Prefs.get("APILevel") >= 2 && this.clusterURL) {
this.identity.prefetchMigrationSentinel(this);
//if (this.clusterURL) {
// this.identity.prefetchMigrationSentinel(this);
//}
// Now let's update our declined engines.
let meta = this.recordManager.get(this.metaURL);
if (!meta) {
this._log.warn("No meta/global; can't update declined state.");
return;
}
// Now let's update our declined engines (but only if we have a metaURL;
// if Sync failed due to no node we will not have one)
if (this.metaURL) {
let meta = this.recordManager.get(this.metaURL);
if (!meta) {
this._log.warn("No meta/global; can't update declined state.");
return;
}
let declinedEngines = new DeclinedEngines(this);
let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
if (!didChange) {
this._log.info("No change to declined engines. Not reuploading meta/global.");
return;
}
this.uploadMetaGlobal(meta);
let declinedEngines = new DeclinedEngines(this);
let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
if (!didChange) {
this._log.info("No change to declined engines. Not reuploading meta/global.");
return;
}
this.uploadMetaGlobal(meta);
}))();
},
@ -1536,7 +1497,7 @@ Sync11Service.prototype = {
// Wipe everything we know about except meta because we just uploaded it
let engines = [this.clientsEngine].concat(this.engineManager.getAll());
let collections = engines.map(engine => engine.name);
let collections = [engine.name for each (engine in engines)];
// TODO: there's a bug here. We should be calling resetClient, no?
// Generate, upload, and download new keys. Do this last so we don't wipe
@ -1555,7 +1516,6 @@ Sync11Service.prototype = {
*/
wipeServer: function wipeServer(collections) {
let response;
let histogram = Services.telemetry.getHistogramById("WEAVE_WIPE_SERVER_SUCCEEDED");
if (!collections) {
// Strip the trailing slash.
let res = this.resource(this.storageURL.slice(0, -1));
@ -1563,17 +1523,14 @@ Sync11Service.prototype = {
try {
response = res.delete();
} catch (ex) {
this._log.debug("Failed to wipe server", ex);
histogram.add(false);
this._log.debug("Failed to wipe server: " + CommonUtils.exceptionStr(ex));
throw ex;
}
if (response.status != 200 && response.status != 404) {
this._log.debug("Aborting wipeServer. Server responded with " +
response.status + " response for " + this.storageURL);
histogram.add(false);
throw response;
}
histogram.add(true);
return response.headers["x-weave-timestamp"];
}
@ -1583,15 +1540,14 @@ Sync11Service.prototype = {
try {
response = this.resource(url).delete();
} catch (ex) {
this._log.debug("Failed to wipe '" + name + "' collection", ex);
histogram.add(false);
this._log.debug("Failed to wipe '" + name + "' collection: " +
Utils.exceptionStr(ex));
throw ex;
}
if (response.status != 200 && response.status != 404) {
this._log.debug("Aborting wipeServer. Server responded with " +
response.status + " response for " + url);
histogram.add(false);
throw response;
}
@ -1599,7 +1555,7 @@ Sync11Service.prototype = {
timestamp = response.headers["x-weave-timestamp"];
}
}
histogram.add(true);
return timestamp;
},
@ -1623,7 +1579,7 @@ Sync11Service.prototype = {
}
// Fully wipe each engine if it's able to decrypt data
for (let engine of engines) {
for each (let engine in engines) {
if (engine.canDecrypt()) {
engine.wipeClient();
}
@ -1701,7 +1657,7 @@ Sync11Service.prototype = {
}
// Have each engine drop any temporary meta data
for (let engine of engines) {
for each (let engine in engines) {
engine.resetClient();
}
})();
@ -1731,7 +1687,8 @@ Sync11Service.prototype = {
return this.getStorageRequest(url).get(function onComplete(error) {
// Note: 'this' is the request.
if (error) {
this._log.debug("Failed to retrieve '" + info_type + "'", error);
this._log.debug("Failed to retrieve '" + info_type + "': " +
Utils.exceptionStr(error));
return callback(error);
}
if (this.response.status != 200) {

6
services/sync/modules/stages/cluster.js
View file

@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["ClusterManager"];
var {utils: Cu} = Components;
const {utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/constants.js");
@ -80,9 +80,6 @@ ClusterManager.prototype = {
return false;
}
// Convert from the funky "String object with additional properties" that
// resource.js returns to a plain-old string.
cluster = cluster.toString();
// Don't update stuff if we already have the right cluster
if (cluster == this.service.clusterURL) {
return false;
@ -90,6 +87,7 @@ ClusterManager.prototype = {
this._log.debug("Setting cluster to " + cluster);
this.service.clusterURL = cluster;
Svc.Prefs.set("lastClusterUpdate", Date.now().toString());
return true;
},

6
services/sync/modules/stages/declined.js
View file

@ -11,7 +11,7 @@
this.EXPORTED_SYMBOLS = ["DeclinedEngines"];
var {utils: Cu} = Components;
const {utils: Cu} = Components;
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
@ -29,8 +29,8 @@ this.DeclinedEngines = function (service) {
}
this.DeclinedEngines.prototype = {
updateDeclined: function (meta, engineManager=this.service.engineManager) {
let enabled = new Set(engineManager.getEnabled().map(e => e.name));
let known = new Set(engineManager.getAll().map(e => e.name));
let enabled = new Set([e.name for each (e in engineManager.getEnabled())]);
let known = new Set([e.name for each (e in engineManager.getAll())]);
let remoteDeclined = new Set(meta.payload.declined || []);
let localDeclined = new Set(engineManager.getDeclined());

147
services/sync/modules/stages/enginesync.js
View file

@ -8,16 +8,13 @@
this.EXPORTED_SYMBOLS = ["EngineSynchronizer"];
var {utils: Cu} = Components;
const {utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/policies.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/observers.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Task.jsm");
/**
* Perform synchronization of engines.
@ -34,7 +31,7 @@ this.EngineSynchronizer = function EngineSynchronizer(service) {
}
EngineSynchronizer.prototype = {
sync: function sync(engineNamesToSync) {
sync: function sync() {
if (!this.onComplete) {
throw new Error("onComplete handler not installed.");
}
@ -99,9 +96,6 @@ EngineSynchronizer.prototype = {
return;
}
// We only honor the "hint" of what engines to Sync if this isn't
// a first sync.
let allowEnginesHint = false;
// Wipe data in the desired direction if necessary
switch (Svc.Prefs.get("firstSync")) {
case "resetClient":
@ -113,9 +107,6 @@ EngineSynchronizer.prototype = {
case "wipeRemote":
this.service.wipeRemote(engineManager.enabledEngineNames);
break;
default:
allowEnginesHint = true;
break;
}
if (this.service.clientsEngine.localCommands) {
@ -145,31 +136,20 @@ EngineSynchronizer.prototype = {
try {
this._updateEnabledEngines();
} catch (ex) {
this._log.debug("Updating enabled engines failed", ex);
this._log.debug("Updating enabled engines failed: " +
Utils.exceptionStr(ex));
this.service.errorHandler.checkServerError(ex);
this.onComplete(ex);
return;
}
// If the engines to sync has been specified, we sync in the order specified.
let enginesToSync;
if (allowEnginesHint && engineNamesToSync) {
this._log.info("Syncing specified engines", engineNamesToSync);
enginesToSync = engineManager.get(engineNamesToSync).filter(e => e.enabled);
} else {
this._log.info("Syncing all enabled engines.");
enginesToSync = engineManager.getEnabled();
}
try {
// We don't bother validating engines that failed to sync.
let enginesToValidate = [];
for (let engine of enginesToSync) {
for (let engine of engineManager.getEnabled()) {
// If there's any problems with syncing the engine, report the failure
if (!(this._syncEngine(engine)) || this.service.status.enforceBackoff) {
this._log.info("Aborting sync for failure in " + engine.name);
break;
}
enginesToValidate.push(engine);
}
// If _syncEngine fails for a 401, we might not have a cluster URL here.
@ -195,8 +175,6 @@ EngineSynchronizer.prototype = {
}
}
Async.promiseSpinningly(this._tryValidateEngines(enginesToValidate));
// If there were no sync engine failures
if (this.service.status.service != SYNC_FAILED_PARTIAL) {
Svc.Prefs.set("lastSync", new Date().toString());
@ -206,7 +184,7 @@ EngineSynchronizer.prototype = {
Svc.Prefs.reset("firstSync");
let syncTime = ((Date.now() - startTime) / 1000).toFixed(2);
let dateStr = Utils.formatTimestamp(new Date());
let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
this._log.info("Sync completed at " + dateStr
+ " after " + syncTime + " secs.");
}
@ -214,106 +192,6 @@ EngineSynchronizer.prototype = {
this.onComplete(null);
},
_tryValidateEngines: Task.async(function* (recentlySyncedEngines) {
if (!Services.telemetry.canRecordBase || !Svc.Prefs.get("validation.enabled", false)) {
this._log.info("Skipping validation: validation or telemetry reporting is disabled");
return;
}
let lastValidation = Svc.Prefs.get("validation.lastTime", 0);
let validationInterval = Svc.Prefs.get("validation.interval");
let nowSeconds = Math.floor(Date.now() / 1000);
if (nowSeconds - lastValidation < validationInterval) {
this._log.info("Skipping validation: too recent since last validation attempt");
return;
}
// Update the time now, even if we may return false still. We don't want to
// check the rest of these more frequently than once a day.
Svc.Prefs.set("validation.lastTime", nowSeconds);
// Validation only occurs a certain percentage of the time.
let validationProbability = Svc.Prefs.get("validation.percentageChance", 0) / 100.0;
if (validationProbability < Math.random()) {
this._log.info("Skipping validation: Probability threshold not met");
return;
}
let maxRecords = Svc.Prefs.get("validation.maxRecords");
if (!maxRecords) {
// Don't bother asking the server for the counts if we know validation
// won't happen anyway.
return;
}
// maxRecords of -1 means "any number", so we can skip asking the server.
// Used for tests.
let info;
if (maxRecords < 0) {
info = {};
for (let e of recentlySyncedEngines) {
info[e.name] = 1; // needs to be < maxRecords
}
maxRecords = 2;
} else {
let collectionCountsURL = this.service.userBaseURL + "info/collection_counts";
try {
let infoResp = this.service._fetchInfo(collectionCountsURL);
if (!infoResp.success) {
this._log.error("Can't run validation: request to info/collection_counts responded with "
+ resp.status);
return;
}
info = infoResp.obj; // might throw because obj is a getter which parses json.
} catch (e) {
// Not running validation is totally fine, so we just write an error log and return.
this._log.error("Can't run validation: Caught error when fetching counts", e);
return;
}
}
if (!info) {
return;
}
let engineLookup = new Map(recentlySyncedEngines.map(e => [e.name, e]));
let toRun = [];
for (let [engineName, recordCount] of Object.entries(info)) {
let engine = engineLookup.get(engineName);
if (recordCount > maxRecords || !engine) {
this._log.debug(`Skipping validation for ${engineName} because it's not an engine or ` +
`the number of records (${recordCount}) is greater than the maximum allowed (${maxRecords}).`);
continue;
}
let validator = engine.getValidator();
if (!validator) {
continue;
}
// Put this in an array so that we know how many we're going to do, so we
// don't tell users we're going to run some validators when we aren't.
toRun.push({ engine, validator });
}
if (!toRun.length) {
return;
}
Services.console.logStringMessage(
"Sync is about to run a consistency check. This may be slow, and " +
"can be controlled using the pref \"services.sync.validation.enabled\".\n" +
"If you encounter any problems because of this, please file a bug.");
for (let { validator, engine } of toRun) {
try {
let result = yield validator.validate(engine);
Observers.notify("weave:engine:validate:finish", result, engine.name);
} catch (e) {
this._log.error(`Failed to run validation on ${engine.name}!`, e);
Observers.notify("weave:engine:validate:error", e, engine.name)
// Keep validating -- there's no reason to think that a failure for one
// validator would mean the others will fail.
}
}
}),
// Returns true if sync should proceed.
// false / no return value means sync should be aborted.
_syncEngine: function _syncEngine(engine) {
@ -346,15 +224,8 @@ EngineSynchronizer.prototype = {
// If we're the only client, and no engines are marked as enabled,
// thumb our noses at the server data: it can't be right.
// Belt-and-suspenders approach to Bug 615926.
let hasEnabledEngines = false;
for (let e in meta.payload.engines) {
if (e != "clients") {
hasEnabledEngines = true;
break;
}
}
if ((numClients <= 1) && !hasEnabledEngines) {
if ((numClients <= 1) &&
([e for (e in meta.payload.engines) if (e != "clients")].length == 0)) {
this._log.info("One client and no enabled engines: not touching local engine status.");
return;
}
@ -418,7 +289,7 @@ EngineSynchronizer.prototype = {
}
// Any remaining engines were either enabled locally or disabled remotely.
for (let engineName of enabled) {
for each (let engineName in enabled) {
let engine = engineManager.get(engineName);
if (Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) {
this._log.trace("The " + engineName + " engine was enabled locally.");

13
services/sync/modules/status.js
View file

@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ["Status"];
var Cc = Components.classes;
var Ci = Components.interfaces;
var Cr = Components.results;
var Cu = Components.utils;
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
@ -30,7 +30,10 @@ this.Status = {
.wrappedJSObject;
let idClass = service.fxAccountsEnabled ? BrowserIDManager : IdentityManager;
this.__authManager = new idClass();
this.__authManager.initialize();
// .initialize returns a promise, so we need to spin until it resolves.
let cb = Async.makeSpinningCallback();
this.__authManager.initialize().then(cb, cb);
cb.wait();
return this.__authManager;
},

578
services/sync/modules/telemetry.js
View file

@ -1,578 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
this.EXPORTED_SYMBOLS = ["SyncTelemetry"];
Cu.import("resource://services-sync/browserid_identity.js");
Cu.import("resource://services-sync/main.js");
Cu.import("resource://services-sync/status.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/observers.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://gre/modules/TelemetryController.jsm");
Cu.import("resource://gre/modules/FxAccounts.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/osfile.jsm", this);
let constants = {};
Cu.import("resource://services-sync/constants.js", constants);
var fxAccountsCommon = {};
Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
XPCOMUtils.defineLazyServiceGetter(this, "Telemetry",
"@mozilla.org/base/telemetry;1",
"nsITelemetry");
const log = Log.repository.getLogger("Sync.Telemetry");
const TOPICS = [
"profile-before-change",
"weave:service:sync:start",
"weave:service:sync:finish",
"weave:service:sync:error",
"weave:engine:sync:start",
"weave:engine:sync:finish",
"weave:engine:sync:error",
"weave:engine:sync:applied",
"weave:engine:sync:uploaded",
"weave:engine:validate:finish",
"weave:engine:validate:error",
];
const PING_FORMAT_VERSION = 1;
// The set of engines we record telemetry for - any other engines are ignored.
const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
"passwords", "prefs", "tabs", "extension-storage"]);
// A regex we can use to replace the profile dir in error messages. We use a
// regexp so we can simply replace all case-insensitive occurences.
// This escaping function is from:
// https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
const reProfileDir = new RegExp(
OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
"gi");
function transformError(error, engineName) {
if (Async.isShutdownException(error)) {
return { name: "shutdownerror" };
}
if (typeof error === "string") {
if (error.startsWith("error.")) {
// This is hacky, but I can't imagine that it's not also accurate.
return { name: "othererror", error };
}
// There's a chance the profiledir is in the error string which is PII we
// want to avoid including in the ping.
error = error.replace(reProfileDir, "[profileDir]");
return { name: "unexpectederror", error };
}
if (error.failureCode) {
return { name: "othererror", error: error.failureCode };
}
if (error instanceof AuthenticationError) {
return { name: "autherror", from: error.source };
}
if (error instanceof Ci.mozIStorageError) {
return { name: "sqlerror", code: error.result };
}
let httpCode = error.status ||
(error.response && error.response.status) ||
error.code;
if (httpCode) {
return { name: "httperror", code: httpCode };
}
if (error.result) {
return { name: "nserror", code: error.result };
}
return {
name: "unexpectederror",
// as above, remove the profile dir value.
error: String(error).replace(reProfileDir, "[profileDir]")
}
}
function tryGetMonotonicTimestamp() {
try {
return Telemetry.msSinceProcessStart();
} catch (e) {
log.warn("Unable to get a monotonic timestamp!");
return -1;
}
}
function timeDeltaFrom(monotonicStartTime) {
let now = tryGetMonotonicTimestamp();
if (monotonicStartTime !== -1 && now !== -1) {
return Math.round(now - monotonicStartTime);
}
return -1;
}
class EngineRecord {
constructor(name) {
// startTime is in ms from process start, but is monotonic (unlike Date.now())
// so we need to keep both it and when.
this.startTime = tryGetMonotonicTimestamp();
this.name = name;
}
toJSON() {
let result = Object.assign({}, this);
delete result.startTime;
return result;
}
finished(error) {
let took = timeDeltaFrom(this.startTime);
if (took > 0) {
this.took = took;
}
if (error) {
this.failureReason = transformError(error, this.name);
}
}
recordApplied(counts) {
if (this.incoming) {
log.error(`Incoming records applied multiple times for engine ${this.name}!`);
return;
}
if (this.name === "clients" && !counts.failed) {
// ignore successful application of client records
// since otherwise they show up every time and are meaningless.
return;
}
let incomingData = {};
let properties = ["applied", "failed", "newFailed", "reconciled"];
// Only record non-zero properties and only record incoming at all if
// there's at least one property we care about.
for (let property of properties) {
if (counts[property]) {
incomingData[property] = counts[property];
this.incoming = incomingData;
}
}
}
recordValidation(validationResult) {
if (this.validation) {
log.error(`Multiple validations occurred for engine ${this.name}!`);
return;
}
let { problems, version, duration, recordCount } = validationResult;
let validation = {
version: version || 0,
checked: recordCount || 0,
};
if (duration > 0) {
validation.took = Math.round(duration);
}
let summarized = problems.getSummary(true).filter(({count}) => count > 0);
if (summarized.length) {
validation.problems = summarized;
}
this.validation = validation;
}
recordValidationError(e) {
if (this.validation) {
log.error(`Multiple validations occurred for engine ${this.name}!`);
return;
}
this.validation = {
failureReason: transformError(e)
};
}
recordUploaded(counts) {
if (counts.sent || counts.failed) {
if (!this.outgoing) {
this.outgoing = [];
}
this.outgoing.push({
sent: counts.sent || undefined,
failed: counts.failed || undefined,
});
}
}
}
class TelemetryRecord {
constructor(allowedEngines) {
this.allowedEngines = allowedEngines;
// Our failure reason. This property only exists in the generated ping if an
// error actually occurred.
this.failureReason = undefined;
this.uid = "";
this.when = Date.now();
this.startTime = tryGetMonotonicTimestamp();
this.took = 0; // will be set later.
// All engines that have finished (ie, does not include the "current" one)
// We omit this from the ping if it's empty.
this.engines = [];
// The engine that has started but not yet stopped.
this.currentEngine = null;
}
toJSON() {
let result = {
when: this.when,
uid: this.uid,
took: this.took,
failureReason: this.failureReason,
status: this.status,
deviceID: this.deviceID,
devices: this.devices,
};
let engines = [];
for (let engine of this.engines) {
engines.push(engine.toJSON());
}
if (engines.length > 0) {
result.engines = engines;
}
return result;
}
finished(error) {
this.took = timeDeltaFrom(this.startTime);
if (this.currentEngine != null) {
log.error("Finished called for the sync before the current engine finished");
this.currentEngine.finished(null);
this.onEngineStop(this.currentEngine.name);
}
if (error) {
this.failureReason = transformError(error);
}
// We don't bother including the "devices" field if we can't come up with a
// UID or device ID for *this* device -- If that's the case, any data we'd
// put there would be likely to be full of garbage anyway.
let includeDeviceInfo = false;
try {
this.uid = Weave.Service.identity.hashedUID();
let deviceID = Weave.Service.identity.deviceID();
if (deviceID) {
// Combine the raw device id with the metrics uid to create a stable
// unique identifier that can't be mapped back to the user's FxA
// identity without knowing the metrics HMAC key.
this.deviceID = Utils.sha256(deviceID + this.uid);
includeDeviceInfo = true;
}
} catch (e) {
this.uid = "0".repeat(32);
this.deviceID = undefined;
}
if (includeDeviceInfo) {
let remoteDevices = Weave.Service.clientsEngine.remoteClients;
this.devices = remoteDevices.map(device => {
return {
os: device.os,
version: device.version,
id: Utils.sha256(device.id + this.uid)
};
});
}
// Check for engine statuses. -- We do this now, and not in engine.finished
// to make sure any statuses that get set "late" are recorded
for (let engine of this.engines) {
let status = Status.engines[engine.name];
if (status && status !== constants.ENGINE_SUCCEEDED) {
engine.status = status;
}
}
let statusObject = {};
let serviceStatus = Status.service;
if (serviceStatus && serviceStatus !== constants.STATUS_OK) {
statusObject.service = serviceStatus;
this.status = statusObject;
}
let syncStatus = Status.sync;
if (syncStatus && syncStatus !== constants.SYNC_SUCCEEDED) {
statusObject.sync = syncStatus;
this.status = statusObject;
}
}
onEngineStart(engineName) {
if (this._shouldIgnoreEngine(engineName, false)) {
return;
}
if (this.currentEngine) {
log.error(`Being told that engine ${engineName} has started, but current engine ${
this.currentEngine.name} hasn't stopped`);
// Just discard the current engine rather than making up data for it.
}
this.currentEngine = new EngineRecord(engineName);
}
onEngineStop(engineName, error) {
// We only care if it's the current engine if we have a current engine.
if (this._shouldIgnoreEngine(engineName, !!this.currentEngine)) {
return;
}
if (!this.currentEngine) {
// It's possible for us to get an error before the start message of an engine
// (somehow), in which case we still want to record that error.
if (!error) {
return;
}
log.error(`Error triggered on ${engineName} when no current engine exists: ${error}`);
this.currentEngine = new EngineRecord(engineName);
}
this.currentEngine.finished(error);
this.engines.push(this.currentEngine);
this.currentEngine = null;
}
onEngineApplied(engineName, counts) {
if (this._shouldIgnoreEngine(engineName)) {
return;
}
this.currentEngine.recordApplied(counts);
}
onEngineValidated(engineName, validationData) {
if (this._shouldIgnoreEngine(engineName, false)) {
return;
}
let engine = this.engines.find(e => e.name === engineName);
if (!engine && this.currentEngine && engineName === this.currentEngine.name) {
engine = this.currentEngine;
}
if (engine) {
engine.recordValidation(validationData);
} else {
log.warn(`Validation event triggered for engine ${engineName}, which hasn't been synced!`);
}
}
onEngineValidateError(engineName, error) {
if (this._shouldIgnoreEngine(engineName, false)) {
return;
}
let engine = this.engines.find(e => e.name === engineName);
if (!engine && this.currentEngine && engineName === this.currentEngine.name) {
engine = this.currentEngine;
}
if (engine) {
engine.recordValidationError(error);
} else {
log.warn(`Validation failure event triggered for engine ${engineName}, which hasn't been synced!`);
}
}
onEngineUploaded(engineName, counts) {
if (this._shouldIgnoreEngine(engineName)) {
return;
}
this.currentEngine.recordUploaded(counts);
}
_shouldIgnoreEngine(engineName, shouldBeCurrent = true) {
if (!this.allowedEngines.has(engineName)) {
log.info(`Notification for engine ${engineName}, but we aren't recording telemetry for it`);
return true;
}
if (shouldBeCurrent) {
if (!this.currentEngine || engineName != this.currentEngine.name) {
log.error(`Notification for engine ${engineName} but it isn't current`);
return true;
}
}
return false;
}
}
class SyncTelemetryImpl {
constructor(allowedEngines) {
log.level = Log.Level[Svc.Prefs.get("log.logger.telemetry", "Trace")];
// This is accessible so we can enable custom engines during tests.
this.allowedEngines = allowedEngines;
this.current = null;
this.setupObservers();
this.payloads = [];
this.discarded = 0;
this.maxPayloadCount = Svc.Prefs.get("telemetry.maxPayloadCount");
this.submissionInterval = Svc.Prefs.get("telemetry.submissionInterval") * 1000;
this.lastSubmissionTime = Telemetry.msSinceProcessStart();
}
getPingJSON(reason) {
return {
why: reason,
discarded: this.discarded || undefined,
version: PING_FORMAT_VERSION,
syncs: this.payloads.slice(),
};
}
finish(reason) {
// Note that we might be in the middle of a sync right now, and so we don't
// want to touch this.current.
let result = this.getPingJSON(reason);
this.payloads = [];
this.discarded = 0;
this.submit(result);
}
setupObservers() {
for (let topic of TOPICS) {
Observers.add(topic, this, this);
}
}
shutdown() {
this.finish("shutdown");
for (let topic of TOPICS) {
Observers.remove(topic, this, this);
}
}
submit(record) {
// We still call submit() with possibly illegal payloads so that tests can
// know that the ping was built. We don't end up submitting them, however.
if (record.syncs.length) {
log.trace(`submitting ${record.syncs.length} sync record(s) to telemetry`);
TelemetryController.submitExternalPing("sync", record);
}
}
onSyncStarted() {
if (this.current) {
log.warn("Observed weave:service:sync:start, but we're already recording a sync!");
// Just discard the old record, consistent with our handling of engines, above.
this.current = null;
}
this.current = new TelemetryRecord(this.allowedEngines);
}
_checkCurrent(topic) {
if (!this.current) {
log.warn(`Observed notification ${topic} but no current sync is being recorded.`);
return false;
}
return true;
}
onSyncFinished(error) {
if (!this.current) {
log.warn("onSyncFinished but we aren't recording");
return;
}
this.current.finished(error);
if (this.payloads.length < this.maxPayloadCount) {
this.payloads.push(this.current.toJSON());
} else {
++this.discarded;
}
this.current = null;
if ((Telemetry.msSinceProcessStart() - this.lastSubmissionTime) > this.submissionInterval) {
this.finish("schedule");
this.lastSubmissionTime = Telemetry.msSinceProcessStart();
}
}
observe(subject, topic, data) {
log.trace(`observed ${topic} ${data}`);
switch (topic) {
case "profile-before-change":
this.shutdown();
break;
/* sync itself state changes */
case "weave:service:sync:start":
this.onSyncStarted();
break;
case "weave:service:sync:finish":
if (this._checkCurrent(topic)) {
this.onSyncFinished(null);
}
break;
case "weave:service:sync:error":
// argument needs to be truthy (this should always be the case)
this.onSyncFinished(subject || "Unknown");
break;
/* engine sync state changes */
case "weave:engine:sync:start":
if (this._checkCurrent(topic)) {
this.current.onEngineStart(data);
}
break;
case "weave:engine:sync:finish":
if (this._checkCurrent(topic)) {
this.current.onEngineStop(data, null);
}
break;
case "weave:engine:sync:error":
if (this._checkCurrent(topic)) {
// argument needs to be truthy (this should always be the case)
this.current.onEngineStop(data, subject || "Unknown");
}
break;
/* engine counts */
case "weave:engine:sync:applied":
if (this._checkCurrent(topic)) {
this.current.onEngineApplied(data, subject);
}
break;
case "weave:engine:sync:uploaded":
if (this._checkCurrent(topic)) {
this.current.onEngineUploaded(data, subject);
}
break;
case "weave:engine:validate:finish":
if (this._checkCurrent(topic)) {
this.current.onEngineValidated(data, subject);
}
break;
case "weave:engine:validate:error":
if (this._checkCurrent(topic)) {
this.current.onEngineValidateError(data, subject || "Unknown");
}
break;
default:
log.warn(`unexpected observer topic ${topic}`);
break;
}
}
}
this.SyncTelemetry = new SyncTelemetryImpl(ENGINES);

2
services/sync/modules/userapi.js
View file

@ -8,7 +8,7 @@ this.EXPORTED_SYMBOLS = [
"UserAPI10Client",
];
var {utils: Cu} = Components;
const {utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");

148
services/sync/modules/util.js
View file

@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["XPCOMUtils", "Services", "Utils", "Async", "Svc", "Str"];
var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/observers.js");
@ -35,6 +35,8 @@ this.Utils = {
// In the ideal world, references to these would be removed.
nextTick: CommonUtils.nextTick,
namedTimer: CommonUtils.namedTimer,
exceptionStr: CommonUtils.exceptionStr,
stackTrace: CommonUtils.stackTrace,
makeURI: CommonUtils.makeURI,
encodeUTF8: CommonUtils.encodeUTF8,
decodeUTF8: CommonUtils.decodeUTF8,
@ -52,7 +54,6 @@ this.Utils = {
digestBytes: CryptoUtils.digestBytes,
sha1: CryptoUtils.sha1,
sha1Base32: CryptoUtils.sha1Base32,
sha256: CryptoUtils.sha256,
makeHMACKey: CryptoUtils.makeHMACKey,
makeHMACHasher: CryptoUtils.makeHMACHasher,
hkdfExpand: CryptoUtils.hkdfExpand,
@ -60,25 +61,6 @@ this.Utils = {
deriveKeyFromPassphrase: CryptoUtils.deriveKeyFromPassphrase,
getHTTPMACSHA1Header: CryptoUtils.getHTTPMACSHA1Header,
/**
* The string to use as the base User-Agent in Sync requests.
* This string will look something like
*
* Firefox/49.0a1 (Windows NT 6.1; WOW64; rv:46.0) FxSync/1.51.0.20160516142357.desktop
*/
_userAgent: null,
get userAgent() {
if (!this._userAgent) {
let hph = Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler);
this._userAgent =
Services.appinfo.name + "/" + Services.appinfo.version + // Product.
" (" + hph.oscpu + ")" + // (oscpu)
" FxSync/" + WEAVE_VERSION + "." + // Sync.
Services.appinfo.appBuildID + "."; // Build.
}
return this._userAgent + Svc.Prefs.get("client.type", "desktop");
},
/**
* Wrap a function to catch all exceptions and log them
*
@ -95,7 +77,7 @@ this.Utils = {
return func.call(thisArg);
}
catch(ex) {
thisArg._log.debug("Exception calling " + (func.name || "anonymous function"), ex);
thisArg._log.debug("Exception: " + Utils.exceptionStr(ex));
if (exceptionCallback) {
return exceptionCallback.call(thisArg, ex);
}
@ -271,14 +253,14 @@ this.Utils = {
*/
base32ToFriendly: function base32ToFriendly(input) {
return input.toLowerCase()
.replace(/l/g, '8')
.replace(/o/g, '9');
.replace("l", '8', "g")
.replace("o", '9', "g");
},
base32FromFriendly: function base32FromFriendly(input) {
return input.toUpperCase()
.replace(/8/g, 'L')
.replace(/9/g, 'O');
.replace("8", 'L', "g")
.replace("9", 'O', "g");
},
/**
@ -356,14 +338,12 @@ this.Utils = {
try {
json = yield CommonUtils.readJSON(path);
} catch (e if e instanceof OS.File.Error && e.becauseNoSuchFile) {
// Ignore non-existent files.
} catch (e) {
if (e instanceof OS.File.Error && e.becauseNoSuchFile) {
// Ignore non-existent files, but explicitly return null.
json = null;
} else {
if (that._log) {
that._log.debug("Failed to load json", e);
}
if (that._log) {
that._log.debug("Failed to load json: " +
CommonUtils.exceptionStr(e));
}
}
@ -411,52 +391,6 @@ this.Utils = {
}
}),
/**
* Move a json file in the profile directory. Will fail if a file exists at the
* destination.
*
* @returns a promise that resolves to undefined on success, or rejects on failure
*
* @param aFrom
* Current path to the JSON file saved on disk, relative to profileDir/weave
* .json will be appended to the file name.
* @param aTo
* New path to the JSON file saved on disk, relative to profileDir/weave
* .json will be appended to the file name.
* @param that
* Object to use for logging
*/
jsonMove(aFrom, aTo, that) {
let pathFrom = OS.Path.join(OS.Constants.Path.profileDir, "weave",
...(aFrom + ".json").split("/"));
let pathTo = OS.Path.join(OS.Constants.Path.profileDir, "weave",
...(aTo + ".json").split("/"));
if (that._log) {
that._log.trace("Moving " + pathFrom + " to " + pathTo);
}
return OS.File.move(pathFrom, pathTo, { noOverwrite: true });
},
/**
* Removes a json file in the profile directory.
*
* @returns a promise that resolves to undefined on success, or rejects on failure
*
* @param filePath
* Current path to the JSON file saved on disk, relative to profileDir/weave
* .json will be appended to the file name.
* @param that
* Object to use for logging
*/
jsonRemove(filePath, that) {
let path = OS.Path.join(OS.Constants.Path.profileDir, "weave",
...(filePath + ".json").split("/"));
if (that._log) {
that._log.trace("Deleting " + path);
}
return OS.File.remove(path, { ignoreAbsent: true });
},
getErrorString: function Utils_getErrorString(error, args) {
try {
return Str.errors.get(error, args || null);
@ -543,7 +477,7 @@ this.Utils = {
// 20-char sync key.
if (pp.length == 23 &&
[5, 11, 17].every(i => pp[i] == '-')) {
[5, 11, 17].every(function(i) pp[i] == '-')) {
return pp.slice(0, 5) + pp.slice(6, 11)
+ pp.slice(12, 17) + pp.slice(18, 23);
@ -551,7 +485,7 @@ this.Utils = {
// "Modern" 26-char key.
if (pp.length == 31 &&
[1, 7, 13, 19, 25].every(i => pp[i] == '-')) {
[1, 7, 13, 19, 25].every(function(i) pp[i] == '-')) {
return pp.slice(0, 1) + pp.slice(2, 7)
+ pp.slice(8, 13) + pp.slice(14, 19)
@ -681,12 +615,30 @@ this.Utils = {
* Get the FxA identity hosts.
*/
getSyncCredentialsHostsFxA: function() {
// This is somewhat expensive and the result static, so we cache the result.
if (this._syncCredentialsHostsFxA) {
return this._syncCredentialsHostsFxA;
}
let result = new Set();
// the FxA host
result.add(FxAccountsCommon.FXA_PWDMGR_HOST);
// We used to include the FxA hosts (hence the Set() result) but we now
// don't give them special treatment (hence the Set() with exactly 1 item)
return result;
//
// The FxA hosts - these almost certainly all have the same hostname, but
// better safe than sorry...
for (let prefName of ["identity.fxaccounts.remote.force_auth.uri",
"identity.fxaccounts.remote.signup.uri",
"identity.fxaccounts.remote.signin.uri",
"identity.fxaccounts.settings.uri"]) {
let prefVal;
try {
prefVal = Services.prefs.getCharPref(prefName);
} catch (_) {
continue;
}
let uri = Services.io.newURI(prefVal, null, null);
result.add(uri.prePath);
}
return this._syncCredentialsHostsFxA = result;
},
getDefaultDeviceName() {
@ -720,32 +672,6 @@ this.Utils = {
Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler).oscpu;
return Str.sync.get("client.name2", [user, appName, system]);
},
getDeviceName() {
const deviceName = Svc.Prefs.get("client.name", "");
if (deviceName === "") {
return this.getDefaultDeviceName();
}
return deviceName;
},
getDeviceType() {
return Svc.Prefs.get("client.type", DEVICE_TYPE_DESKTOP);
},
formatTimestamp(date) {
// Format timestamp as: "%Y-%m-%d %H:%M:%S"
let year = String(date.getFullYear());
let month = String(date.getMonth() + 1).padStart(2, "0");
let day = String(date.getDate()).padStart(2, "0");
let hours = String(date.getHours()).padStart(2, "0");
let minutes = String(date.getMinutes()).padStart(2, "0");
let seconds = String(date.getSeconds()).padStart(2, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
}
};
@ -764,7 +690,7 @@ Svc.Prefs = new Preferences(PREFS_BRANCH);
Svc.DefaultPrefs = new Preferences({branch: PREFS_BRANCH, defaultBranch: true});
Svc.Obs = Observers;
var _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ?
let _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ?
"@mozilla.org/suite/sessionstore;1" :
"@mozilla.org/browser/sessionstore;1";