wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-29 11:57:32 +09:00
Code Issues Projects Releases Packages Wiki Activity

Remove kinto client, Firefox kinto storage adapter, blocklist update client and integration with sync, OneCRL and the custom time check for derives system time.

Browse source
This commit is contained in:
NTD 2018-02-02 09:21:33 -05:00 • committed by Roy Tam
commit f5048f2a1d
28 changed files with 24 additions and 10664 deletions
Download patch file Download diff file Expand all files Collapse all files

310
services/common/blocklist-clients.js
View file

@ -1,310 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = ["AddonBlocklistClient",
"GfxBlocklistClient",
"OneCRLBlocklistClient",
"PluginBlocklistClient",
"FILENAME_ADDONS_JSON",
"FILENAME_GFX_JSON",
"FILENAME_PLUGINS_JSON"];
const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
Cu.import("resource://gre/modules/Services.jsm");
const { Task } = Cu.import("resource://gre/modules/Task.jsm");
const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
Cu.importGlobalProperties(["fetch"]);
const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
const { KintoHttpClient } = Cu.import("resource://services-common/kinto-http-client.js");
const { CanonicalJSON } = Components.utils.import("resource://gre/modules/CanonicalJSON.jsm");
const PREF_SETTINGS_SERVER = "services.settings.server";
const PREF_BLOCKLIST_BUCKET = "services.blocklist.bucket";
const PREF_BLOCKLIST_ONECRL_COLLECTION = "services.blocklist.onecrl.collection";
const PREF_BLOCKLIST_ONECRL_CHECKED_SECONDS = "services.blocklist.onecrl.checked";
const PREF_BLOCKLIST_ADDONS_COLLECTION = "services.blocklist.addons.collection";
const PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS = "services.blocklist.addons.checked";
const PREF_BLOCKLIST_PLUGINS_COLLECTION = "services.blocklist.plugins.collection";
const PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS = "services.blocklist.plugins.checked";
const PREF_BLOCKLIST_GFX_COLLECTION = "services.blocklist.gfx.collection";
const PREF_BLOCKLIST_GFX_CHECKED_SECONDS = "services.blocklist.gfx.checked";
const PREF_BLOCKLIST_ENFORCE_SIGNING = "services.blocklist.signing.enforced";
const INVALID_SIGNATURE = "Invalid content/signature";
this.FILENAME_ADDONS_JSON = "blocklist-addons.json";
this.FILENAME_GFX_JSON = "blocklist-gfx.json";
this.FILENAME_PLUGINS_JSON = "blocklist-plugins.json";
function mergeChanges(localRecords, changes) {
// Kinto.js adds attributes to local records that aren't present on server.
// (e.g. _status)
const stripPrivateProps = (obj) => {
return Object.keys(obj).reduce((current, key) => {
if (!key.startsWith("_")) {
current[key] = obj[key];
}
return current;
}, {});
};
const records = {};
// Local records by id.
localRecords.forEach((record) => records[record.id] = stripPrivateProps(record));
// All existing records are replaced by the version from the server.
changes.forEach((record) => records[record.id] = record);
return Object.values(records)
// Filter out deleted records.
.filter((record) => record.deleted != true)
// Sort list by record id.
.sort((a, b) => a.id < b.id ? -1 : a.id > b.id ? 1 : 0);
}
function fetchCollectionMetadata(collection) {
const client = new KintoHttpClient(collection.api.remote);
return client.bucket(collection.bucket).collection(collection.name).getData()
.then(result => {
return result.signature;
});
}
function fetchRemoteCollection(collection) {
const client = new KintoHttpClient(collection.api.remote);
return client.bucket(collection.bucket)
.collection(collection.name)
.listRecords({sort: "id"});
}
/**
* Helper to instantiate a Kinto client based on preferences for remote server
* URL and bucket name. It uses the `FirefoxAdapter` which relies on SQLite to
* persist the local DB.
*/
function kintoClient() {
let base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
let bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
let Kinto = loadKinto();
let FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
let config = {
remote: base,
bucket: bucket,
adapter: FirefoxAdapter,
};
return new Kinto(config);
}
class BlocklistClient {
constructor(collectionName, lastCheckTimePref, processCallback, signerName) {
this.collectionName = collectionName;
this.lastCheckTimePref = lastCheckTimePref;
this.processCallback = processCallback;
this.signerName = signerName;
}
validateCollectionSignature(payload, collection, ignoreLocal) {
return Task.spawn((function* () {
// this is a content-signature field from an autograph response.
const {x5u, signature} = yield fetchCollectionMetadata(collection);
const certChain = yield fetch(x5u).then((res) => res.text());
const verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
.createInstance(Ci.nsIContentSignatureVerifier);
let toSerialize;
if (ignoreLocal) {
toSerialize = {
last_modified: `${payload.last_modified}`,
data: payload.data
};
} else {
const localRecords = (yield collection.list()).data;
const records = mergeChanges(localRecords, payload.changes);
toSerialize = {
last_modified: `${payload.lastModified}`,
data: records
};
}
const serialized = CanonicalJSON.stringify(toSerialize);
if (verifier.verifyContentSignature(serialized, "p384ecdsa=" + signature,
certChain,
this.signerName)) {
// In case the hash is valid, apply the changes locally.
return payload;
}
throw new Error(INVALID_SIGNATURE);
}).bind(this));
}
/**
* Synchronize from Kinto server, if necessary.
*
* @param {int} lastModified the lastModified date (on the server) for
the remote collection.
* @param {Date} serverTime the current date return by the server.
* @return {Promise} which rejects on sync or process failure.
*/
maybeSync(lastModified, serverTime) {
let db = kintoClient();
let opts = {};
let enforceCollectionSigning =
Services.prefs.getBoolPref(PREF_BLOCKLIST_ENFORCE_SIGNING);
// if there is a signerName and collection signing is enforced, add a
// hook for incoming changes that validates the signature
if (this.signerName && enforceCollectionSigning) {
opts.hooks = {
"incoming-changes": [this.validateCollectionSignature.bind(this)]
}
}
let collection = db.collection(this.collectionName, opts);
return Task.spawn((function* syncCollection() {
try {
yield collection.db.open();
let collectionLastModified = yield collection.db.getLastModified();
// If the data is up to date, there's no need to sync. We still need
// to record the fact that a check happened.
if (lastModified <= collectionLastModified) {
this.updateLastCheck(serverTime);
return;
}
// Fetch changes from server.
try {
let syncResult = yield collection.sync();
if (!syncResult.ok) {
throw new Error("Sync failed");
}
} catch (e) {
if (e.message == INVALID_SIGNATURE) {
// if sync fails with a signature error, it's likely that our
// local data has been modified in some way.
// We will attempt to fix this by retrieving the whole
// remote collection.
let payload = yield fetchRemoteCollection(collection);
yield this.validateCollectionSignature(payload, collection, true);
// if the signature is good (we haven't thrown), and the remote
// last_modified is newer than the local last_modified, replace the
// local data
const localLastModified = yield collection.db.getLastModified();
if (payload.last_modified >= localLastModified) {
yield collection.clear();
yield collection.loadDump(payload.data);
}
} else {
throw e;
}
}
// Read local collection of records.
let list = yield collection.list();
yield this.processCallback(list.data);
// Track last update.
this.updateLastCheck(serverTime);
} finally {
collection.db.close();
}
}).bind(this));
}
/**
* Save last time server was checked in users prefs.
*
* @param {Date} serverTime the current date return by server.
*/
updateLastCheck(serverTime) {
let checkedServerTimeInSeconds = Math.round(serverTime / 1000);
Services.prefs.setIntPref(this.lastCheckTimePref, checkedServerTimeInSeconds);
}
}
/**
* Revoke the appropriate certificates based on the records from the blocklist.
*
* @param {Object} records current records in the local db.
*/
function* updateCertBlocklist(records) {
let certList = Cc["@mozilla.org/security/certblocklist;1"]
.getService(Ci.nsICertBlocklist);
for (let item of records) {
try {
if (item.issuerName && item.serialNumber) {
certList.revokeCertByIssuerAndSerial(item.issuerName,
item.serialNumber);
} else if (item.subject && item.pubKeyHash) {
certList.revokeCertBySubjectAndPubKey(item.subject,
item.pubKeyHash);
}
} catch (e) {
// prevent errors relating to individual blocklist entries from
// causing sync to fail. At some point in the future, we may want to
// accumulate telemetry on these failures.
Cu.reportError(e);
}
}
certList.saveEntries();
}
/**
* Write list of records into JSON file, and notify nsBlocklistService.
*
* @param {String} filename path relative to profile dir.
* @param {Object} records current records in the local db.
*/
function* updateJSONBlocklist(filename, records) {
// Write JSON dump for synchronous load at startup.
const path = OS.Path.join(OS.Constants.Path.profileDir, filename);
const serialized = JSON.stringify({data: records}, null, 2);
try {
yield OS.File.writeAtomic(path, serialized, {tmpPath: path + ".tmp"});
// Notify change to `nsBlocklistService`
const eventData = {filename: filename};
Services.cpmm.sendAsyncMessage("Blocklist:reload-from-disk", eventData);
} catch(e) {
Cu.reportError(e);
}
}
this.OneCRLBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION),
PREF_BLOCKLIST_ONECRL_CHECKED_SECONDS,
updateCertBlocklist,
"onecrl.content-signature.mozilla.org"
);
this.AddonBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS_COLLECTION),
PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS,
updateJSONBlocklist.bind(undefined, FILENAME_ADDONS_JSON)
);
this.GfxBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_GFX_COLLECTION),
PREF_BLOCKLIST_GFX_CHECKED_SECONDS,
updateJSONBlocklist.bind(undefined, FILENAME_GFX_JSON)
);
this.PluginBlocklistClient = new BlocklistClient(
Services.prefs.getCharPref(PREF_BLOCKLIST_PLUGINS_COLLECTION),
PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS,
updateJSONBlocklist.bind(undefined, FILENAME_PLUGINS_JSON)
);

117
services/common/blocklist-updater.js
View file

@ -1,117 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
this.EXPORTED_SYMBOLS = ["checkVersions", "addTestBlocklistClient"];
const { classes: Cc, Constructor: CC, interfaces: Ci, utils: Cu } = Components;
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/Task.jsm");
Cu.importGlobalProperties(['fetch']);
const BlocklistClients = Cu.import("resource://services-common/blocklist-clients.js", {});
const PREF_SETTINGS_SERVER = "services.settings.server";
const PREF_BLOCKLIST_CHANGES_PATH = "services.blocklist.changes.path";
const PREF_BLOCKLIST_BUCKET = "services.blocklist.bucket";
const PREF_BLOCKLIST_LAST_UPDATE = "services.blocklist.last_update_seconds";
const PREF_BLOCKLIST_LAST_ETAG = "services.blocklist.last_etag";
const PREF_BLOCKLIST_CLOCK_SKEW_SECONDS = "services.blocklist.clock_skew_seconds";
const gBlocklistClients = {
[BlocklistClients.OneCRLBlocklistClient.collectionName]: BlocklistClients.OneCRLBlocklistClient,
[BlocklistClients.AddonBlocklistClient.collectionName]: BlocklistClients.AddonBlocklistClient,
[BlocklistClients.GfxBlocklistClient.collectionName]: BlocklistClients.GfxBlocklistClient,
[BlocklistClients.PluginBlocklistClient.collectionName]: BlocklistClients.PluginBlocklistClient
};
// Add a blocklist client for testing purposes. Do not use for any other purpose
this.addTestBlocklistClient = (name, client) => { gBlocklistClients[name] = client; }
// This is called by the ping mechanism.
// returns a promise that rejects if something goes wrong
this.checkVersions = function() {
return Task.spawn(function* syncClients() {
// Fetch a versionInfo object that looks like:
// {"data":[
// {
// "host":"kinto-ota.dev.mozaws.net",
// "last_modified":1450717104423,
// "bucket":"blocklists",
// "collection":"certificates"
// }]}
// Right now, we only use the collection name and the last modified info
let kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
let changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_BLOCKLIST_CHANGES_PATH);
let blocklistsBucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
// Use ETag to obtain a `304 Not modified` when no change occurred.
const headers = {};
if (Services.prefs.prefHasUserValue(PREF_BLOCKLIST_LAST_ETAG)) {
const lastEtag = Services.prefs.getCharPref(PREF_BLOCKLIST_LAST_ETAG);
if (lastEtag) {
headers["If-None-Match"] = lastEtag;
}
}
let response = yield fetch(changesEndpoint, {headers});
let versionInfo;
// No changes since last time. Go on with empty list of changes.
if (response.status == 304) {
versionInfo = {data: []};
} else {
versionInfo = yield response.json();
}
// If the server is failing, the JSON response might not contain the
// expected data (e.g. error response - Bug 1259145)
if (!versionInfo.hasOwnProperty("data")) {
throw new Error("Polling for changes failed.");
}
// Record new update time and the difference between local and server time
let serverTimeMillis = Date.parse(response.headers.get("Date"));
// negative clockDifference means local time is behind server time
// by the absolute of that value in seconds (positive means it's ahead)
let clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
Services.prefs.setIntPref(PREF_BLOCKLIST_CLOCK_SKEW_SECONDS, clockDifference);
Services.prefs.setIntPref(PREF_BLOCKLIST_LAST_UPDATE, serverTimeMillis / 1000);
let firstError;
for (let collectionInfo of versionInfo.data) {
// Skip changes that don't concern configured blocklist bucket.
if (collectionInfo.bucket != blocklistsBucket) {
continue;
}
let collection = collectionInfo.collection;
let client = gBlocklistClients[collection];
if (client && client.maybeSync) {
let lastModified = 0;
if (collectionInfo.last_modified) {
lastModified = collectionInfo.last_modified;
}
try {
yield client.maybeSync(lastModified, serverTimeMillis);
} catch (e) {
if (!firstError) {
firstError = e;
}
}
}
}
if (firstError) {
// cause the promise to reject by throwing the first observed error
throw firstError;
}
// Save current Etag for next poll.
if (response.headers.has("ETag")) {
const currentEtag = response.headers.get("ETag");
Services.prefs.setCharPref(PREF_BLOCKLIST_LAST_ETAG, currentEtag);
}
});
};

1891
services/common/kinto-http-client.js
View file

File diff suppressed because it is too large Load diff

4286
services/common/kinto-offline-client.js
View file

File diff suppressed because it is too large Load diff

4
services/common/moz.build
View file

@ -15,10 +15,6 @@ EXTRA_COMPONENTS += [
EXTRA_JS_MODULES['services-common'] += [
'async.js',
'blocklist-clients.js',
'blocklist-updater.js',
'kinto-http-client.js',
'kinto-offline-client.js',
'logmanager.js',
'observers.js',
'rest.js',

224
services/common/tests/unit/test_blocklist_certificates.js
View file

@ -1,224 +0,0 @@
const { Constructor: CC } = Components;
Cu.import("resource://testing-common/httpd.js");
const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
"nsIBinaryInputStream", "setInputStream");
let server;
// set up what we need to make storage adapters
const Kinto = loadKinto();
const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
const kintoFilename = "kinto.sqlite";
let kintoClient;
function do_get_kinto_collection(collectionName) {
if (!kintoClient) {
let config = {
// Set the remote to be some server that will cause test failure when
// hit since we should never hit the server directly, only via maybeSync()
remote: "https://firefox.settings.services.mozilla.com/v1/",
// Set up the adapter and bucket as normal
adapter: FirefoxAdapter,
bucket: "blocklists"
};
kintoClient = new Kinto(config);
}
return kintoClient.collection(collectionName);
}
// Some simple tests to demonstrate that the logic inside maybeSync works
// correctly and that simple kinto operations are working as expected. There
// are more tests for core Kinto.js (and its storage adapter) in the
// xpcshell tests under /services/common
add_task(function* test_something(){
const configPath = "/v1/";
const recordsPath = "/v1/buckets/blocklists/collections/certificates/records";
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
// register a handler
function handleResponse (request, response) {
try {
const sample = getSampleResponse(request, server.identity.primaryPort);
if (!sample) {
do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
}
response.setStatusLine(null, sample.status.status,
sample.status.statusText);
// send the headers
for (let headerLine of sample.sampleHeaders) {
let headerElements = headerLine.split(':');
response.setHeader(headerElements[0], headerElements[1].trimLeft());
}
response.setHeader("Date", (new Date()).toUTCString());
response.write(sample.responseBody);
} catch (e) {
do_print(e);
}
}
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
// Test an empty db populates
let result = yield OneCRLBlocklistClient.maybeSync(2000, Date.now());
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
let collection = do_get_kinto_collection("certificates");
yield collection.db.open();
let list = yield collection.list();
do_check_eq(list.data.length, 1);
yield collection.db.close();
// Test the db is updated when we call again with a later lastModified value
result = yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
collection = do_get_kinto_collection("certificates");
yield collection.db.open();
list = yield collection.list();
do_check_eq(list.data.length, 3);
yield collection.db.close();
// Try to maybeSync with the current lastModified value - no connection
// should be attempted.
// Clear the kinto base pref so any connections will cause a test failure
Services.prefs.clearUserPref("services.settings.server");
yield OneCRLBlocklistClient.maybeSync(4000, Date.now());
// Try again with a lastModified value at some point in the past
yield OneCRLBlocklistClient.maybeSync(3000, Date.now());
// Check the OneCRL check time pref is modified, even if the collection
// hasn't changed
Services.prefs.setIntPref("services.blocklist.onecrl.checked", 0);
yield OneCRLBlocklistClient.maybeSync(3000, Date.now());
let newValue = Services.prefs.getIntPref("services.blocklist.onecrl.checked");
do_check_neq(newValue, 0);
// Check that a sync completes even when there's bad data in the
// collection. This will throw on fail, so just calling maybeSync is an
// acceptible test.
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
yield OneCRLBlocklistClient.maybeSync(5000, Date.now());
});
function run_test() {
// Ensure that signature verification is disabled to prevent interference
// with basic certificate sync tests
Services.prefs.setBoolPref("services.blocklist.signing.enforced", false);
// Set up an HTTP Server
server = new HttpServer();
server.start(-1);
run_next_test();
do_register_cleanup(function() {
server.stop(() => { });
});
}
// get a response for a given request from sample data
function getSampleResponse(req, port) {
const responses = {
"OPTIONS": {
"sampleHeaders": [
"Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
"Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
"Access-Control-Allow-Origin: *",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
"status": {status: 200, statusText: "OK"},
"responseBody": "null"
},
"GET:/v1/?": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
},
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"3000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"issuerName": "MEQxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xHjAcBgNVBAMTFXRoYXd0ZSBFViBTU0wgQ0EgLSBHMw==",
"serialNumber":"CrTHPEE6AZSfI3jysin2bA==",
"id":"78cf8900-fdea-4ce5-f8fb-b78710617718",
"last_modified":3000
}]})
},
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=3000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"4000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"issuerName":"MFkxCzAJBgNVBAYTAk5MMR4wHAYDVQQKExVTdGFhdCBkZXIgTmVkZXJsYW5kZW4xKjAoBgNVBAMTIVN0YWF0IGRlciBOZWRlcmxhbmRlbiBPdmVyaGVpZCBDQQ",
"serialNumber":"ATFpsA==",
"id":"dabafde9-df4a-ddba-2548-748da04cc02c",
"last_modified":4000
},{
"subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
"pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
"id":"dabafde9-df4a-ddba-2548-748da04cc02d",
"last_modified":4000
}]})
},
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"5000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"issuerName":"not a base64 encoded issuer",
"serialNumber":"not a base64 encoded serial",
"id":"dabafde9-df4a-ddba-2548-748da04cc02e",
"last_modified":5000
},{
"subject":"not a base64 encoded subject",
"pubKeyHash":"not a base64 encoded pubKeyHash",
"id":"dabafde9-df4a-ddba-2548-748da04cc02f",
"last_modified":5000
},{
"subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
"pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
"id":"dabafde9-df4a-ddba-2548-748da04cc02g",
"last_modified":5000
}]})
}
};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[req.method];
}

412
services/common/tests/unit/test_blocklist_clients.js
View file

@ -1,412 +0,0 @@
const { Constructor: CC } = Components;
const KEY_PROFILEDIR = "ProfD";
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://testing-common/httpd.js");
Cu.import("resource://gre/modules/Timer.jsm");
const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm");
const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
const BlocklistClients = Cu.import("resource://services-common/blocklist-clients.js");
const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
"nsIBinaryInputStream", "setInputStream");
const gBlocklistClients = [
{client: BlocklistClients.AddonBlocklistClient, filename: BlocklistClients.FILENAME_ADDONS_JSON, testData: ["i808","i720", "i539"]},
{client: BlocklistClients.PluginBlocklistClient, filename: BlocklistClients.FILENAME_PLUGINS_JSON, testData: ["p1044","p32","p28"]},
{client: BlocklistClients.GfxBlocklistClient, filename: BlocklistClients.FILENAME_GFX_JSON, testData: ["g204","g200","g36"]},
];
let server;
let kintoClient;
function kintoCollection(collectionName) {
if (!kintoClient) {
const Kinto = loadKinto();
const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
const config = {
// Set the remote to be some server that will cause test failure when
// hit since we should never hit the server directly, only via maybeSync()
remote: "https://firefox.settings.services.mozilla.com/v1/",
adapter: FirefoxAdapter,
bucket: "blocklists"
};
kintoClient = new Kinto(config);
}
return kintoClient.collection(collectionName);
}
function* readJSON(filepath) {
const binaryData = yield OS.File.read(filepath);
const textData = (new TextDecoder()).decode(binaryData);
return Promise.resolve(JSON.parse(textData));
}
function* clear_state() {
for (let {client} of gBlocklistClients) {
// Remove last server times.
Services.prefs.clearUserPref(client.lastCheckTimePref);
// Clear local DB.
const collection = kintoCollection(client.collectionName);
try {
yield collection.db.open();
yield collection.clear();
} finally {
yield collection.db.close();
}
}
// Remove profile data.
for (let {filename} of gBlocklistClients) {
const blocklist = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
if (blocklist.exists()) {
blocklist.remove(true);
}
}
}
function run_test() {
// Set up an HTTP Server
server = new HttpServer();
server.start(-1);
// Point the blocklist clients to use this local HTTP server.
Services.prefs.setCharPref("services.settings.server",
`http://localhost:${server.identity.primaryPort}/v1`);
// Setup server fake responses.
function handleResponse(request, response) {
try {
const sample = getSampleResponse(request, server.identity.primaryPort);
if (!sample) {
do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
}
response.setStatusLine(null, sample.status.status,
sample.status.statusText);
// send the headers
for (let headerLine of sample.sampleHeaders) {
let headerElements = headerLine.split(':');
response.setHeader(headerElements[0], headerElements[1].trimLeft());
}
response.setHeader("Date", (new Date()).toUTCString());
response.write(sample.responseBody);
response.finish();
} catch (e) {
do_print(e);
}
}
const configPath = "/v1/";
const addonsRecordsPath = "/v1/buckets/blocklists/collections/addons/records";
const gfxRecordsPath = "/v1/buckets/blocklists/collections/gfx/records";
const pluginsRecordsPath = "/v1/buckets/blocklists/collections/plugins/records";
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(addonsRecordsPath, handleResponse);
server.registerPathHandler(gfxRecordsPath, handleResponse);
server.registerPathHandler(pluginsRecordsPath, handleResponse);
run_next_test();
do_register_cleanup(function() {
server.stop(() => { });
});
}
add_task(function* test_records_obtained_from_server_are_stored_in_db(){
for (let {client} of gBlocklistClients) {
// Test an empty db populates
let result = yield client.maybeSync(2000, Date.now());
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
let collection = kintoCollection(client.collectionName);
yield collection.db.open();
let list = yield collection.list();
equal(list.data.length, 1);
yield collection.db.close();
}
});
add_task(clear_state);
add_task(function* test_list_is_written_to_file_in_profile(){
for (let {client, filename, testData} of gBlocklistClients) {
const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
strictEqual(profFile.exists(), false);
let result = yield client.maybeSync(2000, Date.now());
strictEqual(profFile.exists(), true);
const content = yield readJSON(profFile.path);
equal(content.data[0].blockID, testData[testData.length - 1]);
}
});
add_task(clear_state);
add_task(function* test_current_server_time_is_saved_in_pref(){
for (let {client} of gBlocklistClients) {
const before = Services.prefs.getIntPref(client.lastCheckTimePref);
const serverTime = Date.now();
yield client.maybeSync(2000, serverTime);
const after = Services.prefs.getIntPref(client.lastCheckTimePref);
equal(after, Math.round(serverTime / 1000));
}
});
add_task(clear_state);
add_task(function* test_update_json_file_when_addons_has_changes(){
for (let {client, filename, testData} of gBlocklistClients) {
yield client.maybeSync(2000, Date.now() - 1000);
const before = Services.prefs.getIntPref(client.lastCheckTimePref);
const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
const serverTime = Date.now();
yield client.maybeSync(3001, serverTime);
// File was updated.
notEqual(fileLastModified, profFile.lastModifiedTime);
const content = yield readJSON(profFile.path);
deepEqual(content.data.map((r) => r.blockID), testData);
// Server time was updated.
const after = Services.prefs.getIntPref(client.lastCheckTimePref);
equal(after, Math.round(serverTime / 1000));
}
});
add_task(clear_state);
add_task(function* test_sends_reload_message_when_blocklist_has_changes(){
for (let {client, filename} of gBlocklistClients) {
let received = yield new Promise((resolve, reject) => {
Services.ppmm.addMessageListener("Blocklist:reload-from-disk", {
receiveMessage(aMsg) { resolve(aMsg) }
});
client.maybeSync(2000, Date.now() - 1000);
});
equal(received.data.filename, filename);
}
});
add_task(clear_state);
add_task(function* test_do_nothing_when_blocklist_is_up_to_date(){
for (let {client, filename} of gBlocklistClients) {
yield client.maybeSync(2000, Date.now() - 1000);
const before = Services.prefs.getIntPref(client.lastCheckTimePref);
const profFile = FileUtils.getFile(KEY_PROFILEDIR, [filename]);
const fileLastModified = profFile.lastModifiedTime = profFile.lastModifiedTime - 1000;
const serverTime = Date.now();
yield client.maybeSync(3000, serverTime);
// File was not updated.
equal(fileLastModified, profFile.lastModifiedTime);
// Server time was updated.
const after = Services.prefs.getIntPref(client.lastCheckTimePref);
equal(after, Math.round(serverTime / 1000));
}
});
add_task(clear_state);
// get a response for a given request from sample data
function getSampleResponse(req, port) {
const responses = {
"OPTIONS": {
"sampleHeaders": [
"Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
"Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
"Access-Control-Allow-Origin: *",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
"status": {status: 200, statusText: "OK"},
"responseBody": "null"
},
"GET:/v1/?": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
},
"GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"3000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"prefs": [],
"blockID": "i539",
"last_modified": 3000,
"versionRange": [{
"targetApplication": [],
"maxVersion": "*",
"minVersion": "0",
"severity": "1"
}],
"guid": "ScorpionSaver@jetpack",
"id": "9d500963-d80e-3a91-6e74-66f3811b99cc"
}]})
},
"GET:/v1/buckets/blocklists/collections/plugins/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"3000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"matchFilename": "NPFFAddOn.dll",
"blockID": "p28",
"id": "7b1e0b3c-e390-a817-11b6-a6887f65f56e",
"last_modified": 3000,
"versionRange": []
}]})
},
"GET:/v1/buckets/blocklists/collections/gfx/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"3000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"driverVersionComparator": "LESS_THAN_OR_EQUAL",
"driverVersion": "8.17.12.5896",
"vendor": "0x10de",
"blockID": "g36",
"feature": "DIRECT3D_9_LAYERS",
"devices": ["0x0a6c"],
"featureStatus": "BLOCKED_DRIVER_VERSION",
"last_modified": 3000,
"os": "WINNT 6.1",
"id": "3f947f16-37c2-4e96-d356-78b26363729b"
}]})
},
"GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified&_since=3000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"4000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"prefs": [],
"blockID": "i808",
"last_modified": 4000,
"versionRange": [{
"targetApplication": [],
"maxVersion": "*",
"minVersion": "0",
"severity": "3"
}],
"guid": "{c96d1ae6-c4cf-4984-b110-f5f561b33b5a}",
"id": "9ccfac91-e463-c30c-f0bd-14143794a8dd"
}, {
"prefs": ["browser.startup.homepage"],
"blockID": "i720",
"last_modified": 3500,
"versionRange": [{
"targetApplication": [],
"maxVersion": "*",
"minVersion": "0",
"severity": "1"
}],
"guid": "FXqG@xeeR.net",
"id": "cf9b3129-a97e-dbd7-9525-a8575ac03c25"
}]})
},
"GET:/v1/buckets/blocklists/collections/plugins/records?_sort=-last_modified&_since=3000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"4000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"infoURL": "https://get.adobe.com/flashplayer/",
"blockID": "p1044",
"matchFilename": "libflashplayer\\.so",
"last_modified": 4000,
"versionRange": [{
"targetApplication": [],
"minVersion": "11.2.202.509",
"maxVersion": "11.2.202.539",
"severity": "0",
"vulnerabilityStatus": "1"
}],
"os": "Linux",
"id": "aabad965-e556-ffe7-4191-074f5dee3df3"
}, {
"matchFilename": "npViewpoint.dll",
"blockID": "p32",
"id": "1f48af42-c508-b8ef-b8d5-609d48e4f6c9",
"last_modified": 3500,
"versionRange": [{
"targetApplication": [{
"minVersion": "3.0",
"guid": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
"maxVersion": "*"
}]
}]
}]})
},
"GET:/v1/buckets/blocklists/collections/gfx/records?_sort=-last_modified&_since=3000": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"4000\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{
"vendor": "0x8086",
"blockID": "g204",
"feature": "WEBGL_MSAA",
"devices": [],
"id": "c96bca82-e6bd-044d-14c4-9c1d67e9283a",
"last_modified": 4000,
"os": "Darwin 10",
"featureStatus": "BLOCKED_DEVICE"
}, {
"vendor": "0x10de",
"blockID": "g200",
"feature": "WEBGL_MSAA",
"devices": [],
"id": "c3a15ba9-e0e2-421f-e399-c995e5b8d14e",
"last_modified": 3500,
"os": "Darwin 11",
"featureStatus": "BLOCKED_DEVICE"
}]})
}
};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[req.method];
}

510
services/common/tests/unit/test_blocklist_signatures.js
View file

@ -1,510 +0,0 @@
"use strict";
Cu.import("resource://services-common/blocklist-updater.js");
Cu.import("resource://testing-common/httpd.js");
const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
let server;
const PREF_BLOCKLIST_BUCKET = "services.blocklist.bucket";
const PREF_BLOCKLIST_ENFORCE_SIGNING = "services.blocklist.signing.enforced";
const PREF_BLOCKLIST_ONECRL_COLLECTION = "services.blocklist.onecrl.collection";
const PREF_SETTINGS_SERVER = "services.settings.server";
const PREF_SIGNATURE_ROOT = "security.content.signature.root_hash";
const CERT_DIR = "test_blocklist_signatures/";
const CHAIN_FILES =
["collection_signing_ee.pem",
"collection_signing_int.pem",
"collection_signing_root.pem"];
function getFileData(file) {
const stream = Cc["@mozilla.org/network/file-input-stream;1"]
.createInstance(Ci.nsIFileInputStream);
stream.init(file, -1, 0, 0);
const data = NetUtil.readInputStreamToString(stream, stream.available());
stream.close();
return data;
}
function setRoot() {
const filename = CERT_DIR + CHAIN_FILES[0];
const certFile = do_get_file(filename, false);
const b64cert = getFileData(certFile)
.replace(/-----BEGIN CERTIFICATE-----/, "")
.replace(/-----END CERTIFICATE-----/, "")
.replace(/[\r\n]/g, "");
const certdb = Cc["@mozilla.org/security/x509certdb;1"]
.getService(Ci.nsIX509CertDB);
const cert = certdb.constructX509FromBase64(b64cert);
Services.prefs.setCharPref(PREF_SIGNATURE_ROOT, cert.sha256Fingerprint);
}
function getCertChain() {
const chain = [];
for (let file of CHAIN_FILES) {
chain.push(getFileData(do_get_file(CERT_DIR + file)));
}
return chain.join("\n");
}
function* checkRecordCount(count) {
// open the collection manually
const base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
const bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
const collectionName =
Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION);
const Kinto = loadKinto();
const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
const config = {
remote: base,
bucket: bucket,
adapter: FirefoxAdapter,
};
const db = new Kinto(config);
const collection = db.collection(collectionName);
yield collection.db.open();
// Check we have the expected number of records
let records = yield collection.list();
do_check_eq(count, records.data.length);
// Close the collection so the test can exit cleanly
yield collection.db.close();
}
// Check to ensure maybeSync is called with correct values when a changes
// document contains information on when a collection was last modified
add_task(function* test_check_signatures(){
const port = server.identity.primaryPort;
// a response to give the client when the cert chain is expected
function makeMetaResponseBody(lastModified, signature) {
return {
data: {
id: "certificates",
last_modified: lastModified,
signature: {
x5u: `http://localhost:${port}/test_blocklist_signatures/test_cert_chain.pem`,
public_key: "fake",
"content-signature": `x5u=http://localhost:${port}/test_blocklist_signatures/test_cert_chain.pem;p384ecdsa=${signature}`,
signature_encoding: "rs_base64url",
signature: signature,
hash_algorithm: "sha384",
ref: "1yryrnmzou5rf31ou80znpnq8n"
}
}
};
}
function makeMetaResponse(eTag, body, comment) {
return {
comment: comment,
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
`ETag: \"${eTag}\"`
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify(body)
};
}
function registerHandlers(responses){
function handleResponse (serverTimeMillis, request, response) {
const key = `${request.method}:${request.path}?${request.queryString}`;
const available = responses[key];
const sampled = available.length > 1 ? available.shift() : available[0];
if (!sampled) {
do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
}
response.setStatusLine(null, sampled.status.status,
sampled.status.statusText);
// send the headers
for (let headerLine of sampled.sampleHeaders) {
let headerElements = headerLine.split(':');
response.setHeader(headerElements[0], headerElements[1].trimLeft());
}
// set the server date
response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
response.write(sampled.responseBody);
}
for (let key of Object.keys(responses)) {
const keyParts = key.split(":");
const method = keyParts[0];
const valueParts = keyParts[1].split("?");
const path = valueParts[0];
server.registerPathHandler(path, handleResponse.bind(null, 2000));
}
}
// First, perform a signature verification with known data and signature
// to ensure things are working correctly
let verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
.createInstance(Ci.nsIContentSignatureVerifier);
const emptyData = '[]';
const emptySignature = "p384ecdsa=zbugm2FDitsHwk5-IWsas1PpWwY29f0Fg5ZHeqD8fzep7AVl2vfcaHA7LdmCZ28qZLOioGKvco3qT117Q4-HlqFTJM7COHzxGyU2MMJ0ZTnhJrPOC1fP3cVQjU1PTWi9";
const name = "onecrl.content-signature.mozilla.org";
ok(verifier.verifyContentSignature(emptyData, emptySignature,
getCertChain(), name));
verifier = Cc["@mozilla.org/security/contentsignatureverifier;1"]
.createInstance(Ci.nsIContentSignatureVerifier);
const collectionData = '[{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:43:37Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"97fbf7c4-3ef2-f54f-0029-1ba6540c63ea","issuerName":"MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==","last_modified":2000,"serialNumber":"BAAAAAABA/A35EU="},{"details":{"bug":"https://bugzilla.mozilla.org/show_bug.cgi?id=1155145","created":"2016-01-18T14:48:11Z","name":"GlobalSign certs","who":".","why":"."},"enabled":true,"id":"e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc","issuerName":"MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB","last_modified":3000,"serialNumber":"BAAAAAABI54PryQ="}]';
const collectionSignature = "p384ecdsa=f4pA2tYM5jQgWY6YUmhUwQiBLj6QO5sHLD_5MqLePz95qv-7cNCuQoZnPQwxoptDtW8hcWH3kLb0quR7SB-r82gkpR9POVofsnWJRA-ETb0BcIz6VvI3pDT49ZLlNg3p";
ok(verifier.verifyContentSignature(collectionData, collectionSignature, getCertChain(), name));
// set up prefs so the kinto updater talks to the test server
Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
`http://localhost:${server.identity.primaryPort}/v1`);
// Set up some data we need for our test
let startTime = Date.now();
// These are records we'll use in the test collections
const RECORD1 = {
details: {
bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
created: "2016-01-18T14:43:37Z",
name: "GlobalSign certs",
who: ".",
why: "."
},
enabled: true,
id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
issuerName: "MHExKDAmBgNVBAMTH0dsb2JhbFNpZ24gUm9vdFNpZ24gUGFydG5lcnMgQ0ExHTAbBgNVBAsTFFJvb3RTaWduIFBhcnRuZXJzIENBMRkwFwYDVQQKExBHbG9iYWxTaWduIG52LXNhMQswCQYDVQQGEwJCRQ==",
last_modified: 2000,
serialNumber: "BAAAAAABA/A35EU="
};
const RECORD2 = {
details: {
bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
created: "2016-01-18T14:48:11Z",
name: "GlobalSign certs",
who: ".",
why: "."
},
enabled: true,
id: "e3bd531e-1ee4-7407-27ce-6fdc9cecbbdc",
issuerName: "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
last_modified: 3000,
serialNumber: "BAAAAAABI54PryQ="
};
const RECORD3 = {
details: {
bug: "https://bugzilla.mozilla.org/show_bug.cgi?id=1155145",
created: "2016-01-18T14:48:11Z",
name: "GlobalSign certs",
who: ".",
why: "."
},
enabled: true,
id: "c7c49b69-a4ab-418e-92a9-e1961459aa7f",
issuerName: "MIGBMQswCQYDVQQGEwJCRTEZMBcGA1UEChMQR2xvYmFsU2lnbiBudi1zYTElMCMGA1UECxMcUHJpbWFyeSBPYmplY3QgUHVibGlzaGluZyBDQTEwMC4GA1UEAxMnR2xvYmFsU2lnbiBQcmltYXJ5IE9iamVjdCBQdWJsaXNoaW5nIENB",
last_modified: 4000,
serialNumber: "BAAAAAABI54PryQ="
};
const RECORD1_DELETION = {
deleted: true,
enabled: true,
id: "97fbf7c4-3ef2-f54f-0029-1ba6540c63ea",
last_modified: 3500,
};
// Check that a signature on an empty collection is OK
// We need to set up paths on the HTTP server to return specific data from
// specific paths for each test. Here we prepare data for each response.
// A cert chain response (this the cert chain that contains the signing
// cert, the root and any intermediates in between). This is used in each
// sync.
const RESPONSE_CERT_CHAIN = {
comment: "RESPONSE_CERT_CHAIN",
sampleHeaders: [
"Content-Type: text/plain; charset=UTF-8"
],
status: {status: 200, statusText: "OK"},
responseBody: getCertChain()
};
// A server settings response. This is used in each sync.
const RESPONSE_SERVER_SETTINGS = {
comment: "RESPONSE_SERVER_SETTINGS",
sampleHeaders: [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
};
// This is the initial, empty state of the collection. This is only used
// for the first sync.
const RESPONSE_EMPTY_INITIAL = {
comment: "RESPONSE_EMPTY_INITIAL",
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"1000\""
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"data": []})
};
const RESPONSE_BODY_META_EMPTY_SIG = makeMetaResponseBody(1000,
"vxuAg5rDCB-1pul4a91vqSBQRXJG_j7WOYUTswxRSMltdYmbhLRH8R8brQ9YKuNDF56F-w6pn4HWxb076qgKPwgcEBtUeZAO_RtaHXRkRUUgVzAr86yQL4-aJTbv3D6u");
// The collection metadata containing the signature for the empty
// collection.
const RESPONSE_META_EMPTY_SIG =
makeMetaResponse(1000, RESPONSE_BODY_META_EMPTY_SIG,
"RESPONSE_META_EMPTY_SIG");
// Here, we map request method and path to the available responses
const emptyCollectionResponses = {
"GET:/test_blocklist_signatures/test_cert_chain.pem?":[RESPONSE_CERT_CHAIN],
"GET:/v1/?": [RESPONSE_SERVER_SETTINGS],
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified":
[RESPONSE_EMPTY_INITIAL],
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_EMPTY_SIG]
};
// .. and use this map to register handlers for each path
registerHandlers(emptyCollectionResponses);
// With all of this set up, we attempt a sync. This will resolve if all is
// well and throw if something goes wrong.
yield OneCRLBlocklistClient.maybeSync(1000, startTime);
// Check that some additions (2 records) to the collection have a valid
// signature.
// This response adds two entries (RECORD1 and RECORD2) to the collection
const RESPONSE_TWO_ADDED = {
comment: "RESPONSE_TWO_ADDED",
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"3000\""
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"data": [RECORD2, RECORD1]})
};
const RESPONSE_BODY_META_TWO_ITEMS_SIG = makeMetaResponseBody(3000,
"dwhJeypadNIyzGj3QdI0KMRTPnHhFPF_j73mNrsPAHKMW46S2Ftf4BzsPMvPMB8h0TjDus13wo_R4l432DHe7tYyMIWXY0PBeMcoe5BREhFIxMxTsh9eGVXBD1e3UwRy");
// A signature response for the collection containg RECORD1 and RECORD2
const RESPONSE_META_TWO_ITEMS_SIG =
makeMetaResponse(3000, RESPONSE_BODY_META_TWO_ITEMS_SIG,
"RESPONSE_META_TWO_ITEMS_SIG");
const twoItemsResponses = {
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=1000":
[RESPONSE_TWO_ADDED],
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_TWO_ITEMS_SIG]
};
registerHandlers(twoItemsResponses);
yield OneCRLBlocklistClient.maybeSync(3000, startTime);
// Check the collection with one addition and one removal has a valid
// signature
// Remove RECORD1, add RECORD3
const RESPONSE_ONE_ADDED_ONE_REMOVED = {
comment: "RESPONSE_ONE_ADDED_ONE_REMOVED ",
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"4000\""
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"data": [RECORD3, RECORD1_DELETION]})
};
const RESPONSE_BODY_META_THREE_ITEMS_SIG = makeMetaResponseBody(4000,
"MIEmNghKnkz12UodAAIc3q_Y4a3IJJ7GhHF4JYNYmm8avAGyPM9fYU7NzVo94pzjotG7vmtiYuHyIX2rTHTbT587w0LdRWxipgFd_PC1mHiwUyjFYNqBBG-kifYk7kEw");
// signature response for the collection containing RECORD2 and RECORD3
const RESPONSE_META_THREE_ITEMS_SIG =
makeMetaResponse(4000, RESPONSE_BODY_META_THREE_ITEMS_SIG,
"RESPONSE_META_THREE_ITEMS_SIG");
const oneAddedOneRemovedResponses = {
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=3000":
[RESPONSE_ONE_ADDED_ONE_REMOVED],
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_THREE_ITEMS_SIG]
};
registerHandlers(oneAddedOneRemovedResponses);
yield OneCRLBlocklistClient.maybeSync(4000, startTime);
// Check the signature is still valid with no operation (no changes)
// Leave the collection unchanged
const RESPONSE_EMPTY_NO_UPDATE = {
comment: "RESPONSE_EMPTY_NO_UPDATE ",
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"4000\""
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"data": []})
};
const noOpResponses = {
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
[RESPONSE_EMPTY_NO_UPDATE],
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_THREE_ITEMS_SIG]
};
registerHandlers(noOpResponses);
yield OneCRLBlocklistClient.maybeSync(4100, startTime);
// Check the collection is reset when the signature is invalid
// Prepare a (deliberately) bad signature to check the collection state is
// reset if something is inconsistent
const RESPONSE_COMPLETE_INITIAL = {
comment: "RESPONSE_COMPLETE_INITIAL ",
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"4000\""
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"data": [RECORD2, RECORD3]})
};
const RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID = {
comment: "RESPONSE_COMPLETE_INITIAL ",
sampleHeaders: [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"4000\""
],
status: {status: 200, statusText: "OK"},
responseBody: JSON.stringify({"data": [RECORD3, RECORD2]})
};
const RESPONSE_BODY_META_BAD_SIG = makeMetaResponseBody(4000,
"aW52YWxpZCBzaWduYXR1cmUK");
const RESPONSE_META_BAD_SIG =
makeMetaResponse(4000, RESPONSE_BODY_META_BAD_SIG, "RESPONSE_META_BAD_SIG");
const badSigGoodSigResponses = {
// In this test, we deliberately serve a bad signature initially. The
// subsequent signature returned is a valid one for the three item
// collection.
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_BAD_SIG, RESPONSE_META_THREE_ITEMS_SIG],
// The first collection state is the three item collection (since
// there's a sync with no updates) - but, since the signature is wrong,
// another request will be made...
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
[RESPONSE_EMPTY_NO_UPDATE],
// The next request is for the full collection. This will be checked
// against the valid signature - so the sync should succeed.
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified":
[RESPONSE_COMPLETE_INITIAL],
// The next request is for the full collection sorted by id. This will be
// checked against the valid signature - so the sync should succeed.
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
[RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID]
};
registerHandlers(badSigGoodSigResponses);
yield OneCRLBlocklistClient.maybeSync(5000, startTime);
const badSigGoodOldResponses = {
// In this test, we deliberately serve a bad signature initially. The
// subsequent sitnature returned is a valid one for the three item
// collection.
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_BAD_SIG, RESPONSE_META_EMPTY_SIG],
// The first collection state is the current state (since there's no update
// - but, since the signature is wrong, another request will be made)
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
[RESPONSE_EMPTY_NO_UPDATE],
// The next request is for the full collection sorted by id. This will be
// checked against the valid signature and last_modified times will be
// compared. Sync should fail, even though the signature is good,
// because the local collection is newer.
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
[RESPONSE_EMPTY_INITIAL],
};
// ensure our collection hasn't been replaced with an older, empty one
yield checkRecordCount(2);
registerHandlers(badSigGoodOldResponses);
yield OneCRLBlocklistClient.maybeSync(5000, startTime);
const allBadSigResponses = {
// In this test, we deliberately serve only a bad signature.
"GET:/v1/buckets/blocklists/collections/certificates?":
[RESPONSE_META_BAD_SIG],
// The first collection state is the three item collection (since
// there's a sync with no updates) - but, since the signature is wrong,
// another request will be made...
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=-last_modified&_since=4000":
[RESPONSE_EMPTY_NO_UPDATE],
// The next request is for the full collection sorted by id. This will be
// checked against the valid signature - so the sync should succeed.
"GET:/v1/buckets/blocklists/collections/certificates/records?_sort=id":
[RESPONSE_COMPLETE_INITIAL_SORTED_BY_ID]
};
registerHandlers(allBadSigResponses);
try {
yield OneCRLBlocklistClient.maybeSync(6000, startTime);
do_throw("Sync should fail (the signature is intentionally bad)");
} catch (e) {
yield checkRecordCount(2);
}
});
function run_test() {
// ensure signatures are enforced
Services.prefs.setBoolPref(PREF_BLOCKLIST_ENFORCE_SIGNING, true);
// get a signature verifier to ensure nsNSSComponent is initialized
Cc["@mozilla.org/security/contentsignatureverifier;1"]
.createInstance(Ci.nsIContentSignatureVerifier);
// set the content signing root to our test root
setRoot();
// Set up an HTTP Server
server = new HttpServer();
server.start(-1);
run_next_test();
do_register_cleanup(function() {
server.stop(function() { });
});
}

173
services/common/tests/unit/test_blocklist_updater.js
View file

@ -1,173 +0,0 @@
Cu.import("resource://testing-common/httpd.js");
var server;
const PREF_SETTINGS_SERVER = "services.settings.server";
const PREF_LAST_UPDATE = "services.blocklist.last_update_seconds";
const PREF_LAST_ETAG = "services.blocklist.last_etag";
const PREF_CLOCK_SKEW_SECONDS = "services.blocklist.clock_skew_seconds";
// Check to ensure maybeSync is called with correct values when a changes
// document contains information on when a collection was last modified
add_task(function* test_check_maybeSync(){
const changesPath = "/v1/buckets/monitor/collections/changes/records";
// register a handler
function handleResponse (serverTimeMillis, request, response) {
try {
const sampled = getSampleResponse(request, server.identity.primaryPort);
if (!sampled) {
do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
}
response.setStatusLine(null, sampled.status.status,
sampled.status.statusText);
// send the headers
for (let headerLine of sampled.sampleHeaders) {
let headerElements = headerLine.split(':');
response.setHeader(headerElements[0], headerElements[1].trimLeft());
}
// set the server date
response.setHeader("Date", (new Date(serverTimeMillis)).toUTCString());
response.write(sampled.responseBody);
} catch (e) {
dump(`${e}\n`);
}
}
server.registerPathHandler(changesPath, handleResponse.bind(null, 2000));
// set up prefs so the kinto updater talks to the test server
Services.prefs.setCharPref(PREF_SETTINGS_SERVER,
`http://localhost:${server.identity.primaryPort}/v1`);
// set some initial values so we can check these are updated appropriately
Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
Services.prefs.setIntPref(PREF_CLOCK_SKEW_SECONDS, 0);
Services.prefs.clearUserPref(PREF_LAST_ETAG);
let startTime = Date.now();
let updater = Cu.import("resource://services-common/blocklist-updater.js");
let syncPromise = new Promise(function(resolve, reject) {
// add a test kinto client that will respond to lastModified information
// for a collection called 'test-collection'
updater.addTestBlocklistClient("test-collection", {
maybeSync(lastModified, serverTime) {
do_check_eq(lastModified, 1000);
do_check_eq(serverTime, 2000);
resolve();
}
});
updater.checkVersions();
});
// ensure we get the maybeSync call
yield syncPromise;
// check the last_update is updated
do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
// How does the clock difference look?
let endTime = Date.now();
let clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
// we previously set the serverTime to 2 (seconds past epoch)
do_check_true(clockDifference <= endTime / 1000
&& clockDifference >= Math.floor(startTime / 1000) - 2);
// Last timestamp was saved. An ETag header value is a quoted string.
let lastEtag = Services.prefs.getCharPref(PREF_LAST_ETAG);
do_check_eq(lastEtag, "\"1100\"");
// Simulate a poll with up-to-date collection.
Services.prefs.setIntPref(PREF_LAST_UPDATE, 0);
// If server has no change, a 304 is received, maybeSync() is not called.
updater.addTestBlocklistClient("test-collection", {
maybeSync: () => {throw new Error("Should not be called");}
});
yield updater.checkVersions();
// Last update is overwritten
do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
// Simulate a server error.
function simulateErrorResponse (request, response) {
response.setHeader("Date", (new Date(3000)).toUTCString());
response.setHeader("Content-Type", "application/json; charset=UTF-8");
response.write(JSON.stringify({
code: 503,
errno: 999,
error: "Service Unavailable",
}));
response.setStatusLine(null, 503, "Service Unavailable");
}
server.registerPathHandler(changesPath, simulateErrorResponse);
// checkVersions() fails with adequate error.
let error;
try {
yield updater.checkVersions();
} catch (e) {
error = e;
}
do_check_eq(error.message, "Polling for changes failed.");
// When an error occurs, last update was not overwritten (see Date header above).
do_check_eq(Services.prefs.getIntPref(PREF_LAST_UPDATE), 2);
// check negative clock skew times
// set to a time in the future
server.registerPathHandler(changesPath, handleResponse.bind(null, Date.now() + 10000));
yield updater.checkVersions();
clockDifference = Services.prefs.getIntPref(PREF_CLOCK_SKEW_SECONDS);
// we previously set the serverTime to Date.now() + 10000 ms past epoch
do_check_true(clockDifference <= 0 && clockDifference >= -10);
});
function run_test() {
// Set up an HTTP Server
server = new HttpServer();
server.start(-1);
run_next_test();
do_register_cleanup(function() {
server.stop(function() { });
});
}
// get a response for a given request from sample data
function getSampleResponse(req, port) {
const responses = {
"GET:/v1/buckets/monitor/collections/changes/records?": {
"sampleHeaders": [
"Content-Type: application/json; charset=UTF-8",
"ETag: \"1100\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data": [{
"host": "localhost",
"last_modified": 1100,
"bucket": "blocklists:aurora",
"id": "330a0c5f-fadf-ff0b-40c8-4eb0d924ff6a",
"collection": "test-collection"
}, {
"host": "localhost",
"last_modified": 1000,
"bucket": "blocklists",
"id": "254cbb9e-6888-4d9f-8e60-58b74faa8778",
"collection": "test-collection"
}]})
}
};
if (req.hasHeader("if-none-match") && req.getHeader("if-none-match", "") == "\"1100\"")
return {sampleHeaders: [], status: {status: 304, statusText: "Not Modified"}, responseBody: ""};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[req.method];
}

412
services/common/tests/unit/test_kinto.js
View file

@ -1,412 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
http://creativecommons.org/publicdomain/zero/1.0/ */
Cu.import("resource://services-common/kinto-offline-client.js");
Cu.import("resource://testing-common/httpd.js");
const BinaryInputStream = Components.Constructor("@mozilla.org/binaryinputstream;1",
"nsIBinaryInputStream", "setInputStream");
var server;
// set up what we need to make storage adapters
const Kinto = loadKinto();
const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
const kintoFilename = "kinto.sqlite";
let kintoClient;
function do_get_kinto_collection() {
if (!kintoClient) {
let config = {
remote:`http://localhost:${server.identity.primaryPort}/v1/`,
headers: {Authorization: "Basic " + btoa("user:pass")},
adapter: FirefoxAdapter
};
kintoClient = new Kinto(config);
}
return kintoClient.collection("test_collection");
}
function* clear_collection() {
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
yield collection.clear();
} finally {
yield collection.db.close();
}
}
// test some operations on a local collection
add_task(function* test_kinto_add_get() {
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
let newRecord = { foo: "bar" };
// check a record is created
let createResult = yield collection.create(newRecord);
do_check_eq(createResult.data.foo, newRecord.foo);
// check getting the record gets the same info
let getResult = yield collection.get(createResult.data.id);
deepEqual(createResult.data, getResult.data);
// check what happens if we create the same item again (it should throw
// since you can't create with id)
try {
yield collection.create(createResult.data);
do_throw("Creation of a record with an id should fail");
} catch (err) { }
// try a few creates without waiting for the first few to resolve
let promises = [];
promises.push(collection.create(newRecord));
promises.push(collection.create(newRecord));
promises.push(collection.create(newRecord));
yield collection.create(newRecord);
yield Promise.all(promises);
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
// test some operations on multiple connections
add_task(function* test_kinto_add_get() {
const collection1 = do_get_kinto_collection();
const collection2 = kintoClient.collection("test_collection_2");
try {
yield collection1.db.open();
yield collection2.db.open();
let newRecord = { foo: "bar" };
// perform several write operations alternately without waiting for promises
// to resolve
let promises = [];
for (let i = 0; i < 10; i++) {
promises.push(collection1.create(newRecord));
promises.push(collection2.create(newRecord));
}
// ensure subsequent operations still work
yield Promise.all([collection1.create(newRecord),
collection2.create(newRecord)]);
yield Promise.all(promises);
} finally {
yield collection1.db.close();
yield collection2.db.close();
}
});
add_task(clear_collection);
add_task(function* test_kinto_update() {
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const newRecord = { foo: "bar" };
// check a record is created
let createResult = yield collection.create(newRecord);
do_check_eq(createResult.data.foo, newRecord.foo);
do_check_eq(createResult.data._status, "created");
// check we can update this OK
let copiedRecord = Object.assign(createResult.data, {});
deepEqual(createResult.data, copiedRecord);
copiedRecord.foo = "wibble";
let updateResult = yield collection.update(copiedRecord);
// check the field was updated
do_check_eq(updateResult.data.foo, copiedRecord.foo);
// check the status is still "created", since we haven't synced
// the record
do_check_eq(updateResult.data._status, "created");
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
add_task(function* test_kinto_clear() {
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
// create an expected number of records
const expected = 10;
const newRecord = { foo: "bar" };
for (let i = 0; i < expected; i++) {
yield collection.create(newRecord);
}
// check the collection contains the correct number
let list = yield collection.list();
do_check_eq(list.data.length, expected);
// clear the collection and check again - should be 0
yield collection.clear();
list = yield collection.list();
do_check_eq(list.data.length, 0);
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
add_task(function* test_kinto_delete(){
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const newRecord = { foo: "bar" };
// check a record is created
let createResult = yield collection.create(newRecord);
do_check_eq(createResult.data.foo, newRecord.foo);
// check getting the record gets the same info
let getResult = yield collection.get(createResult.data.id);
deepEqual(createResult.data, getResult.data);
// delete that record
let deleteResult = yield collection.delete(createResult.data.id);
// check the ID is set on the result
do_check_eq(getResult.data.id, deleteResult.data.id);
// and check that get no longer returns the record
try {
getResult = yield collection.get(createResult.data.id);
do_throw("there should not be a result");
} catch (e) { }
} finally {
yield collection.db.close();
}
});
add_task(function* test_kinto_list(){
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const expected = 10;
const created = [];
for (let i = 0; i < expected; i++) {
let newRecord = { foo: "test " + i };
let createResult = yield collection.create(newRecord);
created.push(createResult.data);
}
// check the collection contains the correct number
let list = yield collection.list();
do_check_eq(list.data.length, expected);
// check that all created records exist in the retrieved list
for (let createdRecord of created) {
let found = false;
for (let retrievedRecord of list.data) {
if (createdRecord.id == retrievedRecord.id) {
deepEqual(createdRecord, retrievedRecord);
found = true;
}
}
do_check_true(found);
}
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
add_task(function* test_loadDump_ignores_already_imported_records(){
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const record = {id: "41b71c13-17e9-4ee3-9268-6a41abf9730f", title: "foo", last_modified: 1457896541};
yield collection.loadDump([record]);
let impactedRecords = yield collection.loadDump([record]);
do_check_eq(impactedRecords.length, 0);
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
add_task(function* test_loadDump_should_overwrite_old_records(){
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const record = {id: "41b71c13-17e9-4ee3-9268-6a41abf9730f", title: "foo", last_modified: 1457896541};
yield collection.loadDump([record]);
const updated = Object.assign({}, record, {last_modified: 1457896543});
let impactedRecords = yield collection.loadDump([updated]);
do_check_eq(impactedRecords.length, 1);
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
add_task(function* test_loadDump_should_not_overwrite_unsynced_records(){
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const recordId = "41b71c13-17e9-4ee3-9268-6a41abf9730f";
yield collection.create({id: recordId, title: "foo"}, {useRecordId: true});
const record = {id: recordId, title: "bar", last_modified: 1457896541};
let impactedRecords = yield collection.loadDump([record]);
do_check_eq(impactedRecords.length, 0);
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
add_task(function* test_loadDump_should_not_overwrite_records_without_last_modified(){
const collection = do_get_kinto_collection();
try {
yield collection.db.open();
const recordId = "41b71c13-17e9-4ee3-9268-6a41abf9730f";
yield collection.create({id: recordId, title: "foo"}, {synced: true});
const record = {id: recordId, title: "bar", last_modified: 1457896541};
let impactedRecords = yield collection.loadDump([record]);
do_check_eq(impactedRecords.length, 0);
} finally {
yield collection.db.close();
}
});
add_task(clear_collection);
// Now do some sanity checks against a server - we're not looking to test
// core kinto.js functionality here (there is excellent test coverage in
// kinto.js), more making sure things are basically working as expected.
add_task(function* test_kinto_sync(){
const configPath = "/v1/";
const recordsPath = "/v1/buckets/default/collections/test_collection/records";
// register a handler
function handleResponse (request, response) {
try {
const sampled = getSampleResponse(request, server.identity.primaryPort);
if (!sampled) {
do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
}
response.setStatusLine(null, sampled.status.status,
sampled.status.statusText);
// send the headers
for (let headerLine of sampled.sampleHeaders) {
let headerElements = headerLine.split(':');
response.setHeader(headerElements[0], headerElements[1].trimLeft());
}
response.setHeader("Date", (new Date()).toUTCString());
response.write(sampled.responseBody);
} catch (e) {
dump(`${e}\n`);
}
}
server.registerPathHandler(configPath, handleResponse);
server.registerPathHandler(recordsPath, handleResponse);
// create an empty collection, sync to populate
const collection = do_get_kinto_collection();
try {
let result;
yield collection.db.open();
result = yield collection.sync();
do_check_true(result.ok);
// our test data has a single record; it should be in the local collection
let list = yield collection.list();
do_check_eq(list.data.length, 1);
// now sync again; we should now have 2 records
result = yield collection.sync();
do_check_true(result.ok);
list = yield collection.list();
do_check_eq(list.data.length, 2);
// sync again; the second records should have been modified
const before = list.data[0].title;
result = yield collection.sync();
do_check_true(result.ok);
list = yield collection.list();
const after = list.data[0].title;
do_check_neq(before, after);
} finally {
yield collection.db.close();
}
});
function run_test() {
// Set up an HTTP Server
server = new HttpServer();
server.start(-1);
run_next_test();
do_register_cleanup(function() {
server.stop(function() { });
});
}
// get a response for a given request from sample data
function getSampleResponse(req, port) {
const responses = {
"OPTIONS": {
"sampleHeaders": [
"Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
"Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
"Access-Control-Allow-Origin: *",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
"status": {status: 200, statusText: "OK"},
"responseBody": "null"
},
"GET:/v1/?": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress"
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"settings":{"batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
},
"GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1445606341071\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{"last_modified":1445606341071, "done":false, "id":"68db8313-686e-4fff-835e-07d78ad6f2af", "title":"New test"}]})
},
"GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified&_since=1445606341071": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1445607941223\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{"last_modified":1445607941223, "done":false, "id":"901967b0-f729-4b30-8d8d-499cba7f4b1d", "title":"Another new test"}]})
},
"GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified&_since=1445607941223": {
"sampleHeaders": [
"Access-Control-Allow-Origin: *",
"Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
"Content-Type: application/json; charset=UTF-8",
"Server: waitress",
"Etag: \"1445607541265\""
],
"status": {status: 200, statusText: "OK"},
"responseBody": JSON.stringify({"data":[{"last_modified":1445607541265, "done":false, "id":"901967b0-f729-4b30-8d8d-499cba7f4b1d", "title":"Modified title"}]})
}
};
return responses[`${req.method}:${req.path}?${req.queryString}`] ||
responses[req.method];
}

8
services/common/tests/unit/xpcshell.ini
View file

@ -9,14 +9,6 @@ support-files =
# Test load modules first so syntax failures are caught early.
[test_load_modules.js]
[test_blocklist_certificates.js]
[test_blocklist_clients.js]
[test_blocklist_updater.js]
[test_kinto.js]
[test_blocklist_signatures.js]
[test_storage_adapter.js]
[test_utils_atob.js]
[test_utils_convert_string.js]
[test_utils_dateprefs.js]

277
services/sync/modules/engines/extension-storage.js
View file

@ -1,277 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine', 'EncryptionRemoteTransformer',
'KeyRingEncryptionRemoteTransformer'];
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://services-crypto/utils.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/keys.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/async.js");
XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
"resource://gre/modules/ExtensionStorageSync.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
"resource://gre/modules/FxAccounts.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Task",
"resource://gre/modules/Task.jsm");
/**
* The Engine that manages syncing for the web extension "storage"
* API, and in particular ext.storage.sync.
*
* ext.storage.sync is implemented using Kinto, so it has mechanisms
* for syncing that we do not need to integrate in the Firefox Sync
* framework, so this is something of a stub.
*/
this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
SyncEngine.call(this, "Extension-Storage", service);
};
ExtensionStorageEngine.prototype = {
__proto__: SyncEngine.prototype,
_trackerObj: ExtensionStorageTracker,
// we don't need these since we implement our own sync logic
_storeObj: undefined,
_recordObj: undefined,
syncPriority: 10,
allowSkippedRecord: false,
_sync: function () {
return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
},
get enabled() {
// By default, we sync extension storage if we sync addons. This
// lets us simplify the UX since users probably don't consider
// "extension preferences" a separate category of syncing.
// However, we also respect engine.extension-storage.force, which
// can be set to true or false, if a power user wants to customize
// the behavior despite the lack of UI.
const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
if (forced !== undefined) {
return forced;
}
return Svc.Prefs.get("engine.addons", false);
},
};
function ExtensionStorageTracker(name, engine) {
Tracker.call(this, name, engine);
}
ExtensionStorageTracker.prototype = {
__proto__: Tracker.prototype,
startTracking: function () {
Svc.Obs.add("ext.storage.sync-changed", this);
},
stopTracking: function () {
Svc.Obs.remove("ext.storage.sync-changed", this);
},
observe: function (subject, topic, data) {
Tracker.prototype.observe.call(this, subject, topic, data);
if (this.ignoreAll) {
return;
}
if (topic !== "ext.storage.sync-changed") {
return;
}
// Single adds, removes and changes are not so important on their
// own, so let's just increment score a bit.
this.score += SCORE_INCREMENT_MEDIUM;
},
// Override a bunch of methods which don't do anything for us.
// This is a performance hack.
saveChangedIDs: function() {
},
loadChangedIDs: function() {
},
ignoreID: function() {
},
unignoreID: function() {
},
addChangedID: function() {
},
removeChangedID: function() {
},
clearChangedIDs: function() {
},
};
/**
* Utility function to enforce an order of fields when computing an HMAC.
*/
function ciphertextHMAC(keyBundle, id, IV, ciphertext) {
const hasher = keyBundle.sha256HMACHasher;
return Utils.bytesAsHex(Utils.digestUTF8(id + IV + ciphertext, hasher));
}
/**
* A "remote transformer" that the Kinto library will use to
* encrypt/decrypt records when syncing.
*
* This is an "abstract base class". Subclass this and override
* getKeys() to use it.
*/
class EncryptionRemoteTransformer {
encode(record) {
const self = this;
return Task.spawn(function* () {
const keyBundle = yield self.getKeys();
if (record.ciphertext) {
throw new Error("Attempt to reencrypt??");
}
let id = record.id;
if (!record.id) {
throw new Error("Record ID is missing or invalid");
}
let IV = Svc.Crypto.generateRandomIV();
let ciphertext = Svc.Crypto.encrypt(JSON.stringify(record),
keyBundle.encryptionKeyB64, IV);
let hmac = ciphertextHMAC(keyBundle, id, IV, ciphertext);
const encryptedResult = {ciphertext, IV, hmac, id};
if (record.hasOwnProperty("last_modified")) {
encryptedResult.last_modified = record.last_modified;
}
return encryptedResult;
});
}
decode(record) {
const self = this;
return Task.spawn(function* () {
if (!record.ciphertext) {
// This can happen for tombstones if a record is deleted.
if (record.deleted) {
return record;
}
throw new Error("No ciphertext: nothing to decrypt?");
}
const keyBundle = yield self.getKeys();
// Authenticate the encrypted blob with the expected HMAC
let computedHMAC = ciphertextHMAC(keyBundle, record.id, record.IV, record.ciphertext);
if (computedHMAC != record.hmac) {
Utils.throwHMACMismatch(record.hmac, computedHMAC);
}
// Handle invalid data here. Elsewhere we assume that cleartext is an object.
let cleartext = Svc.Crypto.decrypt(record.ciphertext,
keyBundle.encryptionKeyB64, record.IV);
let jsonResult = JSON.parse(cleartext);
if (!jsonResult || typeof jsonResult !== "object") {
throw new Error("Decryption failed: result is <" + jsonResult + ">, not an object.");
}
// Verify that the encrypted id matches the requested record's id.
// This should always be true, because we compute the HMAC over
// the original record's ID, and that was verified already (above).
if (jsonResult.id != record.id) {
throw new Error("Record id mismatch: " + jsonResult.id + " != " + record.id);
}
if (record.hasOwnProperty("last_modified")) {
jsonResult.last_modified = record.last_modified;
}
return jsonResult;
});
}
/**
* Retrieve keys to use during encryption.
*
* Returns a Promise<KeyBundle>.
*/
getKeys() {
throw new Error("override getKeys in a subclass");
}
}
// You can inject this
EncryptionRemoteTransformer.prototype._fxaService = fxAccounts;
/**
* An EncryptionRemoteTransformer that provides a keybundle derived
* from the user's kB, suitable for encrypting a keyring.
*/
class KeyRingEncryptionRemoteTransformer extends EncryptionRemoteTransformer {
getKeys() {
const self = this;
return Task.spawn(function* () {
const user = yield self._fxaService.getSignedInUser();
// FIXME: we should permit this if the user is self-hosting
// their storage
if (!user) {
throw new Error("user isn't signed in to FxA; can't sync");
}
if (!user.kB) {
throw new Error("user doesn't have kB");
}
let kB = Utils.hexToBytes(user.kB);
let keyMaterial = CryptoUtils.hkdf(kB, undefined,
"identity.mozilla.com/picl/v1/chrome.storage.sync", 2*32);
let bundle = new BulkKeyBundle();
// [encryptionKey, hmacKey]
bundle.keyPair = [keyMaterial.slice(0, 32), keyMaterial.slice(32, 64)];
return bundle;
});
}
// Pass through the kbHash field from the unencrypted record. If
// encryption fails, we can use this to try to detect whether we are
// being compromised or if the record here was encoded with a
// different kB.
encode(record) {
const encodePromise = super.encode(record);
return Task.spawn(function* () {
const encoded = yield encodePromise;
encoded.kbHash = record.kbHash;
return encoded;
});
}
decode(record) {
const decodePromise = super.decode(record);
return Task.spawn(function* () {
try {
return yield decodePromise;
} catch (e) {
if (Utils.isHMACMismatch(e)) {
const currentKBHash = yield ExtensionStorageSync.getKBHash();
if (record.kbHash != currentKBHash) {
// Some other client encoded this with a kB that we don't
// have access to.
KeyRingEncryptionRemoteTransformer.throwOutdatedKB(currentKBHash, record.kbHash);
}
}
throw e;
}
});
}
// Generator and discriminator for KB-is-outdated exceptions.
static throwOutdatedKB(shouldBe, is) {
throw new Error(`kB hash on record is outdated: should be ${shouldBe}, is ${is}`);
}
static isOutdatedKB(exc) {
const kbMessage = "kB hash on record is outdated: ";
return exc && exc.message && exc.message.indexOf &&
(exc.message.indexOf(kbMessage) == 0);
}
}

1
services/sync/modules/service.js
View file

@ -44,7 +44,6 @@ const ENGINE_MODULES = {
Password: "passwords.js",
Prefs: "prefs.js",
Tab: "tabs.js",
ExtensionStorage: "extension-storage.js",
};
const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,

1
services/sync/moz.build
View file

@ -52,7 +52,6 @@ EXTRA_JS_MODULES['services-sync'].engines += [
'modules/engines/addons.js',
'modules/engines/bookmarks.js',
'modules/engines/clients.js',
'modules/engines/extension-storage.js',
'modules/engines/forms.js',
'modules/engines/history.js',
'modules/engines/passwords.js',