import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

toolkit/components/telemetry/tests/addons/dictionary/install.rdf

@@ -0,0 +1,25 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>telemetry-dictionary@tests.mozilla.org</em:id>
+    <em:version>1</em:version>
+    <em:type>64</em:type>
+
+    <em:targetApplication>
+      <Description>
+        <em:id>toolkit@mozilla.org</em:id>
+        <em:minVersion>0</em:minVersion>
+        <em:maxVersion>*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+
+    <!-- Front End MetaData -->
+    <em:name>Telemetry test dictionary</em:name>
+    <em:description>A nice dictionary to prevent all typos for Telemetry.</em:description>
+    <em:bootstrap>true</em:bootstrap>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/experiment/install.rdf

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>telemetry-experiment-1@tests.mozilla.org</em:id>
+    <em:version>1</em:version>
+    <em:type>128</em:type>
+
+    <!-- Front End MetaData -->
+    <em:name>Telemetry test experiment</em:name>
+    <em:description>Yet another experiment that experiments experimentally.</em:description>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/extension-2/install.rdf

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>telemetry-ext-2@tests.mozilla.org</em:id>
+    <em:version>2</em:version>
+    <em:type>2</em:type>
+
+    <!-- Front End MetaData -->
+    <em:name>Telemetry test extension 2</em:name>
+    <em:description>Yet another extension that extends twice.</em:description>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/extension/install.rdf

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>telemetry-ext-1@tests.mozilla.org</em:id>
+    <em:version>1</em:version>
+    <em:type>2</em:type>
+
+    <!-- Front End MetaData -->
+    <em:name>Telemetry test extension</em:name>
+    <em:description>Yet another extension that extends.</em:description>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/long-fields/install.rdf

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>tel-longfields-xpi@tests.mozilla.org</em:id>
+    <em:version>This is a really long addon version, that will get limited to 100 characters. We're much longer, we're at about 116.</em:version>
+
+    <em:targetApplication>
+      <Description>
+        <em:id>toolkit@mozilla.org</em:id>
+        <em:minVersion>0</em:minVersion>
+        <em:maxVersion>*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+
+    <!-- Front End MetaData -->
+    <em:name>This is a really long addon name, that will get limited to 100 characters. We're much longer, we're at about 219. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus nullam sodales. Yeah, Latin placeholder.</em:name>
+    <em:description>This is a really long addon description, that will get limited to 100 characters. We're much longer, we're at about 200. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vivamus nullam sodales.</em:description>
+    <em:bootstrap>true</em:bootstrap>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/restartless/install.rdf

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>tel-restartless-xpi@tests.mozilla.org</em:id>
+    <em:version>1.0</em:version>
+
+    <em:targetApplication>
+      <Description>
+        <em:id>toolkit@mozilla.org</em:id>
+        <em:minVersion>0</em:minVersion>
+        <em:maxVersion>*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+
+    <!-- Front End MetaData -->
+    <em:name>XPI Telemetry Restartless Test</em:name>
+    <em:description>A restartless addon which gets enabled without a reboot.</em:description>
+    <em:bootstrap>true</em:bootstrap>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/signed/META-INF/manifest.mf

@@ -0,0 +1,7 @@
+Manifest-Version: 1.0
+
+Name: install.rdf
+Digest-Algorithms: MD5 SHA1
+MD5-Digest: YEilRfaecTg2bMNPoYqexQ==
+SHA1-Digest: GEnQKM8Coyw83phx/z1oNh327+0=
+

toolkit/components/telemetry/tests/addons/signed/META-INF/mozilla.sf

@@ -0,0 +1,4 @@
+Signature-Version: 1.0
+MD5-Digest-Manifest: Ko2bKTrwTXCdstWHWqCR4w==
+SHA1-Digest-Manifest: k6+jfNGFxXtDd1cSX0ZoIyQ1cww=
+

toolkit/components/telemetry/tests/addons/signed/install.rdf

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>tel-signed-xpi@tests.mozilla.org</em:id>
+    <em:version>1.0</em:version>
+
+    <em:targetApplication>
+      <Description>
+        <em:id>toolkit@mozilla.org</em:id>
+        <em:minVersion>0</em:minVersion>
+        <em:maxVersion>*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+
+    <!-- Front End MetaData -->
+    <em:name>XPI Telemetry Signed Test</em:name>
+    <em:description>A signed addon which gets enabled without a reboot.</em:description>
+    <em:bootstrap>true</em:bootstrap>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/system/install.rdf

@@ -0,0 +1,24 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>tel-system-xpi@tests.mozilla.org</em:id>
+    <em:version>1.0</em:version>
+
+    <em:targetApplication>
+      <Description>
+        <em:id>toolkit@mozilla.org</em:id>
+        <em:minVersion>0</em:minVersion>
+        <em:maxVersion>*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+
+    <!-- Front End MetaData -->
+    <em:name>XPI Telemetry System Add-on Test</em:name>
+    <em:description>A system addon which is shipped with Firefox.</em:description>
+    <em:bootstrap>true</em:bootstrap>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/addons/theme/install.rdf

@@ -0,0 +1,16 @@
+<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>telemetry-theme@tests.mozilla.org</em:id>
+    <em:version>1</em:version>
+    <em:type>4</em:type>
+
+    <!-- Front End MetaData -->
+    <em:name>Telemetry test theme</em:name>
+    <em:description>A good looking test theme for Telemetry.</em:description>
+
+  </Description>
+</RDF>

toolkit/components/telemetry/tests/browser/browser.ini

@@ -0,0 +1,5 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+[browser_TelemetryGC.js]

toolkit/components/telemetry/tests/browser/browser_TelemetryGC.js

@@ -0,0 +1,193 @@
+"use strict";
+
+/*
+ *********************************************************************************
+ *                                                                               *
+ *                                  WARNING                                      *
+ *                                                                               *
+ * If you adjust any of the constants here (slice limit, number of keys, etc.)   *
+ * make sure to update the JSON schema at:                                       *
+ * https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/     *
+ * telemetry/main.schema.json                                                    *
+ *                                                                               *
+ * Otherwise, pings may be dropped by the telemetry backend!                     *
+ *                                                                               *
+ ********************************************************************************/
+
+const {GCTelemetry} = Cu.import("resource://gre/modules/GCTelemetry.jsm", {});
+
+function check(entries) {
+  const FIELDS = ["random", "worst"];
+
+  // Check that all FIELDS are in |entries|.
+  for (let f of FIELDS) {
+    ok(f in entries, `${f} found in entries`);
+  }
+
+  // Check that only FIELDS are in |entries|.
+  for (let k of Object.keys(entries)) {
+    ok(FIELDS.includes(k), `${k} found in FIELDS`);
+  }
+
+  let foundGCs = 0;
+
+  for (let f of FIELDS) {
+    ok(Array.isArray(entries[f]), "have an array of GCs");
+
+    ok(entries[f].length <= 2, "not too many GCs");
+
+    for (let gc of entries[f]) {
+      ok(gc !== null, "GC is non-null");
+
+      foundGCs++;
+
+      ok(Object.keys(gc).length <= 25, "number of keys in GC is not too large");
+
+      // Sanity check the GC data.
+      ok("total_time" in gc, "total_time field present");
+      ok("max_pause" in gc, "max_pause field present");
+
+      ok("slices" in gc, "slices field present");
+      ok(Array.isArray(gc.slices), "slices is an array");
+      ok(gc.slices.length > 0, "slices array non-empty");
+      ok(gc.slices.length <= 4, "slices array is not too long");
+
+      ok("totals" in gc, "totals field present");
+      ok(typeof(gc.totals) == "object", "totals is an object");
+      ok(Object.keys(gc.totals).length <= 65, "totals array is not too long");
+
+      // Make sure we don't skip any big objects.
+      for (let key in gc) {
+        if (key != "slices" && key != "totals") {
+          ok(typeof(gc[key]) != "object", `${key} property should be primitive`);
+        }
+      }
+
+      let phases = new Set();
+
+      for (let slice of gc.slices) {
+        ok(Object.keys(slice).length <= 15, "slice is not too large");
+
+        ok("pause" in slice, "pause field present in slice");
+        ok("reason" in slice, "reason field present in slice");
+        ok("times" in slice, "times field present in slice");
+
+        // Make sure we don't skip any big objects.
+        for (let key in slice) {
+          if (key != "times") {
+            ok(typeof(slice[key]) != "object", `${key} property should be primitive`);
+          }
+        }
+
+        ok(Object.keys(slice.times).length <= 65, "no more than 65 phases");
+
+        for (let phase in slice.times) {
+          phases.add(phase);
+          ok(typeof(slice.times[phase]) == "number", `${phase} property should be a number`);
+        }
+      }
+
+      let totals = gc.totals;
+      // Make sure we don't skip any big objects.
+      for (let phase in totals) {
+        ok(typeof(totals[phase]) == "number", `${phase} property should be a number`);
+      }
+
+      for (let phase of phases) {
+        ok(phase in totals, `${phase} is in totals`);
+      }
+    }
+  }
+
+  ok(foundGCs > 0, "saw at least one GC");
+}
+
+add_task(function* test() {
+  let multiprocess = Services.appinfo.browserTabsRemoteAutostart;
+
+  // Set these prefs to ensure that we get measurements.
+  const prefs = {"set": [["javascript.options.mem.notify", true]]};
+  yield new Promise(resolve => SpecialPowers.pushPrefEnv(prefs, resolve));
+
+  function runRemote(f) {
+    gBrowser.selectedBrowser.messageManager.loadFrameScript(`data:,(${f})()`, false);
+  }
+
+  function initScript() {
+    const {GCTelemetry} = Components.utils.import("resource://gre/modules/GCTelemetry.jsm", {});
+
+    /*
+     * Don't shut down GC telemetry if it was already running before the test!
+     * Note: We need to use a multiline comment here since this code is turned into a data: URI.
+     */
+    let shutdown = GCTelemetry.init();
+
+    function listener() {
+      removeMessageListener("GCTelemTest:Shutdown", listener);
+      if (shutdown) {
+        GCTelemetry.shutdown();
+      }
+    }
+    addMessageListener("GCTelemTest:Shutdown", listener);
+  }
+
+  if (multiprocess) {
+    runRemote(initScript);
+  }
+
+  // Don't shut down GC telemetry if it was already running before the test!
+  let shutdown = GCTelemetry.init();
+  registerCleanupFunction(() => {
+    if (shutdown) {
+      GCTelemetry.shutdown();
+    }
+
+    gBrowser.selectedBrowser.messageManager.sendAsyncMessage("GCTelemTest:Shutdown");
+  });
+
+  let localPromise = new Promise(resolve => {
+    function obs() {
+      Services.obs.removeObserver(obs, "garbage-collection-statistics");
+      resolve();
+    }
+    Services.obs.addObserver(obs, "garbage-collection-statistics", false);
+  });
+
+  let remotePromise;
+  if (multiprocess) {
+    remotePromise = new Promise(resolve => {
+      function obs() {
+        Services.ppmm.removeMessageListener("Telemetry:GCStatistics", obs);
+        resolve();
+      }
+      Services.ppmm.addMessageListener("Telemetry:GCStatistics", obs);
+    });
+  } else {
+    remotePromise = Promise.resolve();
+  }
+
+  // Make sure we have a GC to work with in both processes.
+  Cu.forceGC();
+  if (multiprocess) {
+    runRemote(() => Components.utils.forceGC());
+  }
+
+  info("Waiting for GCs");
+
+  yield Promise.all([localPromise, remotePromise]);
+
+  let localEntries = GCTelemetry.entries("main", true);
+  let remoteEntries = multiprocess ? GCTelemetry.entries("content", true) : localEntries;
+
+  check(localEntries);
+  check(remoteEntries);
+
+  localEntries = GCTelemetry.entries("main", false);
+  remoteEntries = multiprocess ? GCTelemetry.entries("content", false) : localEntries;
+
+  is(localEntries.random.length, 0, "no random GCs after reset");
+  is(localEntries.worst.length, 0, "no worst GCs after reset");
+
+  is(remoteEntries.random.length, 0, "no random GCs after reset");
+  is(remoteEntries.worst.length, 0, "no worst GCs after reset");
+});

toolkit/components/telemetry/tests/search/chrome.manifest

@@ -0,0 +1,3 @@
+locale testsearchplugin ar jar:jar:searchTest.jar!/chrome/searchTest.jar!/
+content testsearchplugin ./
+

toolkit/components/telemetry/tests/unit/.eslintrc.js

@@ -0,0 +1,7 @@
+"use strict";
+
+module.exports = {
+  "extends": [
+    "../../../../../testing/xpcshell/xpcshell.eslintrc.js"
+  ]
+};

toolkit/components/telemetry/tests/unit/TelemetryArchiveTesting.jsm

@@ -0,0 +1,86 @@
+const {utils: Cu} = Components;
+Cu.import("resource://gre/modules/TelemetryArchive.jsm");
+Cu.import("resource://testing-common/Assert.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/TelemetryController.jsm");
+
+this.EXPORTED_SYMBOLS = [
+  "TelemetryArchiveTesting",
+];
+
+function checkForProperties(ping, expected) {
+  for (let [props, val] of expected) {
+    let test = ping;
+    for (let prop of props) {
+      test = test[prop];
+      if (test === undefined) {
+        return false;
+      }
+    }
+    if (test !== val) {
+      return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * A helper object that allows test code to check whether a telemetry ping
+ * was properly saved. To use, first initialize to collect the starting pings
+ * and then check for new ping data.
+ */
+function Checker() {
+}
+Checker.prototype = {
+  promiseInit: function() {
+    this._pingMap = new Map();
+    return TelemetryArchive.promiseArchivedPingList().then((plist) => {
+      for (let ping of plist) {
+        this._pingMap.set(ping.id, ping);
+      }
+    });
+  },
+
+  /**
+   * Find and return a new ping with certain properties.
+   *
+   * @param expected: an array of [['prop'...], 'value'] to check
+   * For example:
+   * [
+   *   [['environment', 'build', 'applicationId'], '20150101010101'],
+   *   [['version'], 1],
+   *   [['metadata', 'OOMAllocationSize'], 123456789],
+   * ]
+   * @returns a matching ping if found, or null
+   */
+  promiseFindPing: Task.async(function*(type, expected) {
+    let candidates = [];
+    let plist = yield TelemetryArchive.promiseArchivedPingList();
+    for (let ping of plist) {
+      if (this._pingMap.has(ping.id)) {
+        continue;
+      }
+      if (ping.type == type) {
+        candidates.push(ping);
+      }
+    }
+
+    for (let candidate of candidates) {
+      let ping = yield TelemetryArchive.promiseArchivedPingById(candidate.id);
+      if (checkForProperties(ping, expected)) {
+        return ping;
+      }
+    }
+    return null;
+  }),
+};
+
+const TelemetryArchiveTesting = {
+  setup: function() {
+    Services.prefs.setCharPref("toolkit.telemetry.log.level", "Trace");
+    Services.prefs.setBoolPref("toolkit.telemetry.archive.enabled", true);
+  },
+
+  Checker: Checker,
+};

toolkit/components/telemetry/tests/unit/engine.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/">
+<ShortName>engine-telemetry</ShortName>
+<Url type="text/html" method="GET" template="http://www.example.com/search">
+  <Param name="q" value="{searchTerms}"/>
+</Url>
+</SearchPlugin>

toolkit/components/telemetry/tests/unit/head.js

@@ -0,0 +1,319 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var { classes: Cc, utils: Cu, interfaces: Ci, results: Cr } = Components;
+
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/PromiseUtils.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/FileUtils.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://testing-common/httpd.js", this);
+Cu.import("resource://gre/modules/AppConstants.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "AddonTestUtils",
+                                  "resource://testing-common/AddonTestUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "OS",
+                                  "resource://gre/modules/osfile.jsm");
+
+const gIsWindows = AppConstants.platform == "win";
+const gIsMac = AppConstants.platform == "macosx";
+const gIsAndroid = AppConstants.platform == "android";
+const gIsGonk = AppConstants.platform == "gonk";
+const gIsLinux = AppConstants.platform == "linux";
+
+const Telemetry = Cc["@mozilla.org/base/telemetry;1"].getService(Ci.nsITelemetry);
+
+const MILLISECONDS_PER_MINUTE = 60 * 1000;
+const MILLISECONDS_PER_HOUR = 60 * MILLISECONDS_PER_MINUTE;
+const MILLISECONDS_PER_DAY = 24 * MILLISECONDS_PER_HOUR;
+
+const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled";
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+var gGlobalScope = this;
+
+const PingServer = {
+  _httpServer: null,
+  _started: false,
+  _defers: [ PromiseUtils.defer() ],
+  _currentDeferred: 0,
+
+  get port() {
+    return this._httpServer.identity.primaryPort;
+  },
+
+  get started() {
+    return this._started;
+  },
+
+  registerPingHandler: function(handler) {
+    const wrapped = wrapWithExceptionHandler(handler);
+    this._httpServer.registerPrefixHandler("/submit/telemetry/", wrapped);
+  },
+
+  resetPingHandler: function() {
+    this.registerPingHandler((request, response) => {
+      let deferred = this._defers[this._defers.length - 1];
+      this._defers.push(PromiseUtils.defer());
+      deferred.resolve(request);
+    });
+  },
+
+  start: function() {
+    this._httpServer = new HttpServer();
+    this._httpServer.start(-1);
+    this._started = true;
+    this.clearRequests();
+    this.resetPingHandler();
+  },
+
+  stop: function() {
+    return new Promise(resolve => {
+      this._httpServer.stop(resolve);
+      this._started = false;
+    });
+  },
+
+  clearRequests: function() {
+    this._defers = [ PromiseUtils.defer() ];
+    this._currentDeferred = 0;
+  },
+
+  promiseNextRequest: function() {
+    const deferred = this._defers[this._currentDeferred++];
+    // Send the ping to the consumer on the next tick, so that the completion gets
+    // signaled to Telemetry.
+    return new Promise(r => Services.tm.currentThread.dispatch(() => r(deferred.promise),
+                                                               Ci.nsIThread.DISPATCH_NORMAL));
+  },
+
+  promiseNextPing: function() {
+    return this.promiseNextRequest().then(request => decodeRequestPayload(request));
+  },
+
+  promiseNextRequests: Task.async(function*(count) {
+    let results = [];
+    for (let i=0; i<count; ++i) {
+      results.push(yield this.promiseNextRequest());
+    }
+
+    return results;
+  }),
+
+  promiseNextPings: function(count) {
+    return this.promiseNextRequests(count).then(requests => {
+      return Array.from(requests, decodeRequestPayload);
+    });
+  },
+};
+
+/**
+ * Decode the payload of an HTTP request into a ping.
+ * @param {Object} request The data representing an HTTP request (nsIHttpRequest).
+ * @return {Object} The decoded ping payload.
+ */
+function decodeRequestPayload(request) {
+  let s = request.bodyInputStream;
+  let payload = null;
+  let decoder = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON)
+
+  if (request.getHeader("content-encoding") == "gzip") {
+    let observer = {
+      buffer: "",
+      onStreamComplete: function(loader, context, status, length, result) {
+        this.buffer = String.fromCharCode.apply(this, result);
+      }
+    };
+
+    let scs = Cc["@mozilla.org/streamConverters;1"]
+              .getService(Ci.nsIStreamConverterService);
+    let listener = Cc["@mozilla.org/network/stream-loader;1"]
+                  .createInstance(Ci.nsIStreamLoader);
+    listener.init(observer);
+    let converter = scs.asyncConvertData("gzip", "uncompressed",
+                                         listener, null);
+    converter.onStartRequest(null, null);
+    converter.onDataAvailable(null, null, s, 0, s.available());
+    converter.onStopRequest(null, null, null);
+    let unicodeConverter = Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                    .createInstance(Ci.nsIScriptableUnicodeConverter);
+    unicodeConverter.charset = "UTF-8";
+    let utf8string = unicodeConverter.ConvertToUnicode(observer.buffer);
+    utf8string += unicodeConverter.Finish();
+    payload = JSON.parse(utf8string);
+  } else {
+    payload = decoder.decodeFromStream(s, s.available());
+  }
+
+  return payload;
+}
+
+function wrapWithExceptionHandler(f) {
+  function wrapper(...args) {
+    try {
+      f(...args);
+    } catch (ex) {
+      if (typeof(ex) != 'object') {
+        throw ex;
+      }
+      dump("Caught exception: " + ex.message + "\n");
+      dump(ex.stack);
+      do_test_finished();
+    }
+  }
+  return wrapper;
+}
+
+function loadAddonManager(...args) {
+  AddonTestUtils.init(gGlobalScope);
+  AddonTestUtils.overrideCertDB();
+  createAppInfo(...args);
+
+  // As we're not running in application, we need to setup the features directory
+  // used by system add-ons.
+  const distroDir = FileUtils.getDir("ProfD", ["sysfeatures", "app0"], true);
+  AddonTestUtils.registerDirectory("XREAppFeat", distroDir);
+  return AddonTestUtils.promiseStartupManager();
+}
+
+var gAppInfo = null;
+
+function createAppInfo(ID="xpcshell@tests.mozilla.org", name="XPCShell",
+                       version="1.0", platformVersion="1.0") {
+  AddonTestUtils.createAppInfo(ID, name, version, platformVersion);
+  gAppInfo = AddonTestUtils.appInfo;
+}
+
+// Fake the timeout functions for the TelemetryScheduler.
+function fakeSchedulerTimer(set, clear) {
+  let session = Cu.import("resource://gre/modules/TelemetrySession.jsm");
+  session.Policy.setSchedulerTickTimeout = set;
+  session.Policy.clearSchedulerTickTimeout = clear;
+}
+
+/**
+ * Fake the current date.
+ * This passes all received arguments to a new Date constructor and
+ * uses the resulting date to fake the time in Telemetry modules.
+ *
+ * @return Date The new faked date.
+ */
+function fakeNow(...args) {
+  const date = new Date(...args);
+  const modules = [
+    Cu.import("resource://gre/modules/TelemetrySession.jsm"),
+    Cu.import("resource://gre/modules/TelemetryEnvironment.jsm"),
+    Cu.import("resource://gre/modules/TelemetryController.jsm"),
+    Cu.import("resource://gre/modules/TelemetryStorage.jsm"),
+    Cu.import("resource://gre/modules/TelemetrySend.jsm"),
+    Cu.import("resource://gre/modules/TelemetryReportingPolicy.jsm"),
+  ];
+
+  for (let m of modules) {
+    m.Policy.now = () => date;
+  }
+
+  return new Date(date);
+}
+
+function fakeMonotonicNow(ms) {
+  const m = Cu.import("resource://gre/modules/TelemetrySession.jsm");
+  m.Policy.monotonicNow = () => ms;
+  return ms;
+}
+
+// Fake the timeout functions for TelemetryController sending.
+function fakePingSendTimer(set, clear) {
+  let module = Cu.import("resource://gre/modules/TelemetrySend.jsm");
+  let obj = Cu.cloneInto({set, clear}, module, {cloneFunctions:true});
+  module.Policy.setSchedulerTickTimeout = obj.set;
+  module.Policy.clearSchedulerTickTimeout = obj.clear;
+}
+
+function fakeMidnightPingFuzzingDelay(delayMs) {
+  let module = Cu.import("resource://gre/modules/TelemetrySend.jsm");
+  module.Policy.midnightPingFuzzingDelay = () => delayMs;
+}
+
+function fakeGeneratePingId(func) {
+  let module = Cu.import("resource://gre/modules/TelemetryController.jsm");
+  module.Policy.generatePingId = func;
+}
+
+function fakeCachedClientId(uuid) {
+  let module = Cu.import("resource://gre/modules/TelemetryController.jsm");
+  module.Policy.getCachedClientID = () => uuid;
+}
+
+// Return a date that is |offset| ms in the future from |date|.
+function futureDate(date, offset) {
+  return new Date(date.getTime() + offset);
+}
+
+function truncateToDays(aMsec) {
+  return Math.floor(aMsec / MILLISECONDS_PER_DAY);
+}
+
+// Returns a promise that resolves to true when the passed promise rejects,
+// false otherwise.
+function promiseRejects(promise) {
+  return promise.then(() => false, () => true);
+}
+
+// Generates a random string of at least a specific length.
+function generateRandomString(length) {
+  let string = "";
+
+  while (string.length < length) {
+    string += Math.random().toString(36);
+  }
+
+  return string.substring(0, length);
+}
+
+// Short-hand for retrieving the histogram with that id.
+function getHistogram(histogramId) {
+  return Telemetry.getHistogramById(histogramId);
+}
+
+// Short-hand for retrieving the snapshot of the Histogram with that id.
+function getSnapshot(histogramId) {
+  return Telemetry.getHistogramById(histogramId).snapshot();
+}
+
+// Helper for setting an empty list of Environment preferences to watch.
+function setEmptyPrefWatchlist() {
+  let TelemetryEnvironment =
+    Cu.import("resource://gre/modules/TelemetryEnvironment.jsm").TelemetryEnvironment;
+  return TelemetryEnvironment.onInitialized().then(() => {
+    TelemetryEnvironment.testWatchPreferences(new Map());
+  });
+}
+
+if (runningInParent) {
+  // Set logging preferences for all the tests.
+  Services.prefs.setCharPref("toolkit.telemetry.log.level", "Trace");
+  // Telemetry archiving should be on.
+  Services.prefs.setBoolPref("toolkit.telemetry.archive.enabled", true);
+  // Telemetry xpcshell tests cannot show the infobar.
+  Services.prefs.setBoolPref("datareporting.policy.dataSubmissionPolicyBypassNotification", true);
+  // FHR uploads should be enabled.
+  Services.prefs.setBoolPref("datareporting.healthreport.uploadEnabled", true);
+
+  fakePingSendTimer((callback, timeout) => {
+    Services.tm.mainThread.dispatch(() => callback(), Ci.nsIThread.DISPATCH_NORMAL);
+  },
+  () => {});
+
+  do_register_cleanup(() => TelemetrySend.shutdown());
+}
+
+TelemetryController.testInitLogging();
+
+// Avoid timers interrupting test behavior.
+fakeSchedulerTimer(() => {}, () => {});
+// Make pind sending predictable.
+fakeMidnightPingFuzzingDelay(0);

toolkit/components/telemetry/tests/unit/test_ChildHistograms.js

@@ -0,0 +1,107 @@
+
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
+Cu.import("resource://gre/modules/PromiseUtils.jsm", this);
+Cu.import("resource://testing-common/ContentTaskUtils.jsm", this);
+
+const MESSAGE_TELEMETRY_PAYLOAD = "Telemetry:Payload";
+const MESSAGE_TELEMETRY_GET_CHILD_PAYLOAD = "Telemetry:GetChildPayload";
+const MESSAGE_CHILD_TEST_DONE = "ChildTest:Done";
+
+const PLATFORM_VERSION = "1.9.2";
+const APP_VERSION = "1";
+const APP_ID = "xpcshell@tests.mozilla.org";
+const APP_NAME = "XPCShell";
+
+function run_child_test() {
+  // Setup histograms with some fixed values.
+  let flagHist = Telemetry.getHistogramById("TELEMETRY_TEST_FLAG");
+  flagHist.add(1);
+  let countHist = Telemetry.getHistogramById("TELEMETRY_TEST_COUNT");
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_COUNT", false);
+  countHist.add();
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_COUNT", true);
+  countHist.add();
+  countHist.add();
+  let categHist = Telemetry.getHistogramById("TELEMETRY_TEST_CATEGORICAL");
+  categHist.add("Label2");
+  categHist.add("Label3");
+
+  let flagKeyed = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_FLAG");
+  flagKeyed.add("a", 1);
+  flagKeyed.add("b", 1);
+  let countKeyed = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_COUNT");
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_KEYED_COUNT", false);
+  countKeyed.add("a");
+  countKeyed.add("b");
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_KEYED_COUNT", true);
+  countKeyed.add("a");
+  countKeyed.add("b");
+  countKeyed.add("b");
+}
+
+function check_histogram_values(payload) {
+  const hs = payload.histograms;
+  Assert.ok("TELEMETRY_TEST_COUNT" in hs, "Should have count test histogram.");
+  Assert.ok("TELEMETRY_TEST_FLAG" in hs, "Should have flag test histogram.");
+  Assert.ok("TELEMETRY_TEST_CATEGORICAL" in hs, "Should have categorical test histogram.");
+  Assert.equal(hs["TELEMETRY_TEST_COUNT"].sum, 2,
+               "Count test histogram should have the right value.");
+  Assert.equal(hs["TELEMETRY_TEST_FLAG"].sum, 1,
+               "Flag test histogram should have the right value.");
+  Assert.equal(hs["TELEMETRY_TEST_CATEGORICAL"].sum, 3,
+               "Categorical test histogram should have the right sum.");
+
+  const kh = payload.keyedHistograms;
+  Assert.ok("TELEMETRY_TEST_KEYED_COUNT" in kh, "Should have keyed count test histogram.");
+  Assert.ok("TELEMETRY_TEST_KEYED_FLAG" in kh, "Should have keyed flag test histogram.");
+  Assert.equal(kh["TELEMETRY_TEST_KEYED_COUNT"]["a"].sum, 1,
+               "Keyed count test histogram should have the right value.");
+  Assert.equal(kh["TELEMETRY_TEST_KEYED_COUNT"]["b"].sum, 2,
+               "Keyed count test histogram should have the right value.");
+  Assert.equal(kh["TELEMETRY_TEST_KEYED_FLAG"]["a"].sum, 1,
+               "Keyed flag test histogram should have the right value.");
+  Assert.equal(kh["TELEMETRY_TEST_KEYED_FLAG"]["b"].sum, 1,
+               "Keyed flag test histogram should have the right value.");
+}
+
+add_task(function*() {
+  if (!runningInParent) {
+    TelemetryController.testSetupContent();
+    run_child_test();
+    dump("... done with child test\n");
+    do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
+    return;
+  }
+
+  // Setup.
+  do_get_profile(true);
+  loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+  yield TelemetryController.testSetup();
+  if (runningInParent) {
+    // Make sure we don't generate unexpected pings due to pref changes.
+    yield setEmptyPrefWatchlist();
+  }
+
+  // Run test in child, don't wait for it to finish.
+  run_test_in_child("test_ChildHistograms.js");
+  yield do_await_remote_message(MESSAGE_CHILD_TEST_DONE);
+
+  yield ContentTaskUtils.waitForCondition(() => {
+    let payload = TelemetrySession.getPayload("test-ping");
+    return payload &&
+           "processes" in payload &&
+           "content" in payload.processes &&
+           "histograms" in payload.processes.content &&
+           "TELEMETRY_TEST_COUNT" in payload.processes.content.histograms;
+  });
+  const payload = TelemetrySession.getPayload("test-ping");
+  Assert.ok("processes" in payload, "Should have processes section");
+  Assert.ok("content" in payload.processes, "Should have child process section");
+  Assert.ok("histograms" in payload.processes.content, "Child process section should have histograms.");
+  Assert.ok("keyedHistograms" in payload.processes.content, "Child process section should have keyed histograms.");
+  check_histogram_values(payload.processes.content);
+
+  do_test_finished();
+});

toolkit/components/telemetry/tests/unit/test_PingAPI.js

@@ -0,0 +1,502 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+// This tests the public Telemetry API for submitting pings.
+
+"use strict";
+
+Cu.import("resource://gre/modules/ClientID.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://gre/modules/osfile.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+
+XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() {
+  return OS.Path.join(OS.Constants.Path.profileDir, "datareporting", "archived");
+});
+
+/**
+ * Fakes the archive storage quota.
+ * @param {Integer} aArchiveQuota The new quota, in bytes.
+ */
+function fakeStorageQuota(aArchiveQuota) {
+  let storage = Cu.import("resource://gre/modules/TelemetryStorage.jsm");
+  storage.Policy.getArchiveQuota = () => aArchiveQuota;
+}
+
+/**
+ * Lists all the valid archived pings and their metadata, sorted by creation date.
+ *
+ * @param aFileName {String} The filename.
+ * @return {Object[]} A list of objects with the extracted data in the form:
+ *                  { timestamp: <number>,
+ *                    id: <string>,
+ *                    type: <string>,
+ *                    size: <integer> }
+ */
+var getArchivedPingsInfo = Task.async(function*() {
+  let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath);
+  let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir);
+  let archivedPings = [];
+
+  // Iterate through the subdirs of |gPingsArchivePath|.
+  for (let dir of subdirs) {
+    let fileIterator = new OS.File.DirectoryIterator(dir.path);
+    let files = (yield fileIterator.nextBatch()).filter(e => !e.isDir);
+
+    // Then get a list of the files for the current subdir.
+    for (let f of files) {
+      let pingInfo = TelemetryStorage._testGetArchivedPingDataFromFileName(f.name);
+      if (!pingInfo) {
+        // This is not a valid archived ping, skip it.
+        continue;
+      }
+      // Find the size of the ping and then add the info to the array.
+      pingInfo.size = (yield OS.File.stat(f.path)).size;
+      archivedPings.push(pingInfo);
+    }
+  }
+
+  // Sort the list by creation date and then return it.
+  archivedPings.sort((a, b) => b.timestamp - a.timestamp);
+  return archivedPings;
+});
+
+add_task(function* test_setup() {
+  do_get_profile(true);
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+});
+
+add_task(function* test_archivedPings() {
+  // TelemetryController should not be fully initialized at this point.
+  // Submitting pings should still work fine.
+
+  const PINGS = [
+    {
+      type: "test-ping-api-1",
+      payload: { foo: "bar"},
+      dateCreated: new Date(2010, 1, 1, 10, 0, 0),
+    },
+    {
+      type: "test-ping-api-2",
+      payload: { moo: "meh"},
+      dateCreated: new Date(2010, 2, 1, 10, 0, 0),
+    },
+  ];
+
+  // Submit pings and check the ping list.
+  let expectedPingList = [];
+
+  for (let data of PINGS) {
+    fakeNow(data.dateCreated);
+    data.id = yield TelemetryController.submitExternalPing(data.type, data.payload);
+    let list = yield TelemetryArchive.promiseArchivedPingList();
+
+    expectedPingList.push({
+      id: data.id,
+      type: data.type,
+      timestampCreated: data.dateCreated.getTime(),
+    });
+    Assert.deepEqual(list, expectedPingList, "Archived ping list should contain submitted pings");
+  }
+
+  // Check loading the archived pings.
+  let checkLoadingPings = Task.async(function*() {
+    for (let data of PINGS) {
+      let ping = yield TelemetryArchive.promiseArchivedPingById(data.id);
+      Assert.equal(ping.id, data.id, "Archived ping should have matching id");
+      Assert.equal(ping.type, data.type, "Archived ping should have matching type");
+      Assert.equal(ping.creationDate, data.dateCreated.toISOString(),
+                   "Archived ping should have matching creation date");
+    }
+  });
+
+  yield checkLoadingPings();
+
+  // Check that we find the archived pings again by scanning after a restart.
+  yield TelemetryController.testReset();
+
+  let pingList = yield TelemetryArchive.promiseArchivedPingList();
+  Assert.deepEqual(expectedPingList, pingList,
+                   "Should have submitted pings in archive list after restart");
+  yield checkLoadingPings();
+
+  // Write invalid pings into the archive with both valid and invalid names.
+  let writeToArchivedDir = Task.async(function*(dirname, filename, content, compressed) {
+    const dirPath = OS.Path.join(gPingsArchivePath, dirname);
+    yield OS.File.makeDir(dirPath, { ignoreExisting: true });
+    const filePath = OS.Path.join(dirPath, filename);
+    const options = { tmpPath: filePath + ".tmp", noOverwrite: false };
+    if (compressed) {
+      options.compression = "lz4";
+    }
+    yield OS.File.writeAtomic(filePath, content, options);
+  });
+
+  const FAKE_ID1 = "10000000-0123-0123-0123-0123456789a1";
+  const FAKE_ID2 = "20000000-0123-0123-0123-0123456789a2";
+  const FAKE_ID3 = "20000000-0123-0123-0123-0123456789a3";
+  const FAKE_TYPE = "foo";
+
+  // These should get rejected.
+  yield writeToArchivedDir("xx", "foo.json", "{}");
+  yield writeToArchivedDir("2010-02", "xx.xx.xx.json", "{}");
+  // This one should get picked up...
+  yield writeToArchivedDir("2010-02", "1." + FAKE_ID1 + "." + FAKE_TYPE + ".json", "{}");
+  // ... but get overwritten by this one.
+  yield writeToArchivedDir("2010-02", "2." + FAKE_ID1 + "." + FAKE_TYPE + ".json", "");
+  // This should get picked up fine.
+  yield writeToArchivedDir("2010-02", "3." + FAKE_ID2 + "." + FAKE_TYPE + ".json", "");
+  // This compressed ping should get picked up fine as well.
+  yield writeToArchivedDir("2010-02", "4." + FAKE_ID3 + "." + FAKE_TYPE + ".jsonlz4", "");
+
+  expectedPingList.push({
+    id: FAKE_ID1,
+    type: "foo",
+    timestampCreated: 2,
+  });
+  expectedPingList.push({
+    id: FAKE_ID2,
+    type: "foo",
+    timestampCreated: 3,
+  });
+  expectedPingList.push({
+    id: FAKE_ID3,
+    type: "foo",
+    timestampCreated: 4,
+  });
+  expectedPingList.sort((a, b) => a.timestampCreated - b.timestampCreated);
+
+  // Reset the TelemetryArchive so we scan the archived dir again.
+  yield TelemetryController.testReset();
+
+  // Check that we are still picking up the valid archived pings on disk,
+  // plus the valid ones above.
+  pingList = yield TelemetryArchive.promiseArchivedPingList();
+  Assert.deepEqual(expectedPingList, pingList, "Should have picked up valid archived pings");
+  yield checkLoadingPings();
+
+  // Now check that we fail to load the two invalid pings from above.
+  Assert.ok((yield promiseRejects(TelemetryArchive.promiseArchivedPingById(FAKE_ID1))),
+            "Should have rejected invalid ping");
+  Assert.ok((yield promiseRejects(TelemetryArchive.promiseArchivedPingById(FAKE_ID2))),
+            "Should have rejected invalid ping");
+});
+
+add_task(function* test_archiveCleanup() {
+  const PING_TYPE = "foo";
+
+  // Empty the archive.
+  yield OS.File.removeDir(gPingsArchivePath);
+
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT").clear();
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT").clear();
+  // Also reset these histograms to make sure normal sized pings don't get counted.
+  Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").clear();
+  Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB").clear();
+
+  // Build the cache. Nothing should be evicted as there's no ping directory.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testCleanupTaskPromise();
+  yield TelemetryArchive.promiseArchivedPingList();
+
+  let h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must report 0 pings scanned if no archive dir exists.");
+  // One directory out of four was removed as well.
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must report 0 evicted dirs if no archive dir exists.");
+
+  let expectedPrunedInfo = [];
+  let expectedNotPrunedInfo = [];
+
+  let checkArchive = Task.async(function*() {
+    // Check that the pruned pings are not on disk anymore.
+    for (let prunedInfo of expectedPrunedInfo) {
+      yield Assert.rejects(TelemetryArchive.promiseArchivedPingById(prunedInfo.id),
+                           "Ping " + prunedInfo.id + " should have been pruned.");
+      const pingPath =
+        TelemetryStorage._testGetArchivedPingPath(prunedInfo.id, prunedInfo.creationDate, PING_TYPE);
+      Assert.ok(!(yield OS.File.exists(pingPath)), "The ping should not be on the disk anymore.");
+    }
+
+    // Check that the expected pings are there.
+    for (let expectedInfo of expectedNotPrunedInfo) {
+      Assert.ok((yield TelemetryArchive.promiseArchivedPingById(expectedInfo.id)),
+                "Ping" + expectedInfo.id + " should be in the archive.");
+    }
+  });
+
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").clear();
+
+  // Create a ping which should be pruned because it is past the retention period.
+  let date = fakeNow(2010, 1, 1, 1, 0, 0);
+  let firstDate = date;
+  let pingId = yield TelemetryController.submitExternalPing(PING_TYPE, {}, {});
+  expectedPrunedInfo.push({ id: pingId, creationDate: date });
+
+  // Create a ping which should be kept because it is within the retention period.
+  const oldestDirectoryDate = fakeNow(2010, 2, 1, 1, 0, 0);
+  pingId = yield TelemetryController.submitExternalPing(PING_TYPE, {}, {});
+  expectedNotPrunedInfo.push({ id: pingId, creationDate: oldestDirectoryDate });
+
+  // Create 20 other pings which are within the retention period, but would be affected
+  // by the disk quota.
+  for (let month of [3, 4]) {
+    for (let minute = 0; minute < 10; minute++) {
+      date = fakeNow(2010, month, 1, 1, minute, 0);
+      pingId = yield TelemetryController.submitExternalPing(PING_TYPE, {}, {});
+      expectedNotPrunedInfo.push({ id: pingId, creationDate: date });
+    }
+  }
+
+  // We expect all the pings we archived to be in this histogram.
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT");
+  Assert.equal(h.snapshot().sum, 22, "All the pings must be live-accumulated in the histogram.");
+  // Reset the histogram that will be populated by the archive scan.
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS").clear();
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE").clear();
+
+  // Move the current date 60 days ahead of the first ping.
+  fakeNow(futureDate(firstDate, 60 * MILLISECONDS_PER_DAY));
+  // Reset TelemetryArchive and TelemetryController to start the startup cleanup.
+  yield TelemetryController.testReset();
+  // Wait for the cleanup to finish.
+  yield TelemetryStorage.testCleanupTaskPromise();
+  // Then scan the archived dir.
+  yield TelemetryArchive.promiseArchivedPingList();
+
+  // Check that the archive is in the correct state.
+  yield checkArchive();
+
+  // Make sure the ping count is correct after the scan (one ping was removed).
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT").snapshot();
+  Assert.equal(h.sum, 21, "The histogram must count all the pings in the archive.");
+  // One directory out of four was removed as well.
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS").snapshot();
+  Assert.equal(h.sum, 1, "Telemetry must correctly report removed archive directories.");
+  // Check that the remaining directories are correctly counted.
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT").snapshot();
+  Assert.equal(h.sum, 3, "Telemetry must correctly report the remaining archive directories.");
+  // Check that the remaining directories are correctly counted.
+  const oldestAgeInMonths = 1;
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE").snapshot();
+  Assert.equal(h.sum, oldestAgeInMonths,
+               "Telemetry must correctly report age of the oldest directory in the archive.");
+
+  // We need to test the archive size before we hit the quota, otherwise a special
+  // value is recorded.
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").clear();
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").clear();
+  Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS").clear();
+
+  // Move the current date 60 days ahead of the second ping.
+  fakeNow(futureDate(oldestDirectoryDate, 60 * MILLISECONDS_PER_DAY));
+  // Reset TelemetryController and TelemetryArchive.
+  yield TelemetryController.testReset();
+  // Wait for the cleanup to finish.
+  yield TelemetryStorage.testCleanupTaskPromise();
+  // Then scan the archived dir again.
+  yield TelemetryArchive.promiseArchivedPingList();
+
+  // Move the oldest ping to the unexpected pings list.
+  expectedPrunedInfo.push(expectedNotPrunedInfo.shift());
+  // Check that the archive is in the correct state.
+  yield checkArchive();
+
+  // Find how much disk space the archive takes.
+  const archivedPingsInfo = yield getArchivedPingsInfo();
+  let archiveSizeInBytes =
+    archivedPingsInfo.reduce((lastResult, element) => lastResult + element.size, 0);
+
+  // Check that the correct values for quota probes are reported when no quota is hit.
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").snapshot();
+  Assert.equal(h.sum, Math.round(archiveSizeInBytes / 1024 / 1024),
+               "Telemetry must report the correct archive size.");
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must report 0 evictions if quota is not hit.");
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must report a null elapsed time if quota is not hit.");
+
+  // Set the quota to 80% of the space.
+  const testQuotaInBytes = archiveSizeInBytes * 0.8;
+  fakeStorageQuota(testQuotaInBytes);
+
+  // The storage prunes archived pings until we reach 90% of the requested storage quota.
+  // Based on that, find how many pings should be kept.
+  const safeQuotaSize = testQuotaInBytes * 0.9;
+  let sizeInBytes = 0;
+  let pingsWithinQuota = [];
+  let pingsOutsideQuota = [];
+
+  for (let pingInfo of archivedPingsInfo) {
+    sizeInBytes += pingInfo.size;
+    if (sizeInBytes >= safeQuotaSize) {
+      pingsOutsideQuota.push({ id: pingInfo.id, creationDate: new Date(pingInfo.timestamp) });
+      continue;
+    }
+    pingsWithinQuota.push({ id: pingInfo.id, creationDate: new Date(pingInfo.timestamp) });
+  }
+
+  expectedNotPrunedInfo = pingsWithinQuota;
+  expectedPrunedInfo = expectedPrunedInfo.concat(pingsOutsideQuota);
+
+  // Reset TelemetryArchive and TelemetryController to start the startup cleanup.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testCleanupTaskPromise();
+  yield TelemetryArchive.promiseArchivedPingList();
+  // Check that the archive is in the correct state.
+  yield checkArchive();
+
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").snapshot();
+  Assert.equal(h.sum, pingsOutsideQuota.length,
+               "Telemetry must correctly report the over quota pings evicted from the archive.");
+  h = Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").snapshot();
+  Assert.equal(h.sum, 300, "Archive quota was hit, a special size must be reported.");
+
+  // Trigger a cleanup again and make sure we're not removing anything.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testCleanupTaskPromise();
+  yield TelemetryArchive.promiseArchivedPingList();
+  yield checkArchive();
+
+  const OVERSIZED_PING_ID = "9b21ec8f-f762-4d28-a2c1-44e1c4694f24";
+  // Create and archive an oversized, uncompressed, ping.
+  const OVERSIZED_PING = {
+    id: OVERSIZED_PING_ID,
+    type: PING_TYPE,
+    creationDate: (new Date()).toISOString(),
+    // Generate a ~2MB string to use as the payload.
+    payload: generateRandomString(2 * 1024 * 1024)
+  };
+  yield TelemetryArchive.promiseArchivePing(OVERSIZED_PING);
+
+  // Get the size of the archived ping.
+  const oversizedPingPath =
+    TelemetryStorage._testGetArchivedPingPath(OVERSIZED_PING.id, new Date(OVERSIZED_PING.creationDate), PING_TYPE) + "lz4";
+  const archivedPingSizeMB = Math.floor((yield OS.File.stat(oversizedPingPath)).size / 1024 / 1024);
+
+  // We expect the oversized ping to be pruned when scanning the archive.
+  expectedPrunedInfo.push({ id: OVERSIZED_PING_ID, creationDate: new Date(OVERSIZED_PING.creationDate) });
+
+  // Scan the archive.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testCleanupTaskPromise();
+  yield TelemetryArchive.promiseArchivedPingList();
+  // The following also checks that non oversized pings are not removed.
+  yield checkArchive();
+
+  // Make sure we're correctly updating the related histograms.
+  h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").snapshot();
+  Assert.equal(h.sum, 1, "Telemetry must report 1 oversized ping in the archive.");
+  h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB").snapshot();
+  Assert.equal(h.counts[archivedPingSizeMB], 1,
+               "Telemetry must report the correct size for the oversized ping.");
+});
+
+add_task(function* test_clientId() {
+  // Check that a ping submitted after the delayed telemetry initialization completed
+  // should get a valid client id.
+  yield TelemetryController.testReset();
+  const clientId = yield ClientID.getClientID();
+
+  let id = yield TelemetryController.submitExternalPing("test-type", {}, {addClientId: true});
+  let ping = yield TelemetryArchive.promiseArchivedPingById(id);
+
+  Assert.ok(!!ping, "Should have loaded the ping.");
+  Assert.ok("clientId" in ping, "Ping should have a client id.");
+  Assert.ok(UUID_REGEX.test(ping.clientId), "Client id is in UUID format.");
+  Assert.equal(ping.clientId, clientId, "Ping client id should match the global client id.");
+
+  // We should have cached the client id now. Lets confirm that by
+  // checking the client id on a ping submitted before the async
+  // controller setup is finished.
+  let promiseSetup = TelemetryController.testReset();
+  id = yield TelemetryController.submitExternalPing("test-type", {}, {addClientId: true});
+  ping = yield TelemetryArchive.promiseArchivedPingById(id);
+  Assert.equal(ping.clientId, clientId);
+
+  // Finish setup.
+  yield promiseSetup;
+});
+
+add_task(function* test_InvalidPingType() {
+  const TYPES = [
+    "a",
+    "-",
+    "¿€€€?",
+    "-foo-",
+    "-moo",
+    "zoo-",
+    ".bar",
+    "asfd.asdf",
+  ];
+
+  for (let type of TYPES) {
+    let histogram = Telemetry.getKeyedHistogramById("TELEMETRY_INVALID_PING_TYPE_SUBMITTED");
+    Assert.equal(histogram.snapshot(type).sum, 0,
+                 "Should not have counted this invalid ping yet: " + type);
+    Assert.ok(promiseRejects(TelemetryController.submitExternalPing(type, {})),
+              "Ping type should have been rejected.");
+    Assert.equal(histogram.snapshot(type).sum, 1,
+                 "Should have counted this as an invalid ping type.");
+  }
+});
+
+add_task(function* test_InvalidPayloadType() {
+  const PAYLOAD_TYPES = [
+    19,
+    "string",
+    [1, 2, 3, 4],
+    null,
+    undefined,
+  ];
+
+  let histogram = Telemetry.getHistogramById("TELEMETRY_INVALID_PAYLOAD_SUBMITTED");
+  for (let i = 0; i < PAYLOAD_TYPES.length; i++) {
+    histogram.clear();
+    Assert.equal(histogram.snapshot().sum, 0,
+      "Should not have counted this invalid payload yet: " + JSON.stringify(PAYLOAD_TYPES[i]));
+    Assert.ok(yield promiseRejects(TelemetryController.submitExternalPing("payload-test", PAYLOAD_TYPES[i])),
+      "Payload type should have been rejected.");
+    Assert.equal(histogram.snapshot().sum, 1,
+      "Should have counted this as an invalid payload type.");
+  }
+});
+
+add_task(function* test_currentPingData() {
+  yield TelemetryController.testSetup();
+
+  // Setup test data.
+  let h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTOUT");
+  h.clear();
+  h.add(1);
+  let k = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT");
+  k.clear();
+  k.add("a", 1);
+
+  // Get current ping data objects and check that their data is sane.
+  for (let subsession of [true, false]) {
+    let ping = TelemetryController.getCurrentPingData(subsession);
+
+    Assert.ok(!!ping, "Should have gotten a ping.");
+    Assert.equal(ping.type, "main", "Ping should have correct type.");
+    const expectedReason = subsession ? "gather-subsession-payload" : "gather-payload";
+    Assert.equal(ping.payload.info.reason, expectedReason, "Ping should have the correct reason.");
+
+    let id = "TELEMETRY_TEST_RELEASE_OPTOUT";
+    Assert.ok(id in ping.payload.histograms, "Payload should have test count histogram.");
+    Assert.equal(ping.payload.histograms[id].sum, 1, "Test count value should match.");
+    id = "TELEMETRY_TEST_KEYED_RELEASE_OPTOUT";
+    Assert.ok(id in ping.payload.keyedHistograms, "Payload should have keyed test histogram.");
+    Assert.equal(ping.payload.keyedHistograms[id]["a"].sum, 1, "Keyed test value should match.");
+  }
+});
+
+add_task(function* test_shutdown() {
+  yield TelemetryController.testShutdown();
+});

toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js

@@ -0,0 +1,236 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+Cu.import("resource://gre/modules/Preferences.jsm", this);
+Cu.import("resource://gre/modules/Promise.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetryEnvironment.jsm", this);
+Cu.import("resource://gre/modules/osfile.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+
+const MS_IN_ONE_HOUR  = 60 * 60 * 1000;
+const MS_IN_ONE_DAY   = 24 * MS_IN_ONE_HOUR;
+
+const PREF_BRANCH = "toolkit.telemetry.";
+const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";
+
+const REASON_ABORTED_SESSION = "aborted-session";
+const REASON_DAILY = "daily";
+const REASON_ENVIRONMENT_CHANGE = "environment-change";
+const REASON_SHUTDOWN = "shutdown";
+
+XPCOMUtils.defineLazyGetter(this, "DATAREPORTING_PATH", function() {
+  return OS.Path.join(OS.Constants.Path.profileDir, "datareporting");
+});
+
+var promiseValidateArchivedPings = Task.async(function*(aExpectedReasons) {
+  // The list of ping reasons which mark the session end (and must reset the subsession
+  // count).
+  const SESSION_END_PING_REASONS = new Set([ REASON_ABORTED_SESSION, REASON_SHUTDOWN ]);
+
+  let list = yield TelemetryArchive.promiseArchivedPingList();
+
+  // We're just interested in the "main" pings.
+  list = list.filter(p => p.type == "main");
+
+  Assert.equal(aExpectedReasons.length, list.length, "All the expected pings must be received.");
+
+  let previousPing = yield TelemetryArchive.promiseArchivedPingById(list[0].id);
+  Assert.equal(aExpectedReasons.shift(), previousPing.payload.info.reason,
+               "Telemetry should only get pings with expected reasons.");
+  Assert.equal(previousPing.payload.info.previousSessionId, null,
+               "The first session must report a null previous session id.");
+  Assert.equal(previousPing.payload.info.previousSubsessionId, null,
+               "The first subsession must report a null previous subsession id.");
+  Assert.equal(previousPing.payload.info.profileSubsessionCounter, 1,
+               "profileSubsessionCounter must be 1 the first time.");
+  Assert.equal(previousPing.payload.info.subsessionCounter, 1,
+               "subsessionCounter must be 1 the first time.");
+
+  let expectedSubsessionCounter = 1;
+  let expectedPreviousSessionId = previousPing.payload.info.sessionId;
+
+  for (let i = 1; i < list.length; i++) {
+    let currentPing = yield TelemetryArchive.promiseArchivedPingById(list[i].id);
+    let currentInfo = currentPing.payload.info;
+    let previousInfo = previousPing.payload.info;
+    do_print("Archive entry " + i + " - id: " + currentPing.id + ", reason: " + currentInfo.reason);
+
+    Assert.equal(aExpectedReasons.shift(), currentInfo.reason,
+                 "Telemetry should only get pings with expected reasons.");
+    Assert.equal(currentInfo.previousSessionId, expectedPreviousSessionId,
+                 "Telemetry must correctly chain session identifiers.");
+    Assert.equal(currentInfo.previousSubsessionId, previousInfo.subsessionId,
+                 "Telemetry must correctly chain subsession identifiers.");
+    Assert.equal(currentInfo.profileSubsessionCounter, previousInfo.profileSubsessionCounter + 1,
+                 "Telemetry must correctly track the profile subsessions count.");
+    Assert.equal(currentInfo.subsessionCounter, expectedSubsessionCounter,
+                 "The subsession counter should be monotonically increasing.");
+
+    // Store the current ping as previous.
+    previousPing = currentPing;
+    // Reset the expected subsession counter, if required. Otherwise increment the expected
+    // subsession counter.
+    // If this is the final subsession of a session we need to update expected values accordingly.
+    if (SESSION_END_PING_REASONS.has(currentInfo.reason)) {
+      expectedSubsessionCounter = 1;
+      expectedPreviousSessionId = currentInfo.sessionId;
+    } else {
+      expectedSubsessionCounter++;
+    }
+  }
+});
+
+add_task(function* test_setup() {
+  do_test_pending();
+
+  // Addon manager needs a profile directory
+  do_get_profile();
+  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  Preferences.set(PREF_TELEMETRY_ENABLED, true);
+});
+
+add_task(function* test_subsessionsChaining() {
+  if (gIsAndroid) {
+    // We don't support subsessions yet on Android, so skip the next checks.
+    return;
+  }
+
+  const PREF_TEST = PREF_BRANCH + "test.pref1";
+  const PREFS_TO_WATCH = new Map([
+    [PREF_TEST, {what: TelemetryEnvironment.RECORD_PREF_VALUE}],
+  ]);
+  Preferences.reset(PREF_TEST);
+
+  // Fake the clock data to manually trigger an aborted-session ping and a daily ping.
+  // This is also helpful to make sure we get the archived pings in an expected order.
+  let now = fakeNow(2009, 9, 18, 0, 0, 0);
+  let monotonicNow = fakeMonotonicNow(1000);
+
+  let moveClockForward = (minutes) => {
+    let ms = minutes * MILLISECONDS_PER_MINUTE;
+    now = fakeNow(futureDate(now, ms));
+    monotonicNow = fakeMonotonicNow(monotonicNow + ms);
+  }
+
+  // Keep track of the ping reasons we're expecting in this test.
+  let expectedReasons = [];
+
+  // Start and shut down Telemetry. We expect a shutdown ping with profileSubsessionCounter: 1,
+  // subsessionCounter: 1, subsessionId: A,  and previousSubsessionId: null to be archived.
+  yield TelemetryController.testSetup();
+  yield TelemetryController.testShutdown();
+  expectedReasons.push(REASON_SHUTDOWN);
+
+  // Start Telemetry but don't wait for it to initialise before shutting down. We expect a
+  // shutdown ping with profileSubsessionCounter: 2, subsessionCounter: 1, subsessionId: B
+  // and previousSubsessionId: A to be archived.
+  moveClockForward(30);
+  TelemetryController.testReset();
+  yield TelemetryController.testShutdown();
+  expectedReasons.push(REASON_SHUTDOWN);
+
+  // Start Telemetry and simulate an aborted-session ping. We expect an aborted-session ping
+  // with profileSubsessionCounter: 3, subsessionCounter: 1, subsessionId: C and
+  // previousSubsessionId: B to be archived.
+  let schedulerTickCallback = null;
+  fakeSchedulerTimer(callback => schedulerTickCallback = callback, () => {});
+  yield TelemetryController.testReset();
+  moveClockForward(6);
+  // Trigger the an aborted session ping save. When testing,we are not saving the aborted-session
+  // ping as soon as Telemetry starts, otherwise we would end up with unexpected pings being
+  // sent when calling |TelemetryController.testReset()|, thus breaking some tests.
+  Assert.ok(!!schedulerTickCallback);
+  yield schedulerTickCallback();
+  expectedReasons.push(REASON_ABORTED_SESSION);
+
+  // Start Telemetry and trigger an environment change through a pref modification. We expect
+  // an environment-change ping with profileSubsessionCounter: 4, subsessionCounter: 1,
+  // subsessionId: D and previousSubsessionId: C to be archived.
+  moveClockForward(30);
+  yield TelemetryController.testReset();
+  TelemetryEnvironment.testWatchPreferences(PREFS_TO_WATCH);
+  moveClockForward(30);
+  Preferences.set(PREF_TEST, 1);
+  expectedReasons.push(REASON_ENVIRONMENT_CHANGE);
+
+  // Shut down Telemetry. We expect a shutdown ping with profileSubsessionCounter: 5,
+  // subsessionCounter: 2, subsessionId: E and previousSubsessionId: D to be archived.
+  moveClockForward(30);
+  yield TelemetryController.testShutdown();
+  expectedReasons.push(REASON_SHUTDOWN);
+
+  // Start Telemetry and trigger a daily ping. We expect a daily ping with
+  // profileSubsessionCounter: 6, subsessionCounter: 1, subsessionId: F and
+  // previousSubsessionId: E to be archived.
+  moveClockForward(30);
+  yield TelemetryController.testReset();
+
+  // Delay the callback around midnight.
+  now = fakeNow(futureDate(now, MS_IN_ONE_DAY));
+  // Trigger the daily ping.
+  yield schedulerTickCallback();
+  expectedReasons.push(REASON_DAILY);
+
+  // Trigger an environment change ping. We expect an environment-changed ping with
+  // profileSubsessionCounter: 7, subsessionCounter: 2, subsessionId: G and
+  // previousSubsessionId: F to be archived.
+  moveClockForward(30);
+  Preferences.set(PREF_TEST, 0);
+  expectedReasons.push(REASON_ENVIRONMENT_CHANGE);
+
+  // Shut down Telemetry and trigger a shutdown ping.
+  moveClockForward(30);
+  yield TelemetryController.testShutdown();
+  expectedReasons.push(REASON_SHUTDOWN);
+
+  // Start Telemetry and trigger an environment change.
+  yield TelemetryController.testReset();
+  TelemetryEnvironment.testWatchPreferences(PREFS_TO_WATCH);
+  moveClockForward(30);
+  Preferences.set(PREF_TEST, 1);
+  expectedReasons.push(REASON_ENVIRONMENT_CHANGE);
+
+  // Don't shut down, instead trigger an aborted-session ping.
+  moveClockForward(6);
+  // Trigger the an aborted session ping save.
+  yield schedulerTickCallback();
+  expectedReasons.push(REASON_ABORTED_SESSION);
+
+  // Start Telemetry and trigger a daily ping.
+  moveClockForward(30);
+  yield TelemetryController.testReset();
+  // Delay the callback around midnight.
+  now = futureDate(now, MS_IN_ONE_DAY);
+  fakeNow(now);
+  // Trigger the daily ping.
+  yield schedulerTickCallback();
+  expectedReasons.push(REASON_DAILY);
+
+  // Trigger an environment change.
+  moveClockForward(30);
+  Preferences.set(PREF_TEST, 0);
+  expectedReasons.push(REASON_ENVIRONMENT_CHANGE);
+
+  // And an aborted-session ping again.
+  moveClockForward(6);
+  // Trigger the an aborted session ping save.
+  yield schedulerTickCallback();
+  expectedReasons.push(REASON_ABORTED_SESSION);
+
+  // Make sure the aborted-session ping gets archived.
+  yield TelemetryController.testReset();
+
+  yield promiseValidateArchivedPings(expectedReasons);
+});
+
+add_task(function* () {
+  yield TelemetryController.testShutdown();
+  do_test_finished();
+});

toolkit/components/telemetry/tests/unit/test_TelemetryController.js

@@ -0,0 +1,507 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+/* This testcase triggers two telemetry pings.
+ *
+ * Telemetry code keeps histograms of past telemetry pings. The first
+ * ping populates these histograms. One of those histograms is then
+ * checked in the second request.
+ */
+
+Cu.import("resource://gre/modules/ClientID.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySend.jsm", this);
+Cu.import("resource://gre/modules/TelemetryArchive.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/Promise.jsm", this);
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+const PING_FORMAT_VERSION = 4;
+const DELETION_PING_TYPE = "deletion";
+const TEST_PING_TYPE = "test-ping-type";
+
+const PLATFORM_VERSION = "1.9.2";
+const APP_VERSION = "1";
+const APP_NAME = "XPCShell";
+
+const PREF_BRANCH = "toolkit.telemetry.";
+const PREF_ENABLED = PREF_BRANCH + "enabled";
+const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";
+const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";
+const PREF_UNIFIED = PREF_BRANCH + "unified";
+
+var gClientID = null;
+
+function sendPing(aSendClientId, aSendEnvironment) {
+  if (PingServer.started) {
+    TelemetrySend.setServer("http://localhost:" + PingServer.port);
+  } else {
+    TelemetrySend.setServer("http://doesnotexist");
+  }
+
+  let options = {
+    addClientId: aSendClientId,
+    addEnvironment: aSendEnvironment,
+  };
+  return TelemetryController.submitExternalPing(TEST_PING_TYPE, {}, options);
+}
+
+function checkPingFormat(aPing, aType, aHasClientId, aHasEnvironment) {
+  const MANDATORY_PING_FIELDS = [
+    "type", "id", "creationDate", "version", "application", "payload"
+  ];
+
+  const APPLICATION_TEST_DATA = {
+    buildId: gAppInfo.appBuildID,
+    name: APP_NAME,
+    version: APP_VERSION,
+    displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY,
+    vendor: "Mozilla",
+    platformVersion: PLATFORM_VERSION,
+    xpcomAbi: "noarch-spidermonkey",
+  };
+
+  // Check that the ping contains all the mandatory fields.
+  for (let f of MANDATORY_PING_FIELDS) {
+    Assert.ok(f in aPing, f + " must be available.");
+  }
+
+  Assert.equal(aPing.type, aType, "The ping must have the correct type.");
+  Assert.equal(aPing.version, PING_FORMAT_VERSION, "The ping must have the correct version.");
+
+  // Test the application section.
+  for (let f in APPLICATION_TEST_DATA) {
+    Assert.equal(aPing.application[f], APPLICATION_TEST_DATA[f],
+                 f + " must have the correct value.");
+  }
+
+  // We can't check the values for channel and architecture. Just make
+  // sure they are in.
+  Assert.ok("architecture" in aPing.application,
+            "The application section must have an architecture field.");
+  Assert.ok("channel" in aPing.application,
+            "The application section must have a channel field.");
+
+  // Check the clientId and environment fields, as needed.
+  Assert.equal("clientId" in aPing, aHasClientId);
+  Assert.equal("environment" in aPing, aHasEnvironment);
+}
+
+add_task(function* test_setup() {
+  // Addon manager needs a profile directory
+  do_get_profile();
+  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  Services.prefs.setBoolPref(PREF_ENABLED, true);
+  Services.prefs.setBoolPref(PREF_FHR_UPLOAD_ENABLED, true);
+
+  yield new Promise(resolve =>
+    Telemetry.asyncFetchTelemetryData(wrapWithExceptionHandler(resolve)));
+});
+
+add_task(function* asyncSetup() {
+  yield TelemetryController.testSetup();
+});
+
+// Ensure that not overwriting an existing file fails silently
+add_task(function* test_overwritePing() {
+  let ping = {id: "foo"};
+  yield TelemetryStorage.savePing(ping, true);
+  yield TelemetryStorage.savePing(ping, false);
+  yield TelemetryStorage.cleanupPingFile(ping);
+});
+
+// Checks that a sent ping is correctly received by a dummy http server.
+add_task(function* test_simplePing() {
+  PingServer.start();
+  // Update the Telemetry Server preference with the address of the local server.
+  // Otherwise we might end up sending stuff to a non-existing server after
+  // |TelemetryController.testReset| is called.
+  Preferences.set(TelemetryController.Constants.PREF_SERVER, "http://localhost:" + PingServer.port);
+
+  yield sendPing(false, false);
+  let request = yield PingServer.promiseNextRequest();
+
+  // Check that we have a version query parameter in the URL.
+  Assert.notEqual(request.queryString, "");
+
+  // Make sure the version in the query string matches the new ping format version.
+  let params = request.queryString.split("&");
+  Assert.ok(params.find(p => p == ("v=" + PING_FORMAT_VERSION)));
+
+  let ping = decodeRequestPayload(request);
+  checkPingFormat(ping, TEST_PING_TYPE, false, false);
+});
+
+add_task(function* test_disableDataUpload() {
+  const isUnified = Preferences.get(PREF_UNIFIED, false);
+  if (!isUnified) {
+    // Skipping the test if unified telemetry is off, as no deletion ping will
+    // be generated.
+    return;
+  }
+
+  // Disable FHR upload: this should trigger a deletion ping.
+  Preferences.set(PREF_FHR_UPLOAD_ENABLED, false);
+
+  let ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, DELETION_PING_TYPE, true, false);
+  // Wait on ping activity to settle.
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  // Restore FHR Upload.
+  Preferences.set(PREF_FHR_UPLOAD_ENABLED, true);
+
+  // Simulate a failure in sending the deletion ping by disabling the HTTP server.
+  yield PingServer.stop();
+
+  // Try to send a ping. It will be saved as pending  and get deleted when disabling upload.
+  TelemetryController.submitExternalPing(TEST_PING_TYPE, {});
+
+  // Disable FHR upload to send a deletion ping again.
+  Preferences.set(PREF_FHR_UPLOAD_ENABLED, false);
+
+  // Wait on sending activity to settle, as |TelemetryController.testReset()| doesn't do that.
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  // Wait for the pending pings to be deleted. Resetting TelemetryController doesn't
+  // trigger the shutdown, so we need to call it ourselves.
+  yield TelemetryStorage.shutdown();
+  // Simulate a restart, and spin the send task.
+  yield TelemetryController.testReset();
+
+  // Disabling Telemetry upload must clear out all the pending pings.
+  let pendingPings = yield TelemetryStorage.loadPendingPingList();
+  Assert.equal(pendingPings.length, 1,
+               "All the pending pings but the deletion ping should have been deleted");
+
+  // Enable the ping server again.
+  PingServer.start();
+  // We set the new server using the pref, otherwise it would get reset with
+  // |TelemetryController.testReset|.
+  Preferences.set(TelemetryController.Constants.PREF_SERVER, "http://localhost:" + PingServer.port);
+
+  // Stop the sending task and then start it again.
+  yield TelemetrySend.shutdown();
+  // Reset the controller to spin the ping sending task.
+  yield TelemetryController.testReset();
+  ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, DELETION_PING_TYPE, true, false);
+
+  // Wait on ping activity to settle before moving on to the next test. If we were
+  // to shut down telemetry, even though the PingServer caught the expected pings,
+  // TelemetrySend could still be processing them (clearing pings would happen in
+  // a couple of ticks). Shutting down would cancel the request and save them as
+  // pending pings.
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  // Restore FHR Upload.
+  Preferences.set(PREF_FHR_UPLOAD_ENABLED, true);
+});
+
+add_task(function* test_pingHasClientId() {
+  const PREF_CACHED_CLIENTID = "toolkit.telemetry.cachedClientID";
+
+  // Make sure we have no cached client ID for this test: we'll try to send
+  // a ping with it while Telemetry is being initialized.
+  Preferences.reset(PREF_CACHED_CLIENTID);
+  yield TelemetryController.testShutdown();
+  yield ClientID._reset();
+  yield TelemetryStorage.testClearPendingPings();
+  // And also clear the counter histogram since we're here.
+  let h = Telemetry.getHistogramById("TELEMETRY_PING_SUBMISSION_WAITING_CLIENTID");
+  h.clear();
+
+  // Init telemetry and try to send a ping with a client ID.
+  let promisePingSetup = TelemetryController.testReset();
+  yield sendPing(true, false);
+  Assert.equal(h.snapshot().sum, 1,
+               "We must have a ping waiting for the clientId early during startup.");
+  // Wait until we are fully initialized. Pings will be assembled but won't get
+  // sent before then.
+  yield promisePingSetup;
+
+  let ping = yield PingServer.promiseNextPing();
+  // Fetch the client ID after initializing and fetching the the ping, so we
+  // don't unintentionally trigger its loading. We'll still need the client ID
+  // to see if the ping looks sane.
+  gClientID = yield ClientID.getClientID();
+
+  checkPingFormat(ping, TEST_PING_TYPE, true, false);
+  Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");
+
+  // Shutdown Telemetry so we can safely restart it.
+  yield TelemetryController.testShutdown();
+  yield TelemetryStorage.testClearPendingPings();
+
+  // We should have cached the client ID now. Lets confirm that by checking it before
+  // the async ping setup is finished.
+  h.clear();
+  promisePingSetup = TelemetryController.testReset();
+  yield sendPing(true, false);
+  yield promisePingSetup;
+
+  // Check that we received the cached client id.
+  Assert.equal(h.snapshot().sum, 0, "We must have used the cached clientId.");
+  ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, TEST_PING_TYPE, true, false);
+  Assert.equal(ping.clientId, gClientID,
+               "Telemetry should report the correct cached clientId.");
+
+  // Check that sending a ping without relying on the cache, after the
+  // initialization, still works.
+  Preferences.reset(PREF_CACHED_CLIENTID);
+  yield TelemetryController.testShutdown();
+  yield TelemetryStorage.testClearPendingPings();
+  yield TelemetryController.testReset();
+  yield sendPing(true, false);
+  ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, TEST_PING_TYPE, true, false);
+  Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");
+  Assert.equal(h.snapshot().sum, 0, "No ping should have been waiting for a clientId.");
+});
+
+add_task(function* test_pingHasEnvironment() {
+  // Send a ping with the environment data.
+  yield sendPing(false, true);
+  let ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, TEST_PING_TYPE, false, true);
+
+  // Test a field in the environment build section.
+  Assert.equal(ping.application.buildId, ping.environment.build.buildId);
+});
+
+add_task(function* test_pingHasEnvironmentAndClientId() {
+  // Send a ping with the environment data and client id.
+  yield sendPing(true, true);
+  let ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, TEST_PING_TYPE, true, true);
+
+  // Test a field in the environment build section.
+  Assert.equal(ping.application.buildId, ping.environment.build.buildId);
+  // Test that we have the correct clientId.
+  Assert.equal(ping.clientId, gClientID, "The correct clientId must be reported.");
+});
+
+add_task(function* test_archivePings() {
+  let now = new Date(2009, 10, 18, 12, 0, 0);
+  fakeNow(now);
+
+  // Disable ping upload so that pings don't get sent.
+  // With unified telemetry the FHR upload pref controls this,
+  // with non-unified telemetry the Telemetry enabled pref.
+  const isUnified = Preferences.get(PREF_UNIFIED, false);
+  const uploadPref = isUnified ? PREF_FHR_UPLOAD_ENABLED : PREF_ENABLED;
+  Preferences.set(uploadPref, false);
+
+  // If we're using unified telemetry, disabling ping upload will generate a "deletion"
+  // ping. Catch it.
+  if (isUnified) {
+    let ping = yield PingServer.promiseNextPing();
+    checkPingFormat(ping, DELETION_PING_TYPE, true, false);
+  }
+
+  // Register a new Ping Handler that asserts if a ping is received, then send a ping.
+  PingServer.registerPingHandler(() => Assert.ok(false, "Telemetry must not send pings if not allowed to."));
+  let pingId = yield sendPing(true, true);
+
+  // Check that the ping was archived, even with upload disabled.
+  let ping = yield TelemetryArchive.promiseArchivedPingById(pingId);
+  Assert.equal(ping.id, pingId, "TelemetryController should still archive pings.");
+
+  // Check that pings don't get archived if not allowed to.
+  now = new Date(2010, 10, 18, 12, 0, 0);
+  fakeNow(now);
+  Preferences.set(PREF_ARCHIVE_ENABLED, false);
+  pingId = yield sendPing(true, true);
+  let promise = TelemetryArchive.promiseArchivedPingById(pingId);
+  Assert.ok((yield promiseRejects(promise)),
+    "TelemetryController should not archive pings if the archive pref is disabled.");
+
+  // Enable archiving and the upload so that pings get sent and archived again.
+  Preferences.set(uploadPref, true);
+  Preferences.set(PREF_ARCHIVE_ENABLED, true);
+
+  now = new Date(2014, 6, 18, 22, 0, 0);
+  fakeNow(now);
+  // Restore the non asserting ping handler.
+  PingServer.resetPingHandler();
+  pingId = yield sendPing(true, true);
+
+  // Check that we archive pings when successfully sending them.
+  yield PingServer.promiseNextPing();
+  ping = yield TelemetryArchive.promiseArchivedPingById(pingId);
+  Assert.equal(ping.id, pingId,
+    "TelemetryController should still archive pings if ping upload is enabled.");
+});
+
+// Test that we fuzz the submission time around midnight properly
+// to avoid overloading the telemetry servers.
+add_task(function* test_midnightPingSendFuzzing() {
+  const fuzzingDelay = 60 * 60 * 1000;
+  fakeMidnightPingFuzzingDelay(fuzzingDelay);
+  let now = new Date(2030, 5, 1, 11, 0, 0);
+  fakeNow(now);
+
+  let waitForTimer = () => new Promise(resolve => {
+    fakePingSendTimer((callback, timeout) => {
+      resolve([callback, timeout]);
+    }, () => {});
+  });
+
+  PingServer.clearRequests();
+  yield TelemetryController.testReset();
+
+  // A ping after midnight within the fuzzing delay should not get sent.
+  now = new Date(2030, 5, 2, 0, 40, 0);
+  fakeNow(now);
+  PingServer.registerPingHandler((req, res) => {
+    Assert.ok(false, "No ping should be received yet.");
+  });
+  let timerPromise = waitForTimer();
+  yield sendPing(true, true);
+  let [timerCallback, timerTimeout] = yield timerPromise;
+  Assert.ok(!!timerCallback);
+  Assert.deepEqual(futureDate(now, timerTimeout), new Date(2030, 5, 2, 1, 0, 0));
+
+  // A ping just before the end of the fuzzing delay should not get sent.
+  now = new Date(2030, 5, 2, 0, 59, 59);
+  fakeNow(now);
+  timerPromise = waitForTimer();
+  yield sendPing(true, true);
+  [timerCallback, timerTimeout] = yield timerPromise;
+  Assert.deepEqual(timerTimeout, 1 * 1000);
+
+  // Restore the previous ping handler.
+  PingServer.resetPingHandler();
+
+  // Setting the clock to after the fuzzing delay, we should trigger the two ping sends
+  // with the timer callback.
+  now = futureDate(now, timerTimeout);
+  fakeNow(now);
+  yield timerCallback();
+  const pings = yield PingServer.promiseNextPings(2);
+  for (let ping of pings) {
+    checkPingFormat(ping, TEST_PING_TYPE, true, true);
+  }
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  // Moving the clock further we should still send pings immediately.
+  now = futureDate(now, 5 * 60 * 1000);
+  yield sendPing(true, true);
+  let ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, TEST_PING_TYPE, true, true);
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  // Check that pings shortly before midnight are immediately sent.
+  now = fakeNow(2030, 5, 3, 23, 59, 0);
+  yield sendPing(true, true);
+  ping = yield PingServer.promiseNextPing();
+  checkPingFormat(ping, TEST_PING_TYPE, true, true);
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  // Clean-up.
+  fakeMidnightPingFuzzingDelay(0);
+  fakePingSendTimer(() => {}, () => {});
+});
+
+add_task(function* test_changePingAfterSubmission() {
+  // Submit a ping with a custom payload.
+  let payload = { canary: "test" };
+  let pingPromise = TelemetryController.submitExternalPing(TEST_PING_TYPE, payload, options);
+
+  // Change the payload with a predefined value.
+  payload.canary = "changed";
+
+  // Wait for the ping to be archived.
+  const pingId = yield pingPromise;
+
+  // Make sure our changes didn't affect the submitted payload.
+  let archivedCopy = yield TelemetryArchive.promiseArchivedPingById(pingId);
+  Assert.equal(archivedCopy.payload.canary, "test",
+               "The payload must not be changed after being submitted.");
+});
+
+add_task(function* test_telemetryEnabledUnexpectedValue() {
+  // Remove the default value for toolkit.telemetry.enabled from the default prefs.
+  // Otherwise, we wouldn't be able to set the pref to a string.
+  let defaultPrefBranch = Services.prefs.getDefaultBranch(null);
+  defaultPrefBranch.deleteBranch(PREF_ENABLED);
+
+  // Set the preferences controlling the Telemetry status to a string.
+  Preferences.set(PREF_ENABLED, "false");
+  // Check that Telemetry is not enabled.
+  yield TelemetryController.testReset();
+  Assert.equal(Telemetry.canRecordExtended, false,
+               "Invalid values must not enable Telemetry recording.");
+
+  // Delete the pref again.
+  defaultPrefBranch.deleteBranch(PREF_ENABLED);
+
+  // Make sure that flipping it to true works.
+  Preferences.set(PREF_ENABLED, true);
+  yield TelemetryController.testReset();
+  Assert.equal(Telemetry.canRecordExtended, true,
+               "True must enable Telemetry recording.");
+
+  // Also check that the false works as well.
+  Preferences.set(PREF_ENABLED, false);
+  yield TelemetryController.testReset();
+  Assert.equal(Telemetry.canRecordExtended, false,
+               "False must disable Telemetry recording.");
+});
+
+add_task(function* test_telemetryCleanFHRDatabase() {
+  const FHR_DBNAME_PREF = "datareporting.healthreport.dbName";
+  const CUSTOM_DB_NAME = "unlikely.to.be.used.sqlite";
+  const DEFAULT_DB_NAME = "healthreport.sqlite";
+
+  // Check that we're able to remove a FHR DB with a custom name.
+  const CUSTOM_DB_PATHS = [
+    OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME),
+    OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME + "-wal"),
+    OS.Path.join(OS.Constants.Path.profileDir, CUSTOM_DB_NAME + "-shm"),
+  ];
+  Preferences.set(FHR_DBNAME_PREF, CUSTOM_DB_NAME);
+
+  // Write fake DB files to the profile directory.
+  for (let dbFilePath of CUSTOM_DB_PATHS) {
+    yield OS.File.writeAtomic(dbFilePath, "some data");
+  }
+
+  // Trigger the cleanup and check that the files were removed.
+  yield TelemetryStorage.removeFHRDatabase();
+  for (let dbFilePath of CUSTOM_DB_PATHS) {
+    Assert.ok(!(yield OS.File.exists(dbFilePath)), "The DB must not be on the disk anymore: " + dbFilePath);
+  }
+
+  // We should not break anything if there's no DB file.
+  yield TelemetryStorage.removeFHRDatabase();
+
+  // Check that we're able to remove a FHR DB with the default name.
+  Preferences.reset(FHR_DBNAME_PREF);
+
+  const DEFAULT_DB_PATHS = [
+    OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME),
+    OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME + "-wal"),
+    OS.Path.join(OS.Constants.Path.profileDir, DEFAULT_DB_NAME + "-shm"),
+  ];
+
+  // Write fake DB files to the profile directory.
+  for (let dbFilePath of DEFAULT_DB_PATHS) {
+    yield OS.File.writeAtomic(dbFilePath, "some data");
+  }
+
+  // Trigger the cleanup and check that the files were removed.
+  yield TelemetryStorage.removeFHRDatabase();
+  for (let dbFilePath of DEFAULT_DB_PATHS) {
+    Assert.ok(!(yield OS.File.exists(dbFilePath)), "The DB must not be on the disk anymore: " + dbFilePath);
+  }
+});
+
+add_task(function* stopServer() {
+  yield PingServer.stop();
+});

toolkit/components/telemetry/tests/unit/test_TelemetryControllerBuildID.js

@@ -0,0 +1,70 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+/* Test inclusion of previous build ID in telemetry pings when build ID changes.
+ * bug 841028
+ *
+ * Cases to cover:
+ * 1) Run with no "previousBuildID" stored in prefs:
+ *     -> no previousBuildID in telemetry system info, new value set in prefs.
+ * 2) previousBuildID in prefs, equal to current build ID:
+ *     -> no previousBuildID in telemetry, prefs not updated.
+ * 3) previousBuildID in prefs, not equal to current build ID:
+ *     -> previousBuildID in telemetry, new value set in prefs.
+ */
+
+"use strict";
+
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+// Force the Telemetry enabled preference so that TelemetrySession.testReset() doesn't exit early.
+Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+
+// Set up our dummy AppInfo object so we can control the appBuildID.
+Cu.import("resource://testing-common/AppInfo.jsm", this);
+updateAppInfo();
+
+// Check that when run with no previous build ID stored, we update the pref but do not
+// put anything into the metadata.
+add_task(function* test_firstRun() {
+  yield TelemetryController.testReset();
+  let metadata = TelemetrySession.getMetadata();
+  do_check_false("previousBuildID" in metadata);
+  let appBuildID = getAppInfo().appBuildID;
+  let buildIDPref = Services.prefs.getCharPref(TelemetrySession.Constants.PREF_PREVIOUS_BUILDID);
+  do_check_eq(appBuildID, buildIDPref);
+});
+
+// Check that a subsequent run with the same build ID does not put prev build ID in
+// metadata. Assumes testFirstRun() has already been called to set the previousBuildID pref.
+add_task(function* test_secondRun() {
+  yield TelemetryController.testReset();
+  let metadata = TelemetrySession.getMetadata();
+  do_check_false("previousBuildID" in metadata);
+});
+
+// Set up telemetry with a different app build ID and check that the old build ID
+// is returned in the metadata and the pref is updated to the new build ID.
+// Assumes testFirstRun() has been called to set the previousBuildID pref.
+const NEW_BUILD_ID = "20130314";
+add_task(function* test_newBuild() {
+  let info = getAppInfo();
+  let oldBuildID = info.appBuildID;
+  info.appBuildID = NEW_BUILD_ID;
+  yield TelemetryController.testReset();
+  let metadata = TelemetrySession.getMetadata();
+  do_check_eq(metadata.previousBuildId, oldBuildID);
+  let buildIDPref = Services.prefs.getCharPref(TelemetrySession.Constants.PREF_PREVIOUS_BUILDID);
+  do_check_eq(NEW_BUILD_ID, buildIDPref);
+});
+
+
+function run_test() {
+  // Make sure we have a profile directory.
+  do_get_profile();
+
+  run_next_test();
+}

toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js

@@ -0,0 +1,70 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Test that TelemetryController sends close to shutdown don't lead
+// to AsyncShutdown timeouts.
+
+"use strict";
+
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySend.jsm", this);
+Cu.import("resource://gre/modules/Timer.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://gre/modules/AsyncShutdown.jsm", this);
+Cu.import("resource://testing-common/httpd.js", this);
+
+const PREF_BRANCH = "toolkit.telemetry.";
+const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";
+
+function contentHandler(metadata, response)
+{
+  dump("contentHandler called for path: " + metadata._path + "\n");
+  // We intentionally don't finish writing the response here to let the
+  // client time out.
+  response.processAsync();
+  response.setHeader("Content-Type", "text/plain");
+}
+
+add_task(function* test_setup() {
+  // Addon manager needs a profile directory
+  do_get_profile();
+  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+  Services.prefs.setBoolPref(PREF_FHR_UPLOAD_ENABLED, true);
+});
+
+/**
+ * Ensures that TelemetryController does not hang processing shutdown
+ * phases. Assumes that Telemetry shutdown routines do not take longer than
+ * CRASH_TIMEOUT_MS to complete.
+ */
+add_task(function* test_sendTelemetryShutsDownWithinReasonableTimeout() {
+  const CRASH_TIMEOUT_MS = 5 * 1000;
+  // Enable testing mode for AsyncShutdown, otherwise some testing-only functionality
+  // is not available.
+  Services.prefs.setBoolPref("toolkit.asyncshutdown.testing", true);
+  // Reducing the max delay for waitiing on phases to complete from 1 minute
+  // (standard) to 10 seconds to avoid blocking the tests in case of misbehavior.
+  Services.prefs.setIntPref("toolkit.asyncshutdown.crash_timeout", CRASH_TIMEOUT_MS);
+
+  let httpServer = new HttpServer();
+  httpServer.registerPrefixHandler("/", contentHandler);
+  httpServer.start(-1);
+
+  yield TelemetryController.testSetup();
+  TelemetrySend.setServer("http://localhost:" + httpServer.identity.primaryPort);
+  let submissionPromise = TelemetryController.submitExternalPing("test-ping-type", {});
+
+  // Trigger the AsyncShutdown phase TelemetryController hangs off.
+  AsyncShutdown.profileBeforeChange._trigger();
+  AsyncShutdown.sendTelemetry._trigger();
+  // Now wait for the ping submission.
+  yield submissionPromise;
+
+  // If we get here, we didn't time out in the shutdown routines.
+  Assert.ok(true, "Didn't time out on shutdown.");
+});

toolkit/components/telemetry/tests/unit/test_TelemetryController_idle.js

@@ -0,0 +1,73 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Check that TelemetrySession notifies correctly on idle-daily.
+
+Cu.import("resource://testing-common/httpd.js", this);
+Cu.import("resource://gre/modules/PromiseUtils.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySend.jsm", this);
+
+const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";
+
+var gHttpServer = null;
+
+add_task(function* test_setup() {
+  do_get_profile();
+
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+  Services.prefs.setBoolPref(PREF_FHR_UPLOAD_ENABLED, true);
+
+  // Start the webserver to check if the pending ping correctly arrives.
+  gHttpServer = new HttpServer();
+  gHttpServer.start(-1);
+});
+
+add_task(function* testSendPendingOnIdleDaily() {
+  // Create a valid pending ping.
+  const PENDING_PING = {
+    id: "2133234d-4ea1-44f4-909e-ce8c6c41e0fc",
+    type: "test-ping",
+    version: 4,
+    application: {},
+    payload: {},
+  };
+  yield TelemetryStorage.savePing(PENDING_PING, true);
+
+  // Telemetry will not send this ping at startup, because it's not overdue.
+  yield TelemetryController.testSetup();
+  TelemetrySend.setServer("http://localhost:" + gHttpServer.identity.primaryPort);
+
+  let pendingPromise = new Promise(resolve =>
+    gHttpServer.registerPrefixHandler("/submit/telemetry/", request => resolve(request)));
+
+  let gatherPromise = PromiseUtils.defer();
+  Services.obs.addObserver(gatherPromise.resolve, "gather-telemetry", false);
+
+  // Check that we are correctly receiving the gather-telemetry notification.
+  TelemetrySession.observe(null, "idle-daily", null);
+  yield gatherPromise;
+  Assert.ok(true, "Received gather-telemetry notification.");
+
+  Services.obs.removeObserver(gatherPromise.resolve, "gather-telemetry");
+
+  // Check that the pending ping is correctly received.
+  let ns = {};
+  let module = Cu.import("resource://gre/modules/TelemetrySend.jsm", ns);
+  module.TelemetrySendImpl.observe(null, "idle-daily", null);
+  let request = yield pendingPromise;
+  let ping = decodeRequestPayload(request);
+
+  // Validate the ping data.
+  Assert.equal(ping.id, PENDING_PING.id);
+  Assert.equal(ping.type, PENDING_PING.type);
+
+  yield new Promise(resolve => gHttpServer.stop(resolve));
+});

toolkit/components/telemetry/tests/unit/test_TelemetryEvents.js

@@ -0,0 +1,249 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+const OPTIN = Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN;
+const OPTOUT = Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTOUT;
+
+function checkEventFormat(events) {
+  Assert.ok(Array.isArray(events), "Events should be serialized to an array.");
+  for (let e of events) {
+    Assert.ok(Array.isArray(e), "Event should be an array.");
+    Assert.greaterOrEqual(e.length, 4, "Event should have at least 4 elements.");
+    Assert.lessOrEqual(e.length, 6, "Event should have at most 6 elements.");
+
+    Assert.equal(typeof(e[0]), "number", "Element 0 should be a number.");
+    Assert.equal(typeof(e[1]), "string", "Element 1 should be a string.");
+    Assert.equal(typeof(e[2]), "string", "Element 2 should be a string.");
+    Assert.equal(typeof(e[3]), "string", "Element 3 should be a string.");
+
+    if (e.length > 4) {
+      Assert.ok(e[4] === null || typeof(e[4]) == "string",
+                "Event element 4 should be null or a string.");
+    }
+    if (e.length > 5) {
+      Assert.ok(e[5] === null || typeof(e[5]) == "object",
+                "Event element 5 should be null or an object.");
+    }
+
+    let extra = e[5];
+    if (extra) {
+      Assert.ok(Object.keys(extra).every(k => typeof(k) == "string"),
+                "All extra keys should be strings.");
+      Assert.ok(Object.values(extra).every(v => typeof(v) == "string"),
+                "All extra values should be strings.");
+    }
+  }
+}
+
+add_task(function* test_recording() {
+  Telemetry.clearEvents();
+
+  // Record some events.
+  let expected = [
+    {optout: false, event: ["telemetry.test", "test1", "object1"]},
+    {optout: false, event: ["telemetry.test", "test2", "object2"]},
+
+    {optout: false, event: ["telemetry.test", "test1", "object1", "value"]},
+    {optout: false, event: ["telemetry.test", "test1", "object1", "value", null]},
+    {optout: false, event: ["telemetry.test", "test1", "object1", null, {"key1": "value1"}]},
+    {optout: false, event: ["telemetry.test", "test1", "object1", "value", {"key1": "value1", "key2": "value2"}]},
+
+    {optout: true,  event: ["telemetry.test", "optout", "object1"]},
+    {optout: false, event: ["telemetry.test.second", "test", "object1"]},
+    {optout: false, event: ["telemetry.test.second", "test", "object1", null, {"key1": "value1"}]},
+  ];
+
+  for (let entry of expected) {
+    entry.tsBefore = Math.floor(Telemetry.msSinceProcessStart());
+    try {
+      Telemetry.recordEvent(...entry.event);
+    } catch (ex) {
+      Assert.ok(false, `Failed to record event ${JSON.stringify(entry.event)}: ${ex}`);
+    }
+    entry.tsAfter = Math.floor(Telemetry.msSinceProcessStart());
+  }
+
+  // Strip off trailing null values to match the serialized events.
+  for (let entry of expected) {
+    let e = entry.event;
+    while ((e.length >= 3) && (e[e.length - 1] === null)) {
+      e.pop();
+    }
+  }
+
+  // The following should not result in any recorded events.
+  Assert.throws(() => Telemetry.recordEvent("unknown.category", "test1", "object1"),
+                /Error: Unknown event: \["unknown.category", "test1", "object1"\]/,
+                "Should throw on unknown category.");
+  Assert.throws(() => Telemetry.recordEvent("telemetry.test", "unknown", "object1"),
+                /Error: Unknown event: \["telemetry.test", "unknown", "object1"\]/,
+                "Should throw on unknown method.");
+  Assert.throws(() => Telemetry.recordEvent("telemetry.test", "test1", "unknown"),
+                /Error: Unknown event: \["telemetry.test", "test1", "unknown"\]/,
+                "Should throw on unknown object.");
+
+  let checkEvents = (events, expectedEvents) => {
+    checkEventFormat(events);
+    Assert.equal(events.length, expectedEvents.length,
+                 "Snapshot should have the right number of events.");
+
+    for (let i = 0; i < events.length; ++i) {
+      let {tsBefore, tsAfter} = expectedEvents[i];
+      let ts = events[i][0];
+      Assert.greaterOrEqual(ts, tsBefore, "The recorded timestamp should be greater than the one before recording.");
+      Assert.lessOrEqual(ts, tsAfter, "The recorded timestamp should be less than the one after recording.");
+
+      let recordedData = events[i].slice(1);
+      let expectedData = expectedEvents[i].event.slice();
+      Assert.deepEqual(recordedData, expectedData, "The recorded event data should match.");
+    }
+  };
+
+  // Check that the expected events were recorded.
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, false);
+  checkEvents(events, expected);
+
+  // Check serializing only opt-out events.
+  events = Telemetry.snapshotBuiltinEvents(OPTOUT, false);
+  filtered = expected.filter(e => e.optout == true);
+  checkEvents(events, filtered);
+});
+
+add_task(function* test_clear() {
+  Telemetry.clearEvents();
+
+  const COUNT = 10;
+  for (let i = 0; i < COUNT; ++i) {
+    Telemetry.recordEvent("telemetry.test", "test1", "object1");
+    Telemetry.recordEvent("telemetry.test.second", "test", "object1");
+  }
+
+  // Check that events were recorded.
+  // The events are cleared by passing the respective flag.
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 2 * COUNT, `Should have recorded ${2 * COUNT} events.`);
+
+  // Now the events should be cleared.
+  events = Telemetry.snapshotBuiltinEvents(OPTIN, false);
+  Assert.equal(events.length, 0, `Should have cleared the events.`);
+});
+
+add_task(function* test_expiry() {
+  Telemetry.clearEvents();
+
+  // Recording call with event that is expired by version.
+  Telemetry.recordEvent("telemetry.test", "expired_version", "object1");
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 0, "Should not record event with expired version.");
+
+  // Recording call with event that is expired by date.
+  Telemetry.recordEvent("telemetry.test", "expired_date", "object1");
+  events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 0, "Should not record event with expired date.");
+
+  // Recording call with event that has expiry_version and expiry_date in the future.
+  Telemetry.recordEvent("telemetry.test", "not_expired_optout", "object1");
+  events = Telemetry.snapshotBuiltinEvents(OPTOUT, true);
+  Assert.equal(events.length, 1, "Should record event when date and version are not expired.");
+});
+
+add_task(function* test_invalidParams() {
+  Telemetry.clearEvents();
+
+  // Recording call with wrong type for value argument.
+  Telemetry.recordEvent("telemetry.test", "test1", "object1", 1);
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 0, "Should not record event when value argument with invalid type is passed.");
+
+  // Recording call with wrong type for extra argument.
+  Telemetry.recordEvent("telemetry.test", "test1", "object1", null, "invalid");
+  events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 0, "Should not record event when extra argument with invalid type is passed.");
+
+  // Recording call with unknown extra key.
+  Telemetry.recordEvent("telemetry.test", "test1", "object1", null, {"key3": "x"});
+  events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 0, "Should not record event when extra argument with invalid key is passed.");
+
+  // Recording call with invalid value type.
+  Telemetry.recordEvent("telemetry.test", "test1", "object1", null, {"key3": 1});
+  events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 0, "Should not record event when extra argument with invalid value type is passed.");
+});
+
+add_task(function* test_storageLimit() {
+  Telemetry.clearEvents();
+
+  // Record more events than the storage limit allows.
+  let LIMIT = 1000;
+  let COUNT = LIMIT + 10;
+  for (let i = 0; i < COUNT; ++i) {
+    Telemetry.recordEvent("telemetry.test", "test1", "object1", String(i));
+  }
+
+  // Check that the right events were recorded.
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, LIMIT, `Should have only recorded ${LIMIT} events`);
+  Assert.ok(events.every((e, idx) => e[4] === String(idx)),
+            "Should have recorded all events from before hitting the limit.");
+});
+
+add_task(function* test_valueLimits() {
+  Telemetry.clearEvents();
+
+  // Record values that are at or over the limits for string lengths.
+  let LIMIT = 80;
+  let expected = [
+    ["telemetry.test", "test1", "object1", "a".repeat(LIMIT - 10), null],
+    ["telemetry.test", "test1", "object1", "a".repeat(LIMIT     ), null],
+    ["telemetry.test", "test1", "object1", "a".repeat(LIMIT +  1), null],
+    ["telemetry.test", "test1", "object1", "a".repeat(LIMIT + 10), null],
+
+    ["telemetry.test", "test1", "object1", null, {key1: "a".repeat(LIMIT - 10)}],
+    ["telemetry.test", "test1", "object1", null, {key1: "a".repeat(LIMIT     )}],
+    ["telemetry.test", "test1", "object1", null, {key1: "a".repeat(LIMIT +  1)}],
+    ["telemetry.test", "test1", "object1", null, {key1: "a".repeat(LIMIT + 10)}],
+  ];
+
+  for (let event of expected) {
+    Telemetry.recordEvent(...event);
+    if (event[3]) {
+      event[3] = event[3].substr(0, LIMIT);
+    }
+    if (event[4]) {
+      event[4].key1 = event[4].key1.substr(0, LIMIT);
+    }
+  }
+
+  // Strip off trailing null values to match the serialized events.
+  for (let e of expected) {
+    while ((e.length >= 3) && (e[e.length - 1] === null)) {
+      e.pop();
+    }
+  }
+
+  // Check that the right events were recorded.
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, expected.length,
+               "Should have recorded the expected number of events");
+  for (let i = 0; i < expected.length; ++i) {
+    Assert.deepEqual(events[i].slice(1), expected[i],
+                     "Should have recorded the expected event data.");
+  }
+});
+
+add_task(function* test_unicodeValues() {
+  Telemetry.clearEvents();
+
+  // Record string values containing unicode characters.
+  let value = "漢語";
+  Telemetry.recordEvent("telemetry.test", "test1", "object1", value);
+  Telemetry.recordEvent("telemetry.test", "test1", "object1", null, {"key1": value});
+
+  // Check that the values were correctly recorded.
+  let events = Telemetry.snapshotBuiltinEvents(OPTIN, true);
+  Assert.equal(events.length, 2, "Should have recorded 2 events.");
+  Assert.equal(events[0][4], value, "Should have recorded the right value.");
+  Assert.equal(events[1][5]["key1"], value, "Should have recorded the right extra value.");
+});

toolkit/components/telemetry/tests/unit/test_TelemetryFlagClear.js

@@ -0,0 +1,14 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+function run_test()
+{
+  let testFlag = Services.telemetry.getHistogramById("TELEMETRY_TEST_FLAG");
+  equal(JSON.stringify(testFlag.snapshot().counts), "[1,0,0]", "Original value is correct");
+  testFlag.add(1);
+  equal(JSON.stringify(testFlag.snapshot().counts), "[0,1,0]", "Value is correct after ping.");
+  testFlag.clear();
+  equal(JSON.stringify(testFlag.snapshot().counts), "[1,0,0]", "Value is correct after calling clear()");
+  testFlag.add(1);
+  equal(JSON.stringify(testFlag.snapshot().counts), "[0,1,0]", "Value is correct after ping.");
+}

toolkit/components/telemetry/tests/unit/test_TelemetryLateWrites.js

@@ -0,0 +1,127 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+/* A testcase to make sure reading late writes stacks works.  */
+
+Cu.import("resource://gre/modules/Services.jsm", this);
+
+// Constants from prio.h for nsIFileOutputStream.init
+const PR_WRONLY = 0x2;
+const PR_CREATE_FILE = 0x8;
+const PR_TRUNCATE = 0x20;
+const RW_OWNER = parseInt("0600", 8);
+
+const STACK_SUFFIX1 = "stack1.txt";
+const STACK_SUFFIX2 = "stack2.txt";
+const STACK_BOGUS_SUFFIX = "bogus.txt";
+const LATE_WRITE_PREFIX = "Telemetry.LateWriteFinal-";
+
+// The names and IDs don't matter, but the format of the IDs does.
+const LOADED_MODULES = {
+  '4759A7E6993548C89CAF716A67EC242D00': 'libtest.so',
+  'F77AF15BB8D6419FA875954B4A3506CA00': 'libxul.so',
+  '1E2F7FB590424E8F93D60BB88D66B8C500': 'libc.so'
+};
+const N_MODULES = Object.keys(LOADED_MODULES).length;
+
+// Format of individual items is [index, offset-in-library].
+const STACK1 = [
+  [ 0, 0 ],
+  [ 1, 1 ],
+  [ 2, 2 ]
+];
+const STACK2 = [
+  [ 0, 0 ],
+  [ 1, 5 ],
+  [ 2, 10 ],
+];
+// XXX The only error checking is for a zero-sized stack.
+const STACK_BOGUS = [];
+
+function write_string_to_file(file, contents) {
+  let ostream = Cc["@mozilla.org/network/safe-file-output-stream;1"]
+                .createInstance(Ci.nsIFileOutputStream);
+  ostream.init(file, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE,
+	       RW_OWNER, ostream.DEFER_OPEN);
+  ostream.write(contents, contents.length);
+  ostream.QueryInterface(Ci.nsISafeOutputStream).finish();
+  ostream.close();
+}
+
+function construct_file(suffix) {
+  let profileDirectory = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  let file = profileDirectory.clone();
+  file.append(LATE_WRITE_PREFIX + suffix);
+  return file;
+}
+
+function write_late_writes_file(stack, suffix)
+{
+  let file = construct_file(suffix);
+  let contents = N_MODULES + "\n";
+  for (let id in LOADED_MODULES) {
+    contents += id + " " + LOADED_MODULES[id] + "\n";
+  }
+
+  contents += stack.length + "\n";
+  for (let element of stack) {
+    contents += element[0] + " " + element[1].toString(16) + "\n";
+  }
+
+  write_string_to_file(file, contents);
+}
+
+function run_test() {
+  do_get_profile();
+
+  write_late_writes_file(STACK1, STACK_SUFFIX1);
+  write_late_writes_file(STACK2, STACK_SUFFIX2);
+  write_late_writes_file(STACK_BOGUS, STACK_BOGUS_SUFFIX);
+
+  let lateWrites = Telemetry.lateWrites;
+  do_check_true("memoryMap" in lateWrites);
+  do_check_eq(lateWrites.memoryMap.length, 0);
+  do_check_true("stacks" in lateWrites);
+  do_check_eq(lateWrites.stacks.length, 0);
+
+  do_test_pending();
+  Telemetry.asyncFetchTelemetryData(function () {
+    actual_test();
+  });
+}
+
+function actual_test() {
+  do_check_false(construct_file(STACK_SUFFIX1).exists());
+  do_check_false(construct_file(STACK_SUFFIX2).exists());
+  do_check_false(construct_file(STACK_BOGUS_SUFFIX).exists());
+
+  let lateWrites = Telemetry.lateWrites;
+
+  do_check_true("memoryMap" in lateWrites);
+  do_check_eq(lateWrites.memoryMap.length, N_MODULES);
+  for (let id in LOADED_MODULES) {
+    let matchingLibrary = lateWrites.memoryMap.filter(function(library, idx, array) {
+                                                        return library[1] == id;
+                                                      });
+    do_check_eq(matchingLibrary.length, 1);
+    let library = matchingLibrary[0]
+    let name = library[0];
+    do_check_eq(LOADED_MODULES[id], name);
+  }
+
+  do_check_true("stacks" in lateWrites);
+  do_check_eq(lateWrites.stacks.length, 2);
+  let uneval_STACKS = [uneval(STACK1), uneval(STACK2)];
+  let first_stack = lateWrites.stacks[0];
+  let second_stack = lateWrites.stacks[1];
+  function stackChecker(canonicalStack) {
+    let unevalCanonicalStack = uneval(canonicalStack);
+    return function(obj, idx, array) {
+      return unevalCanonicalStack == obj;
+    }
+  }
+  do_check_eq(uneval_STACKS.filter(stackChecker(first_stack)).length, 1);
+  do_check_eq(uneval_STACKS.filter(stackChecker(second_stack)).length, 1);
+
+  do_test_finished();
+}

toolkit/components/telemetry/tests/unit/test_TelemetryLockCount.js

@@ -0,0 +1,53 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+/* A testcase to make sure reading the failed profile lock count works.  */
+
+Cu.import("resource://gre/modules/Services.jsm", this);
+
+const LOCK_FILE_NAME = "Telemetry.FailedProfileLocks.txt";
+const N_FAILED_LOCKS = 10;
+
+// Constants from prio.h for nsIFileOutputStream.init
+const PR_WRONLY = 0x2;
+const PR_CREATE_FILE = 0x8;
+const PR_TRUNCATE = 0x20;
+const RW_OWNER = parseInt("0600", 8);
+
+function write_string_to_file(file, contents) {
+  let ostream = Cc["@mozilla.org/network/safe-file-output-stream;1"]
+                .createInstance(Ci.nsIFileOutputStream);
+  ostream.init(file, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE,
+	       RW_OWNER, ostream.DEFER_OPEN);
+  ostream.write(contents, contents.length);
+  ostream.QueryInterface(Ci.nsISafeOutputStream).finish();
+  ostream.close();
+}
+
+function construct_file() {
+  let profileDirectory = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  let file = profileDirectory.clone();
+  file.append(LOCK_FILE_NAME);
+  return file;
+}
+
+function run_test() {
+  do_get_profile();
+
+  do_check_eq(Telemetry.failedProfileLockCount, 0);
+
+  write_string_to_file(construct_file(), N_FAILED_LOCKS.toString());
+
+  // Make sure that we're not eagerly reading the count now that the
+  // file exists.
+  do_check_eq(Telemetry.failedProfileLockCount, 0);
+
+  do_test_pending();
+  Telemetry.asyncFetchTelemetryData(actual_test);
+}
+
+function actual_test() {
+  do_check_eq(Telemetry.failedProfileLockCount, N_FAILED_LOCKS);
+  do_check_false(construct_file().exists());
+  do_test_finished();
+}

toolkit/components/telemetry/tests/unit/test_TelemetryLog.js

@@ -0,0 +1,51 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://gre/modules/TelemetryLog.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
+
+const TEST_PREFIX = "TEST-";
+const TEST_REGEX = new RegExp("^" + TEST_PREFIX);
+
+function check_event(event, id, data)
+{
+  do_print("Checking message " + id);
+  do_check_eq(event[0], id);
+  do_check_true(event[1] > 0);
+
+  if (data === undefined) {
+    do_check_true(event.length == 2);
+  } else {
+    do_check_eq(event.length, data.length + 2);
+    for (var i = 0; i < data.length; ++i) {
+      do_check_eq(typeof(event[i + 2]), "string");
+      do_check_eq(event[i + 2], data[i]);
+    }
+  }
+}
+
+add_task(function* ()
+{
+  do_get_profile();
+  // TODO: After Bug 1254550 lands we should not need to set the pref here.
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+  yield TelemetryController.testSetup();
+
+  TelemetryLog.log(TEST_PREFIX + "1", ["val", 123, undefined]);
+  TelemetryLog.log(TEST_PREFIX + "2", []);
+  TelemetryLog.log(TEST_PREFIX + "3");
+
+  var log = TelemetrySession.getPayload().log.filter(function(e) {
+    // Only want events that were generated by the test.
+    return TEST_REGEX.test(e[0]);
+  });
+
+  do_check_eq(log.length, 3);
+  check_event(log[0], TEST_PREFIX + "1", ["val", "123", "undefined"]);
+  check_event(log[1], TEST_PREFIX + "2", []);
+  check_event(log[2], TEST_PREFIX + "3", undefined);
+  do_check_true(log[0][1] <= log[1][1]);
+  do_check_true(log[1][1] <= log[2][1]);
+
+  yield TelemetryController.testShutdown();
+});

toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js

@@ -0,0 +1,268 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+// Test that TelemetryController sends close to shutdown don't lead
+// to AsyncShutdown timeouts.
+
+"use strict";
+
+Cu.import("resource://gre/modules/Preferences.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySend.jsm", this);
+Cu.import("resource://gre/modules/TelemetryReportingPolicy.jsm", this);
+Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);
+Cu.import("resource://gre/modules/Timer.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://gre/modules/UpdateUtils.jsm", this);
+
+const PREF_BRANCH = "toolkit.telemetry.";
+const PREF_SERVER = PREF_BRANCH + "server";
+
+const TEST_CHANNEL = "TestChannelABC";
+
+const PREF_POLICY_BRANCH = "datareporting.policy.";
+const PREF_BYPASS_NOTIFICATION = PREF_POLICY_BRANCH + "dataSubmissionPolicyBypassNotification";
+const PREF_DATA_SUBMISSION_ENABLED = PREF_POLICY_BRANCH + "dataSubmissionEnabled";
+const PREF_CURRENT_POLICY_VERSION = PREF_POLICY_BRANCH + "currentPolicyVersion";
+const PREF_MINIMUM_POLICY_VERSION = PREF_POLICY_BRANCH + "minimumPolicyVersion";
+const PREF_MINIMUM_CHANNEL_POLICY_VERSION = PREF_MINIMUM_POLICY_VERSION + ".channel-" + TEST_CHANNEL;
+const PREF_ACCEPTED_POLICY_VERSION = PREF_POLICY_BRANCH + "dataSubmissionPolicyAcceptedVersion";
+const PREF_ACCEPTED_POLICY_DATE = PREF_POLICY_BRANCH + "dataSubmissionPolicyNotifiedTime";
+
+function fakeShowPolicyTimeout(set, clear) {
+  let reportingPolicy = Cu.import("resource://gre/modules/TelemetryReportingPolicy.jsm");
+  reportingPolicy.Policy.setShowInfobarTimeout = set;
+  reportingPolicy.Policy.clearShowInfobarTimeout = clear;
+}
+
+function fakeResetAcceptedPolicy() {
+  Preferences.reset(PREF_ACCEPTED_POLICY_DATE);
+  Preferences.reset(PREF_ACCEPTED_POLICY_VERSION);
+}
+
+function setMinimumPolicyVersion(aNewPolicyVersion) {
+  const CHANNEL_NAME = UpdateUtils.getUpdateChannel(false);
+  // We might have channel-dependent minimum policy versions.
+  const CHANNEL_DEPENDENT_PREF = PREF_MINIMUM_POLICY_VERSION + ".channel-" + CHANNEL_NAME;
+
+  // Does the channel-dependent pref exist? If so, set its value.
+  if (Preferences.get(CHANNEL_DEPENDENT_PREF, undefined)) {
+    Preferences.set(CHANNEL_DEPENDENT_PREF, aNewPolicyVersion);
+    return;
+  }
+
+  // We don't have a channel specific minimu, so set the common one.
+  Preferences.set(PREF_MINIMUM_POLICY_VERSION, aNewPolicyVersion);
+}
+
+add_task(function* test_setup() {
+  // Addon manager needs a profile directory
+  do_get_profile(true);
+  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+  // Don't bypass the notifications in this test, we'll fake it.
+  Services.prefs.setBoolPref(PREF_BYPASS_NOTIFICATION, false);
+
+  TelemetryReportingPolicy.setup();
+});
+
+add_task(function* test_firstRun() {
+  const PREF_FIRST_RUN = "toolkit.telemetry.reportingpolicy.firstRun";
+  const FIRST_RUN_TIMEOUT_MSEC = 60 * 1000; // 60s
+  const OTHER_RUNS_TIMEOUT_MSEC = 10 * 1000; // 10s
+
+  Preferences.reset(PREF_FIRST_RUN);
+
+  let startupTimeout = 0;
+  fakeShowPolicyTimeout((callback, timeout) => startupTimeout = timeout, () => {});
+  TelemetryReportingPolicy.reset();
+
+  Services.obs.notifyObservers(null, "sessionstore-windows-restored", null);
+  Assert.equal(startupTimeout, FIRST_RUN_TIMEOUT_MSEC,
+               "The infobar display timeout should be 60s on the first run.");
+
+  // Run again, and check that we actually wait only 10 seconds.
+  TelemetryReportingPolicy.reset();
+  Services.obs.notifyObservers(null, "sessionstore-windows-restored", null);
+  Assert.equal(startupTimeout, OTHER_RUNS_TIMEOUT_MSEC,
+               "The infobar display timeout should be 10s on other runs.");
+});
+
+add_task(function* test_prefs() {
+  TelemetryReportingPolicy.reset();
+
+  let now = fakeNow(2009, 11, 18);
+
+  // If the date is not valid (earlier than 2012), we don't regard the policy as accepted.
+  TelemetryReportingPolicy.testInfobarShown();
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified());
+  Assert.equal(Preferences.get(PREF_ACCEPTED_POLICY_DATE, null), 0,
+               "Invalid dates should not make the policy accepted.");
+
+  // Check that the notification date and version are correctly saved to the prefs.
+  now = fakeNow(2012, 11, 18);
+  TelemetryReportingPolicy.testInfobarShown();
+  Assert.equal(Preferences.get(PREF_ACCEPTED_POLICY_DATE, null), now.getTime(),
+               "A valid date must correctly be saved.");
+
+  // Now that user is notified, check if we are allowed to upload.
+  Assert.ok(TelemetryReportingPolicy.canUpload(),
+            "We must be able to upload after the policy is accepted.");
+
+  // Disable submission and check that we're no longer allowed to upload.
+  Preferences.set(PREF_DATA_SUBMISSION_ENABLED, false);
+  Assert.ok(!TelemetryReportingPolicy.canUpload(),
+            "We must not be able to upload if data submission is disabled.");
+
+  // Turn the submission back on.
+  Preferences.set(PREF_DATA_SUBMISSION_ENABLED, true);
+  Assert.ok(TelemetryReportingPolicy.canUpload(),
+            "We must be able to upload if data submission is enabled and the policy was accepted.");
+
+  // Set a new minimum policy version and check that user is no longer notified.
+  let newMinimum = Preferences.get(PREF_CURRENT_POLICY_VERSION, 1) + 1;
+  setMinimumPolicyVersion(newMinimum);
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified(),
+            "A greater minimum policy version must invalidate the policy and disable upload.");
+
+  // Eventually accept the policy and make sure user is notified.
+  Preferences.set(PREF_CURRENT_POLICY_VERSION, newMinimum);
+  TelemetryReportingPolicy.testInfobarShown();
+  Assert.ok(TelemetryReportingPolicy.testIsUserNotified(),
+            "Accepting the policy again should show the user as notified.");
+  Assert.ok(TelemetryReportingPolicy.canUpload(),
+            "Accepting the policy again should let us upload data.");
+
+  // Set a new, per channel, minimum policy version. Start by setting a test current channel.
+  let defaultPrefs = new Preferences({ defaultBranch: true });
+  defaultPrefs.set("app.update.channel", TEST_CHANNEL);
+
+  // Increase and set the new minimum version, then check that we're not notified anymore.
+  newMinimum++;
+  Preferences.set(PREF_MINIMUM_CHANNEL_POLICY_VERSION, newMinimum);
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified(),
+            "Increasing the minimum policy version should invalidate the policy.");
+
+  // Eventually accept the policy and make sure user is notified.
+  Preferences.set(PREF_CURRENT_POLICY_VERSION, newMinimum);
+  TelemetryReportingPolicy.testInfobarShown();
+  Assert.ok(TelemetryReportingPolicy.testIsUserNotified(),
+            "Accepting the policy again should show the user as notified.");
+  Assert.ok(TelemetryReportingPolicy.canUpload(),
+            "Accepting the policy again should let us upload data.");
+});
+
+add_task(function* test_migratePrefs() {
+  const DEPRECATED_FHR_PREFS = {
+    "datareporting.policy.dataSubmissionPolicyAccepted": true,
+    "datareporting.policy.dataSubmissionPolicyBypassAcceptance": true,
+    "datareporting.policy.dataSubmissionPolicyResponseType": "foxyeah",
+    "datareporting.policy.dataSubmissionPolicyResponseTime": Date.now().toString(),
+  };
+
+  // Make sure the preferences are set before setting up the policy.
+  for (let name in DEPRECATED_FHR_PREFS) {
+    Preferences.set(name, DEPRECATED_FHR_PREFS[name]);
+  }
+  // Set up the policy.
+  TelemetryReportingPolicy.reset();
+  // They should have been removed by now.
+  for (let name in DEPRECATED_FHR_PREFS) {
+    Assert.ok(!Preferences.has(name), name + " should have been removed.");
+  }
+});
+
+add_task(function* test_userNotifiedOfCurrentPolicy() {
+  fakeResetAcceptedPolicy();
+  TelemetryReportingPolicy.reset();
+
+  // User should be reported as not notified by default.
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified(),
+            "The initial state should be unnotified.");
+
+  // Forcing a policy version should not automatically make the user notified.
+  Preferences.set(PREF_ACCEPTED_POLICY_VERSION,
+                  TelemetryReportingPolicy.DEFAULT_DATAREPORTING_POLICY_VERSION);
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified(),
+                 "The default state of the date should have a time of 0 and it should therefore fail");
+
+  // Showing the notification bar should make the user notified.
+  fakeNow(2012, 11, 11);
+  TelemetryReportingPolicy.testInfobarShown();
+  Assert.ok(TelemetryReportingPolicy.testIsUserNotified(),
+            "Using the proper API causes user notification to report as true.");
+
+  // It is assumed that later versions of the policy will incorporate previous
+  // ones, therefore this should also return true.
+  let newVersion =
+    Preferences.get(PREF_CURRENT_POLICY_VERSION, 1) + 1;
+  Preferences.set(PREF_ACCEPTED_POLICY_VERSION, newVersion);
+  Assert.ok(TelemetryReportingPolicy.testIsUserNotified(),
+            "A future version of the policy should pass.");
+
+  newVersion =
+    Preferences.get(PREF_CURRENT_POLICY_VERSION, 1) - 1;
+  Preferences.set(PREF_ACCEPTED_POLICY_VERSION, newVersion);
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified(),
+            "A previous version of the policy should fail.");
+});
+
+add_task(function* test_canSend() {
+  const TEST_PING_TYPE = "test-ping";
+
+  PingServer.start();
+  Preferences.set(PREF_SERVER, "http://localhost:" + PingServer.port);
+
+  yield TelemetryController.testReset();
+  TelemetryReportingPolicy.reset();
+
+  // User should be reported as not notified by default.
+  Assert.ok(!TelemetryReportingPolicy.testIsUserNotified(),
+            "The initial state should be unnotified.");
+
+  // Assert if we receive any ping before the policy is accepted.
+  PingServer.registerPingHandler(() => Assert.ok(false, "Should not have received any pings now"));
+  yield TelemetryController.submitExternalPing(TEST_PING_TYPE, {});
+
+  // Reset the ping handler.
+  PingServer.resetPingHandler();
+
+  // Fake the infobar: this should also trigger the ping send task.
+  TelemetryReportingPolicy.testInfobarShown();
+  let ping = yield PingServer.promiseNextPings(1);
+  Assert.equal(ping.length, 1, "We should have received one ping.");
+  Assert.equal(ping[0].type, TEST_PING_TYPE,
+               "We should have received the previous ping.");
+
+  // Submit another ping, to make sure it gets sent.
+  yield TelemetryController.submitExternalPing(TEST_PING_TYPE, {});
+
+  // Get the ping and check its type.
+  ping = yield PingServer.promiseNextPings(1);
+  Assert.equal(ping.length, 1, "We should have received one ping.");
+  Assert.equal(ping[0].type, TEST_PING_TYPE, "We should have received the new ping.");
+
+  // Fake a restart with a pending ping.
+  yield TelemetryController.addPendingPing(TEST_PING_TYPE, {});
+  yield TelemetryController.testReset();
+
+  // We should be immediately sending the ping out.
+  ping = yield PingServer.promiseNextPings(1);
+  Assert.equal(ping.length, 1, "We should have received one ping.");
+  Assert.equal(ping[0].type, TEST_PING_TYPE, "We should have received the pending ping.");
+
+  // Submit another ping, to make sure it gets sent.
+  yield TelemetryController.submitExternalPing(TEST_PING_TYPE, {});
+
+  // Get the ping and check its type.
+  ping = yield PingServer.promiseNextPings(1);
+  Assert.equal(ping.length, 1, "We should have received one ping.");
+  Assert.equal(ping[0].type, TEST_PING_TYPE, "We should have received the new ping.");
+
+  yield PingServer.stop();
+});

toolkit/components/telemetry/tests/unit/test_TelemetryScalars.js

@@ -0,0 +1,574 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+const UINT_SCALAR = "telemetry.test.unsigned_int_kind";
+const STRING_SCALAR = "telemetry.test.string_kind";
+const BOOLEAN_SCALAR = "telemetry.test.boolean_kind";
+const KEYED_UINT_SCALAR = "telemetry.test.keyed_unsigned_int";
+
+add_task(function* test_serializationFormat() {
+  Telemetry.clearScalars();
+
+  // Set the scalars to a known value.
+  const expectedUint = 3785;
+  const expectedString = "some value";
+  Telemetry.scalarSet(UINT_SCALAR, expectedUint);
+  Telemetry.scalarSet(STRING_SCALAR, expectedString);
+  Telemetry.scalarSet(BOOLEAN_SCALAR, true);
+  Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, "first_key", 1234);
+
+  // Get a snapshot of the scalars.
+  const scalars =
+    Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  // Check that they are serialized to the correct format.
+  Assert.equal(typeof(scalars[UINT_SCALAR]), "number",
+               UINT_SCALAR + " must be serialized to the correct format.");
+  Assert.ok(Number.isInteger(scalars[UINT_SCALAR]),
+               UINT_SCALAR + " must be a finite integer.");
+  Assert.equal(scalars[UINT_SCALAR], expectedUint,
+               UINT_SCALAR + " must have the correct value.");
+  Assert.equal(typeof(scalars[STRING_SCALAR]), "string",
+               STRING_SCALAR + " must be serialized to the correct format.");
+  Assert.equal(scalars[STRING_SCALAR], expectedString,
+               STRING_SCALAR + " must have the correct value.");
+  Assert.equal(typeof(scalars[BOOLEAN_SCALAR]), "boolean",
+               BOOLEAN_SCALAR + " must be serialized to the correct format.");
+  Assert.equal(scalars[BOOLEAN_SCALAR], true,
+               BOOLEAN_SCALAR + " must have the correct value.");
+  Assert.ok(!(KEYED_UINT_SCALAR in scalars),
+            "Keyed scalars must be reported in a separate section.");
+});
+
+add_task(function* test_keyedSerializationFormat() {
+  Telemetry.clearScalars();
+
+  const expectedKey = "first_key";
+  const expectedOtherKey = "漢語";
+  const expectedUint = 3785;
+  const expectedOtherValue = 1107;
+
+  Telemetry.scalarSet(UINT_SCALAR, expectedUint);
+  Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, expectedKey, expectedUint);
+  Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, expectedOtherKey, expectedOtherValue);
+
+  // Get a snapshot of the scalars.
+  const keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  Assert.ok(!(UINT_SCALAR in keyedScalars),
+            UINT_SCALAR + " must not be serialized with the keyed scalars.");
+  Assert.ok(KEYED_UINT_SCALAR in keyedScalars,
+            KEYED_UINT_SCALAR + " must be serialized with the keyed scalars.");
+  Assert.equal(Object.keys(keyedScalars[KEYED_UINT_SCALAR]).length, 2,
+               "The keyed scalar must contain exactly 2 keys.");
+  Assert.ok(expectedKey in keyedScalars[KEYED_UINT_SCALAR],
+            KEYED_UINT_SCALAR + " must contain the expected keys.");
+  Assert.ok(expectedOtherKey in keyedScalars[KEYED_UINT_SCALAR],
+            KEYED_UINT_SCALAR + " must contain the expected keys.");
+  Assert.ok(Number.isInteger(keyedScalars[KEYED_UINT_SCALAR][expectedKey]),
+               KEYED_UINT_SCALAR + "." + expectedKey + " must be a finite integer.");
+  Assert.equal(keyedScalars[KEYED_UINT_SCALAR][expectedKey], expectedUint,
+               KEYED_UINT_SCALAR + "." + expectedKey + " must have the correct value.");
+  Assert.equal(keyedScalars[KEYED_UINT_SCALAR][expectedOtherKey], expectedOtherValue,
+               KEYED_UINT_SCALAR + "." + expectedOtherKey + " must have the correct value.");
+});
+
+add_task(function* test_nonexistingScalar() {
+  const NON_EXISTING_SCALAR = "telemetry.test.non_existing";
+
+  Telemetry.clearScalars();
+
+  // Make sure we throw on any operation for non-existing scalars.
+  Assert.throws(() => Telemetry.scalarAdd(NON_EXISTING_SCALAR, 11715),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Adding to a non existing scalar must throw.");
+  Assert.throws(() => Telemetry.scalarSet(NON_EXISTING_SCALAR, 11715),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting a non existing scalar must throw.");
+  Assert.throws(() => Telemetry.scalarSetMaximum(NON_EXISTING_SCALAR, 11715),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting the maximum of a non existing scalar must throw.");
+
+  // Make sure we throw on any operation for non-existing scalars.
+  Assert.throws(() => Telemetry.keyedScalarAdd(NON_EXISTING_SCALAR, "some_key", 11715),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Adding to a non existing keyed scalar must throw.");
+  Assert.throws(() => Telemetry.keyedScalarSet(NON_EXISTING_SCALAR, "some_key", 11715),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting a non existing keyed scalar must throw.");
+  Assert.throws(() => Telemetry.keyedScalarSetMaximum(NON_EXISTING_SCALAR, "some_key", 11715),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting the maximum of a non keyed existing scalar must throw.");
+
+  // Get a snapshot of the scalars.
+  const scalars =
+    Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  Assert.ok(!(NON_EXISTING_SCALAR in scalars), "The non existing scalar must not be persisted.");
+
+  const keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  Assert.ok(!(NON_EXISTING_SCALAR in keyedScalars),
+            "The non existing keyed scalar must not be persisted.");
+});
+
+add_task(function* test_expiredScalar() {
+  const EXPIRED_SCALAR = "telemetry.test.expired";
+  const EXPIRED_KEYED_SCALAR = "telemetry.test.keyed_expired";
+  const UNEXPIRED_SCALAR = "telemetry.test.unexpired";
+
+  Telemetry.clearScalars();
+
+  // Try to set the expired scalar to some value. We will not be recording the value,
+  // but we shouldn't throw.
+  Telemetry.scalarAdd(EXPIRED_SCALAR, 11715);
+  Telemetry.scalarSet(EXPIRED_SCALAR, 11715);
+  Telemetry.scalarSetMaximum(EXPIRED_SCALAR, 11715);
+  Telemetry.keyedScalarAdd(EXPIRED_KEYED_SCALAR, "some_key", 11715);
+  Telemetry.keyedScalarSet(EXPIRED_KEYED_SCALAR, "some_key", 11715);
+  Telemetry.keyedScalarSetMaximum(EXPIRED_KEYED_SCALAR, "some_key", 11715);
+
+  // The unexpired scalar has an expiration version, but far away in the future.
+  const expectedValue = 11716;
+  Telemetry.scalarSet(UNEXPIRED_SCALAR, expectedValue);
+
+  // Get a snapshot of the scalars.
+  const scalars =
+    Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+  const keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  Assert.ok(!(EXPIRED_SCALAR in scalars), "The expired scalar must not be persisted.");
+  Assert.equal(scalars[UNEXPIRED_SCALAR], expectedValue,
+               "The unexpired scalar must be persisted with the correct value.");
+  Assert.ok(!(EXPIRED_KEYED_SCALAR in keyedScalars),
+            "The expired keyed scalar must not be persisted.");
+});
+
+add_task(function* test_unsignedIntScalar() {
+  let checkScalar = (expectedValue) => {
+    const scalars =
+      Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.equal(scalars[UINT_SCALAR], expectedValue,
+                 UINT_SCALAR + " must contain the expected value.");
+  };
+
+  Telemetry.clearScalars();
+
+  // Let's start with an accumulation without a prior set.
+  Telemetry.scalarAdd(UINT_SCALAR, 1);
+  Telemetry.scalarAdd(UINT_SCALAR, 2);
+  // Do we get what we expect?
+  checkScalar(3);
+
+  // Let's test setting the scalar to a value.
+  Telemetry.scalarSet(UINT_SCALAR, 3785);
+  checkScalar(3785);
+  Telemetry.scalarAdd(UINT_SCALAR, 1);
+  checkScalar(3786);
+
+  // Does setMaximum work?
+  Telemetry.scalarSet(UINT_SCALAR, 2);
+  checkScalar(2);
+  Telemetry.scalarSetMaximum(UINT_SCALAR, 5);
+  checkScalar(5);
+  // The value of the probe should still be 5, as the previous value
+  // is greater than the one we want to set.
+  Telemetry.scalarSetMaximum(UINT_SCALAR, 3);
+  checkScalar(5);
+
+  // Check that non-integer numbers get truncated and set.
+  Telemetry.scalarSet(UINT_SCALAR, 3.785);
+  checkScalar(3);
+
+  // Setting or adding a negative number must report an error through
+  // the console and drop the change (shouldn't throw).
+  Telemetry.scalarAdd(UINT_SCALAR, -5);
+  Telemetry.scalarSet(UINT_SCALAR, -5);
+  Telemetry.scalarSetMaximum(UINT_SCALAR, -1);
+  checkScalar(3);
+
+  // What happens if we try to set a value of a different type?
+  Telemetry.scalarSet(UINT_SCALAR, 1);
+  Assert.throws(() => Telemetry.scalarSet(UINT_SCALAR, "unexpected value"),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting the scalar to an unexpected value type must throw.");
+  Assert.throws(() => Telemetry.scalarAdd(UINT_SCALAR, "unexpected value"),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Adding an unexpected value type must throw.");
+  Assert.throws(() => Telemetry.scalarSetMaximum(UINT_SCALAR, "unexpected value"),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting the scalar to an unexpected value type must throw.");
+  // The stored value must not be compromised.
+  checkScalar(1);
+});
+
+add_task(function* test_stringScalar() {
+  let checkExpectedString = (expectedString) => {
+    const scalars =
+      Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.equal(scalars[STRING_SCALAR], expectedString,
+                 STRING_SCALAR + " must contain the expected string value.");
+  };
+
+  Telemetry.clearScalars();
+
+  // Let's check simple strings...
+  let expected = "test string";
+  Telemetry.scalarSet(STRING_SCALAR, expected);
+  checkExpectedString(expected);
+  expected = "漢語";
+  Telemetry.scalarSet(STRING_SCALAR, expected);
+  checkExpectedString(expected);
+
+  // We have some unsupported operations for strings.
+  Assert.throws(() => Telemetry.scalarAdd(STRING_SCALAR, 1),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarAdd(STRING_SCALAR, "string value"),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarSetMaximum(STRING_SCALAR, 1),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarSetMaximum(STRING_SCALAR, "string value"),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarSet(STRING_SCALAR, 1),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "The string scalar must throw if we're not setting a string.");
+
+  // Try to set the scalar to a string longer than the maximum length limit.
+  const LONG_STRING = "browser.qaxfiuosnzmhlg.rpvxicawolhtvmbkpnludhedobxvkjwqyeyvmv";
+  Telemetry.scalarSet(STRING_SCALAR, LONG_STRING);
+  checkExpectedString(LONG_STRING.substr(0, 50));
+});
+
+add_task(function* test_booleanScalar() {
+  let checkExpectedBool = (expectedBoolean) => {
+    const scalars =
+      Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.equal(scalars[BOOLEAN_SCALAR], expectedBoolean,
+                 BOOLEAN_SCALAR + " must contain the expected boolean value.");
+  };
+
+  Telemetry.clearScalars();
+
+  // Set a test boolean value.
+  let expected = false;
+  Telemetry.scalarSet(BOOLEAN_SCALAR, expected);
+  checkExpectedBool(expected);
+  expected = true;
+  Telemetry.scalarSet(BOOLEAN_SCALAR, expected);
+  checkExpectedBool(expected);
+
+  // Check that setting a numeric value implicitly converts to boolean.
+  Telemetry.scalarSet(BOOLEAN_SCALAR, 1);
+  checkExpectedBool(true);
+  Telemetry.scalarSet(BOOLEAN_SCALAR, 0);
+  checkExpectedBool(false);
+  Telemetry.scalarSet(BOOLEAN_SCALAR, 1.0);
+  checkExpectedBool(true);
+  Telemetry.scalarSet(BOOLEAN_SCALAR, 0.0);
+  checkExpectedBool(false);
+
+  // Check that unsupported operations for booleans throw.
+  Assert.throws(() => Telemetry.scalarAdd(BOOLEAN_SCALAR, 1),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarAdd(BOOLEAN_SCALAR, "string value"),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarSetMaximum(BOOLEAN_SCALAR, 1),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarSetMaximum(BOOLEAN_SCALAR, "string value"),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+  Assert.throws(() => Telemetry.scalarSet(BOOLEAN_SCALAR, "true"),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "The boolean scalar must throw if we're not setting a boolean.");
+});
+
+add_task(function* test_scalarRecording() {
+  const OPTIN_SCALAR = "telemetry.test.release_optin";
+  const OPTOUT_SCALAR = "telemetry.test.release_optout";
+
+  let checkValue = (scalarName, expectedValue) => {
+    const scalars =
+      Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.equal(scalars[scalarName], expectedValue,
+                 scalarName + " must contain the expected value.");
+  };
+
+  let checkNotSerialized = (scalarName) => {
+    const scalars =
+      Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.ok(!(scalarName in scalars), scalarName + " was not recorded.");
+  };
+
+  Telemetry.canRecordBase = false;
+  Telemetry.canRecordExtended = false;
+  Telemetry.clearScalars();
+
+  // Check that no scalar is recorded if both base and extended recording are off.
+  Telemetry.scalarSet(OPTOUT_SCALAR, 3);
+  Telemetry.scalarSet(OPTIN_SCALAR, 3);
+  checkNotSerialized(OPTOUT_SCALAR);
+  checkNotSerialized(OPTIN_SCALAR);
+
+  // Check that opt-out scalars are recorded, while opt-in are not.
+  Telemetry.canRecordBase = true;
+  Telemetry.scalarSet(OPTOUT_SCALAR, 3);
+  Telemetry.scalarSet(OPTIN_SCALAR, 3);
+  checkValue(OPTOUT_SCALAR, 3);
+  checkNotSerialized(OPTIN_SCALAR);
+
+  // Check that both opt-out and opt-in scalars are recorded.
+  Telemetry.canRecordExtended = true;
+  Telemetry.scalarSet(OPTOUT_SCALAR, 5);
+  Telemetry.scalarSet(OPTIN_SCALAR, 6);
+  checkValue(OPTOUT_SCALAR, 5);
+  checkValue(OPTIN_SCALAR, 6);
+});
+
+add_task(function* test_keyedScalarRecording() {
+  const OPTIN_SCALAR = "telemetry.test.keyed_release_optin";
+  const OPTOUT_SCALAR = "telemetry.test.keyed_release_optout";
+  const testKey = "policy_key";
+
+  let checkValue = (scalarName, expectedValue) => {
+    const scalars =
+      Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.equal(scalars[scalarName][testKey], expectedValue,
+                 scalarName + " must contain the expected value.");
+  };
+
+  let checkNotSerialized = (scalarName) => {
+    const scalars =
+      Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    Assert.ok(!(scalarName in scalars), scalarName + " was not recorded.");
+  };
+
+  Telemetry.canRecordBase = false;
+  Telemetry.canRecordExtended = false;
+  Telemetry.clearScalars();
+
+  // Check that no scalar is recorded if both base and extended recording are off.
+  Telemetry.keyedScalarSet(OPTOUT_SCALAR, testKey, 3);
+  Telemetry.keyedScalarSet(OPTIN_SCALAR, testKey, 3);
+  checkNotSerialized(OPTOUT_SCALAR);
+  checkNotSerialized(OPTIN_SCALAR);
+
+  // Check that opt-out scalars are recorded, while opt-in are not.
+  Telemetry.canRecordBase = true;
+  Telemetry.keyedScalarSet(OPTOUT_SCALAR, testKey, 3);
+  Telemetry.keyedScalarSet(OPTIN_SCALAR, testKey, 3);
+  checkValue(OPTOUT_SCALAR, 3);
+  checkNotSerialized(OPTIN_SCALAR);
+
+  // Check that both opt-out and opt-in scalars are recorded.
+  Telemetry.canRecordExtended = true;
+  Telemetry.keyedScalarSet(OPTOUT_SCALAR, testKey, 5);
+  Telemetry.keyedScalarSet(OPTIN_SCALAR, testKey, 6);
+  checkValue(OPTOUT_SCALAR, 5);
+  checkValue(OPTIN_SCALAR, 6);
+});
+
+add_task(function* test_subsession() {
+  Telemetry.clearScalars();
+
+  // Set the scalars to a known value.
+  Telemetry.scalarSet(UINT_SCALAR, 3785);
+  Telemetry.scalarSet(STRING_SCALAR, "some value");
+  Telemetry.scalarSet(BOOLEAN_SCALAR, false);
+  Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, "some_random_key", 12);
+
+  // Get a snapshot and reset the subsession. The value we set must be there.
+  let scalars =
+    Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true);
+  let keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true);
+
+  Assert.equal(scalars[UINT_SCALAR], 3785,
+               UINT_SCALAR + " must contain the expected value.");
+  Assert.equal(scalars[STRING_SCALAR], "some value",
+               STRING_SCALAR + " must contain the expected value.");
+  Assert.equal(scalars[BOOLEAN_SCALAR], false,
+               BOOLEAN_SCALAR + " must contain the expected value.");
+  Assert.equal(keyedScalars[KEYED_UINT_SCALAR]["some_random_key"], 12,
+               KEYED_UINT_SCALAR + " must contain the expected value.");
+
+  // Get a new snapshot and reset the subsession again. Since no new value
+  // was set, the scalars should not be reported.
+  scalars =
+    Telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true);
+  keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, true);
+
+  Assert.ok(!(UINT_SCALAR in scalars), UINT_SCALAR + " must be empty and not reported.");
+  Assert.ok(!(STRING_SCALAR in scalars), STRING_SCALAR + " must be empty and not reported.");
+  Assert.ok(!(BOOLEAN_SCALAR in scalars), BOOLEAN_SCALAR + " must be empty and not reported.");
+  Assert.ok(!(KEYED_UINT_SCALAR in keyedScalars), KEYED_UINT_SCALAR + " must be empty and not reported.");
+});
+
+add_task(function* test_keyed_uint() {
+  Telemetry.clearScalars();
+
+  const KEYS = [ "a_key", "another_key", "third_key" ];
+  let expectedValues = [ 1, 1, 1 ];
+
+  // Set all the keys to a baseline value.
+  for (let key of KEYS) {
+    Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, key, 1);
+  }
+
+  // Increment only one key.
+  Telemetry.keyedScalarAdd(KEYED_UINT_SCALAR, KEYS[1], 1);
+  expectedValues[1]++;
+
+  // Use SetMaximum on the third key.
+  Telemetry.keyedScalarSetMaximum(KEYED_UINT_SCALAR, KEYS[2], 37);
+  expectedValues[2] = 37;
+
+  // Get a snapshot of the scalars and make sure the keys contain
+  // the correct values.
+  const keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  for (let k = 0; k < 3; k++) {
+    const keyName = KEYS[k];
+    Assert.equal(keyedScalars[KEYED_UINT_SCALAR][keyName], expectedValues[k],
+                 KEYED_UINT_SCALAR + "." + keyName + " must contain the correct value.");
+  }
+
+  // Are we still throwing when doing unsupported things on uint keyed scalars?
+  // Just test one single unsupported operation, the other are covered in the plain
+  // unsigned scalar test.
+  Assert.throws(() => Telemetry.scalarSet(KEYED_UINT_SCALAR, "new_key", "unexpected value"),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Setting the scalar to an unexpected value type must throw.");
+});
+
+add_task(function* test_keyed_boolean() {
+  Telemetry.clearScalars();
+
+  const KEYED_BOOLEAN_TYPE = "telemetry.test.keyed_boolean_kind";
+  const first_key = "first_key";
+  const second_key = "second_key";
+
+  // Set the initial values.
+  Telemetry.keyedScalarSet(KEYED_BOOLEAN_TYPE, first_key, true);
+  Telemetry.keyedScalarSet(KEYED_BOOLEAN_TYPE, second_key, false);
+
+  // Get a snapshot of the scalars and make sure the keys contain
+  // the correct values.
+  let keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+  Assert.equal(keyedScalars[KEYED_BOOLEAN_TYPE][first_key], true,
+               "The key must contain the expected value.");
+  Assert.equal(keyedScalars[KEYED_BOOLEAN_TYPE][second_key], false,
+               "The key must contain the expected value.");
+
+  // Now flip the values and make sure we get the expected values back.
+  Telemetry.keyedScalarSet(KEYED_BOOLEAN_TYPE, first_key, false);
+  Telemetry.keyedScalarSet(KEYED_BOOLEAN_TYPE, second_key, true);
+
+  keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+  Assert.equal(keyedScalars[KEYED_BOOLEAN_TYPE][first_key], false,
+               "The key must contain the expected value.");
+  Assert.equal(keyedScalars[KEYED_BOOLEAN_TYPE][second_key], true,
+               "The key must contain the expected value.");
+
+  // Are we still throwing when doing unsupported things on a boolean keyed scalars?
+  // Just test one single unsupported operation, the other are covered in the plain
+  // boolean scalar test.
+  Assert.throws(() => Telemetry.keyedScalarAdd(KEYED_BOOLEAN_TYPE, "somehey", 1),
+                /NS_ERROR_NOT_AVAILABLE/,
+                "Using an unsupported operation must throw.");
+});
+
+add_task(function* test_keyed_keys_length() {
+  Telemetry.clearScalars();
+
+  const LONG_KEY_STRING =
+    "browser.qaxfiuosnzmhlg.rpvxicawolhtvmbkpnludhedobxvkjwqyeyvmv.somemoresowereach70chars";
+  const NORMAL_KEY = "a_key";
+
+  // Set the value for a key within the length limits.
+  Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, NORMAL_KEY, 1);
+
+  // Now try to set and modify the value for a very long key.
+  Assert.throws(() => Telemetry.keyedScalarAdd(KEYED_UINT_SCALAR, LONG_KEY_STRING, 10),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Using keys longer than 70 characters must throw.");
+  Assert.throws(() => Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, LONG_KEY_STRING, 1),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Using keys longer than 70 characters must throw.");
+  Assert.throws(() => Telemetry.keyedScalarSetMaximum(KEYED_UINT_SCALAR, LONG_KEY_STRING, 10),
+                /NS_ERROR_ILLEGAL_VALUE/,
+                "Using keys longer than 70 characters must throw.");
+
+  // Make sure the key with the right length contains the expected value.
+  let keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+  Assert.equal(Object.keys(keyedScalars[KEYED_UINT_SCALAR]).length, 1,
+               "The keyed scalar must contain exactly 1 key.");
+  Assert.ok(NORMAL_KEY in keyedScalars[KEYED_UINT_SCALAR],
+            "The keyed scalar must contain the expected key.");
+  Assert.equal(keyedScalars[KEYED_UINT_SCALAR][NORMAL_KEY], 1,
+               "The key must contain the expected value.");
+  Assert.ok(!(LONG_KEY_STRING in keyedScalars[KEYED_UINT_SCALAR]),
+            "The data for the long key should not have been recorded.");
+});
+
+add_task(function* test_keyed_max_keys() {
+  Telemetry.clearScalars();
+
+  // Generate the names for the first 100 keys.
+  let keyNamesSet = new Set();
+  for (let k = 0; k < 100; k++) {
+    keyNamesSet.add("key_" + k);
+  }
+
+  // Add 100 keys to an histogram and set their initial value.
+  let valueToSet = 0;
+  keyNamesSet.forEach(keyName => {
+    Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, keyName, valueToSet++);
+  });
+
+  // Perform some operations on the 101th key. This should throw, as
+  // we're not allowed to have more than 100 keys.
+  const LAST_KEY_NAME = "overflowing_key";
+  Assert.throws(() => Telemetry.keyedScalarAdd(KEYED_UINT_SCALAR, LAST_KEY_NAME, 10),
+                /NS_ERROR_FAILURE/,
+                "Using more than 100 keys must throw.");
+  Assert.throws(() => Telemetry.keyedScalarSet(KEYED_UINT_SCALAR, LAST_KEY_NAME, 1),
+                /NS_ERROR_FAILURE/,
+                "Using more than 100 keys must throw.");
+  Assert.throws(() => Telemetry.keyedScalarSetMaximum(KEYED_UINT_SCALAR, LAST_KEY_NAME, 10),
+                /NS_ERROR_FAILURE/,
+                "Using more than 100 keys must throw.");
+
+  // Make sure all the keys except the last one are available and have the correct
+  // values.
+  let keyedScalars =
+    Telemetry.snapshotKeyedScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+
+  // Check that the keyed scalar only contain the first 100 keys.
+  const reportedKeysSet = new Set(Object.keys(keyedScalars[KEYED_UINT_SCALAR]));
+  Assert.ok([...keyNamesSet].filter(x => reportedKeysSet.has(x)) &&
+            [...reportedKeysSet].filter(x => keyNamesSet.has(x)),
+            "The keyed scalar must contain all the 100 keys, and drop the others.");
+
+  // Check that all the keys recorded the expected values.
+  let expectedValue = 0;
+  keyNamesSet.forEach(keyName => {
+    Assert.equal(keyedScalars[KEYED_UINT_SCALAR][keyName], expectedValue++,
+                 "The key must contain the expected value.");
+  });
+});

toolkit/components/telemetry/tests/unit/test_TelemetrySend.js

@@ -0,0 +1,427 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+*/
+
+// This tests the public Telemetry API for submitting pings.
+
+"use strict";
+
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySend.jsm", this);
+Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/Preferences.jsm", this);
+Cu.import("resource://gre/modules/osfile.jsm", this);
+
+const PREF_TELEMETRY_SERVER = "toolkit.telemetry.server";
+
+const MS_IN_A_MINUTE = 60 * 1000;
+
+function countPingTypes(pings) {
+  let countByType = new Map();
+  for (let p of pings) {
+    countByType.set(p.type, 1 + (countByType.get(p.type) || 0));
+  }
+  return countByType;
+}
+
+function setPingLastModified(id, timestamp) {
+  const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, id);
+  return OS.File.setDates(path, null, timestamp);
+}
+
+// Mock out the send timer activity.
+function waitForTimer() {
+  return new Promise(resolve => {
+    fakePingSendTimer((callback, timeout) => {
+      resolve([callback, timeout]);
+    }, () => {});
+  });
+}
+
+// Allow easy faking of readable ping ids.
+// This helps with debugging issues with e.g. ordering in the send logic.
+function fakePingId(type, number) {
+  const HEAD = "93bd0011-2c8f-4e1c-bee0-";
+  const TAIL = "000000000000";
+  const N = String(number);
+  const id = HEAD + type + TAIL.slice(type.length, - N.length) + N;
+  fakeGeneratePingId(() => id);
+  return id;
+}
+
+var checkPingsSaved = Task.async(function* (pingIds) {
+  let allFound = true;
+  for (let id of pingIds) {
+    const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, id);
+    let exists = false;
+    try {
+      exists = yield OS.File.exists(path);
+    } catch (ex) {}
+
+    if (!exists) {
+      dump("checkPingsSaved - failed to find ping: " + path + "\n");
+      allFound = false;
+    }
+  }
+
+  return allFound;
+});
+
+function histogramValueCount(h) {
+  return h.counts.reduce((a, b) => a + b);
+}
+
+add_task(function* test_setup() {
+  // Trigger a proper telemetry init.
+  do_get_profile(true);
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+});
+
+// Test the ping sending logic.
+add_task(function* test_sendPendingPings() {
+  const TYPE_PREFIX = "test-sendPendingPings-";
+  const TEST_TYPE_A = TYPE_PREFIX + "A";
+  const TEST_TYPE_B = TYPE_PREFIX + "B";
+
+  const TYPE_A_COUNT = 20;
+  const TYPE_B_COUNT = 5;
+
+  let histSuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
+  let histSendTimeSuccess = Telemetry.getHistogramById("TELEMETRY_SEND_SUCCESS");
+  let histSendTimeFail = Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE");
+  histSuccess.clear();
+  histSendTimeSuccess.clear();
+  histSendTimeFail.clear();
+
+  // Fake a current date.
+  let now = TelemetryUtils.truncateToDays(new Date());
+  now = fakeNow(futureDate(now, 10 * 60 * MS_IN_A_MINUTE));
+
+  // Enable test-mode for TelemetrySend, otherwise we won't store pending pings
+  // before the module is fully initialized later.
+  TelemetrySend.setTestModeEnabled(true);
+
+  // Submit some pings without the server and telemetry started yet.
+  for (let i = 0; i < TYPE_A_COUNT; ++i) {
+    fakePingId("a", i);
+    const id = yield TelemetryController.submitExternalPing(TEST_TYPE_A, {});
+    yield setPingLastModified(id, now.getTime() + (i * 1000));
+  }
+
+  Assert.equal(TelemetrySend.pendingPingCount, TYPE_A_COUNT,
+               "Should have correct pending ping count");
+
+  // Submit some more pings of a different type.
+  now = fakeNow(futureDate(now, 5 * MS_IN_A_MINUTE));
+  for (let i = 0; i < TYPE_B_COUNT; ++i) {
+    fakePingId("b", i);
+    const id = yield TelemetryController.submitExternalPing(TEST_TYPE_B, {});
+    yield setPingLastModified(id, now.getTime() + (i * 1000));
+  }
+
+  Assert.equal(TelemetrySend.pendingPingCount, TYPE_A_COUNT + TYPE_B_COUNT,
+               "Should have correct pending ping count");
+
+  Assert.deepEqual(histSuccess.snapshot().counts, [0, 0, 0],
+               "Should not have recorded any sending in histograms yet.");
+  Assert.equal(histSendTimeSuccess.snapshot().sum, 0,
+               "Should not have recorded any sending in histograms yet.");
+  Assert.equal(histSendTimeFail.snapshot().sum, 0,
+               "Should not have recorded any sending in histograms yet.");
+
+  // Now enable sending to the ping server.
+  now = fakeNow(futureDate(now, MS_IN_A_MINUTE));
+  PingServer.start();
+  Preferences.set(PREF_TELEMETRY_SERVER, "http://localhost:" + PingServer.port);
+
+  let timerPromise = waitForTimer();
+  yield TelemetryController.testReset();
+  let [pingSendTimerCallback, pingSendTimeout] = yield timerPromise;
+  Assert.ok(!!pingSendTimerCallback, "Should have a timer callback");
+
+  // We should have received 10 pings from the first send batch:
+  // 5 of type B and 5 of type A, as sending is newest-first.
+  // The other pings should be delayed by the 10-pings-per-minute limit.
+  let pings = yield PingServer.promiseNextPings(10);
+  Assert.equal(TelemetrySend.pendingPingCount, TYPE_A_COUNT - 5,
+               "Should have correct pending ping count");
+  PingServer.registerPingHandler(() => Assert.ok(false, "Should not have received any pings now"));
+  let countByType = countPingTypes(pings);
+
+  Assert.equal(countByType.get(TEST_TYPE_B), TYPE_B_COUNT,
+               "Should have received the correct amount of type B pings");
+  Assert.equal(countByType.get(TEST_TYPE_A), 10 - TYPE_B_COUNT,
+               "Should have received the correct amount of type A pings");
+
+  Assert.deepEqual(histSuccess.snapshot().counts, [0, 10, 0],
+               "Should have recorded sending success in histograms.");
+  Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 10,
+               "Should have recorded successful send times in histograms.");
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), 0,
+               "Should not have recorded any failed sending in histograms yet.");
+
+  // As we hit the ping send limit and still have pending pings, a send tick should
+  // be scheduled in a minute.
+  Assert.ok(!!pingSendTimerCallback, "Timer callback should be set");
+  Assert.equal(pingSendTimeout, MS_IN_A_MINUTE, "Send tick timeout should be correct");
+
+  // Trigger the next tick - we should receive the next 10 type A pings.
+  PingServer.resetPingHandler();
+  now = fakeNow(futureDate(now, pingSendTimeout));
+  timerPromise = waitForTimer();
+  pingSendTimerCallback();
+  [pingSendTimerCallback, pingSendTimeout] = yield timerPromise;
+
+  pings = yield PingServer.promiseNextPings(10);
+  PingServer.registerPingHandler(() => Assert.ok(false, "Should not have received any pings now"));
+  countByType = countPingTypes(pings);
+
+  Assert.equal(countByType.get(TEST_TYPE_A), 10, "Should have received the correct amount of type A pings");
+
+  // We hit the ping send limit again and still have pending pings, a send tick should
+  // be scheduled in a minute.
+  Assert.equal(pingSendTimeout, MS_IN_A_MINUTE, "Send tick timeout should be correct");
+
+  // Trigger the next tick - we should receive the remaining type A pings.
+  PingServer.resetPingHandler();
+  now = fakeNow(futureDate(now, pingSendTimeout));
+  yield pingSendTimerCallback();
+
+  pings = yield PingServer.promiseNextPings(5);
+  PingServer.registerPingHandler(() => Assert.ok(false, "Should not have received any pings now"));
+  countByType = countPingTypes(pings);
+
+  Assert.equal(countByType.get(TEST_TYPE_A), 5, "Should have received the correct amount of type A pings");
+
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  PingServer.resetPingHandler();
+});
+
+add_task(function* test_sendDateHeader() {
+  fakeNow(new Date(Date.UTC(2011, 1, 1, 11, 0, 0)));
+  yield TelemetrySend.reset();
+
+  let pingId = yield TelemetryController.submitExternalPing("test-send-date-header", {});
+  let req = yield PingServer.promiseNextRequest();
+  let ping = decodeRequestPayload(req);
+  Assert.equal(req.getHeader("Date"), "Tue, 01 Feb 2011 11:00:00 GMT",
+               "Telemetry should send the correct Date header with requests.");
+  Assert.equal(ping.id, pingId, "Should have received the correct ping id.");
+});
+
+// Test the backoff timeout behavior after send failures.
+add_task(function* test_backoffTimeout() {
+  const TYPE_PREFIX = "test-backoffTimeout-";
+  const TEST_TYPE_C = TYPE_PREFIX + "C";
+  const TEST_TYPE_D = TYPE_PREFIX + "D";
+  const TEST_TYPE_E = TYPE_PREFIX + "E";
+
+  let histSuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
+  let histSendTimeSuccess = Telemetry.getHistogramById("TELEMETRY_SEND_SUCCESS");
+  let histSendTimeFail = Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE");
+
+  // Failing a ping send now should trigger backoff behavior.
+  let now = fakeNow(2010, 1, 1, 11, 0, 0);
+  yield TelemetrySend.reset();
+  PingServer.stop();
+
+  histSuccess.clear();
+  histSendTimeSuccess.clear();
+  histSendTimeFail.clear();
+
+  fakePingId("c", 0);
+  now = fakeNow(futureDate(now, MS_IN_A_MINUTE));
+  let sendAttempts = 0;
+  let timerPromise = waitForTimer();
+  yield TelemetryController.submitExternalPing(TEST_TYPE_C, {});
+  let [pingSendTimerCallback, pingSendTimeout] = yield timerPromise;
+  Assert.equal(TelemetrySend.pendingPingCount, 1, "Should have one pending ping.");
+  ++sendAttempts;
+
+  const MAX_BACKOFF_TIMEOUT = 120 * MS_IN_A_MINUTE;
+  for (let timeout = 2 * MS_IN_A_MINUTE; timeout <= MAX_BACKOFF_TIMEOUT; timeout *= 2) {
+    Assert.ok(!!pingSendTimerCallback, "Should have received a timer callback");
+    Assert.equal(pingSendTimeout, timeout, "Send tick timeout should be correct");
+
+    let callback = pingSendTimerCallback;
+    now = fakeNow(futureDate(now, pingSendTimeout));
+    timerPromise = waitForTimer();
+    yield callback();
+    [pingSendTimerCallback, pingSendTimeout] = yield timerPromise;
+    ++sendAttempts;
+  }
+
+  timerPromise = waitForTimer();
+  yield pingSendTimerCallback();
+  [pingSendTimerCallback, pingSendTimeout] = yield timerPromise;
+  Assert.equal(pingSendTimeout, MAX_BACKOFF_TIMEOUT, "Tick timeout should be capped");
+  ++sendAttempts;
+
+  Assert.deepEqual(histSuccess.snapshot().counts, [sendAttempts, 0, 0],
+               "Should have recorded sending failure in histograms.");
+  Assert.equal(histSendTimeSuccess.snapshot().sum, 0,
+               "Should not have recorded any sending success in histograms yet.");
+  Assert.greater(histSendTimeFail.snapshot().sum, 0,
+               "Should have recorded send failure times in histograms.");
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), sendAttempts,
+               "Should have recorded send failure times in histograms.");
+
+  // Submitting a new ping should reset the backoff behavior.
+  fakePingId("d", 0);
+  now = fakeNow(futureDate(now, MS_IN_A_MINUTE));
+  timerPromise = waitForTimer();
+  yield TelemetryController.submitExternalPing(TEST_TYPE_D, {});
+  [pingSendTimerCallback, pingSendTimeout] = yield timerPromise;
+  Assert.equal(pingSendTimeout, 2 * MS_IN_A_MINUTE, "Send tick timeout should be correct");
+  sendAttempts += 2;
+
+  // With the server running again, we should send out the pending pings immediately
+  // when a new ping is submitted.
+  PingServer.start();
+  TelemetrySend.setServer("http://localhost:" + PingServer.port);
+  fakePingId("e", 0);
+  now = fakeNow(futureDate(now, MS_IN_A_MINUTE));
+  timerPromise = waitForTimer();
+  yield TelemetryController.submitExternalPing(TEST_TYPE_E, {});
+
+  let pings = yield PingServer.promiseNextPings(3);
+  let countByType = countPingTypes(pings);
+
+  Assert.equal(countByType.get(TEST_TYPE_C), 1, "Should have received the correct amount of type C pings");
+  Assert.equal(countByType.get(TEST_TYPE_D), 1, "Should have received the correct amount of type D pings");
+  Assert.equal(countByType.get(TEST_TYPE_E), 1, "Should have received the correct amount of type E pings");
+
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  Assert.equal(TelemetrySend.pendingPingCount, 0, "Should have no pending pings left");
+
+  Assert.deepEqual(histSuccess.snapshot().counts, [sendAttempts, 3, 0],
+               "Should have recorded sending failure in histograms.");
+  Assert.greater(histSendTimeSuccess.snapshot().sum, 0,
+               "Should have recorded sending success in histograms.");
+  Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 3,
+               "Should have recorded sending success in histograms.");
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), sendAttempts,
+               "Should have recorded send failure times in histograms.");
+});
+
+add_task(function* test_discardBigPings() {
+  const TEST_PING_TYPE = "test-ping-type";
+
+  let histSizeExceeded = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND");
+  let histDiscardedSize = Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB");
+  let histSuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
+  let histSendTimeSuccess = Telemetry.getHistogramById("TELEMETRY_SEND_SUCCESS");
+  let histSendTimeFail = Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE");
+  for (let h of [histSizeExceeded, histDiscardedSize, histSuccess, histSendTimeSuccess, histSendTimeFail]) {
+    h.clear();
+  }
+
+  // Generate a 2MB string and create an oversized payload.
+  const OVERSIZED_PAYLOAD = {"data": generateRandomString(2 * 1024 * 1024)};
+
+  // Reset the histograms.
+  Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_SEND").clear();
+  Telemetry.getHistogramById("TELEMETRY_DISCARDED_SEND_PINGS_SIZE_MB").clear();
+
+  // Submit a ping of a normal size and check that we don't count it in the histogram.
+  yield TelemetryController.submitExternalPing(TEST_PING_TYPE, { test: "test" });
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  Assert.equal(histSizeExceeded.snapshot().sum, 0, "Telemetry must report no oversized ping submitted.");
+  Assert.equal(histDiscardedSize.snapshot().sum, 0, "Telemetry must report no oversized pings.");
+  Assert.deepEqual(histSuccess.snapshot().counts, [0, 1, 0], "Should have recorded sending success.");
+  Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 1, "Should have recorded send success time.");
+  Assert.greater(histSendTimeSuccess.snapshot().sum, 0, "Should have recorded send success time.");
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), 0, "Should not have recorded send failure time.");
+
+  // Submit an oversized ping and check that it gets discarded.
+  yield TelemetryController.submitExternalPing(TEST_PING_TYPE, OVERSIZED_PAYLOAD);
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  Assert.equal(histSizeExceeded.snapshot().sum, 1, "Telemetry must report 1 oversized ping submitted.");
+  Assert.equal(histDiscardedSize.snapshot().counts[2], 1, "Telemetry must report a 2MB, oversized, ping submitted.");
+  Assert.deepEqual(histSuccess.snapshot().counts, [0, 1, 0], "Should have recorded sending success.");
+  Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 1, "Should have recorded send success time.");
+  Assert.greater(histSendTimeSuccess.snapshot().sum, 0, "Should have recorded send success time.");
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), 0, "Should not have recorded send failure time.");
+});
+
+add_task(function* test_evictedOnServerErrors() {
+  const TEST_TYPE = "test-evicted";
+
+  yield TelemetrySend.reset();
+
+  let histEvicted = Telemetry.getHistogramById("TELEMETRY_PING_EVICTED_FOR_SERVER_ERRORS");
+  let histSuccess = Telemetry.getHistogramById("TELEMETRY_SUCCESS");
+  let histSendTimeSuccess = Telemetry.getHistogramById("TELEMETRY_SEND_SUCCESS");
+  let histSendTimeFail = Telemetry.getHistogramById("TELEMETRY_SEND_FAILURE");
+  for (let h of [histEvicted, histSuccess, histSendTimeSuccess, histSendTimeFail]) {
+    h.clear();
+  }
+
+  // Write a custom ping handler which will return 403. This will trigger ping eviction
+  // on client side.
+  PingServer.registerPingHandler((req, res) => {
+    res.setStatusLine(null, 403, "Forbidden");
+    res.processAsync();
+    res.finish();
+  });
+
+  // Clear the histogram and submit a ping.
+  let pingId = yield TelemetryController.submitExternalPing(TEST_TYPE, {});
+  yield TelemetrySend.testWaitOnOutgoingPings();
+
+  Assert.equal(histEvicted.snapshot().sum, 1,
+               "Telemetry must report a ping evicted due to server errors");
+  Assert.deepEqual(histSuccess.snapshot().counts, [0, 1, 0]);
+  Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 1);
+  Assert.greater(histSendTimeSuccess.snapshot().sum, 0);
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), 0);
+
+  // The ping should not be persisted.
+  yield Assert.rejects(TelemetryStorage.loadPendingPing(pingId), "The ping must not be persisted.");
+
+  // Reset the ping handler and submit a new ping.
+  PingServer.resetPingHandler();
+  pingId = yield TelemetryController.submitExternalPing(TEST_TYPE, {});
+
+  let ping = yield PingServer.promiseNextPings(1);
+  Assert.equal(ping[0].id, pingId, "The correct ping must be received");
+
+  // We should not have updated the error histogram.
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  Assert.equal(histEvicted.snapshot().sum, 1, "Telemetry must report only one ping evicted due to server errors");
+  Assert.deepEqual(histSuccess.snapshot().counts, [0, 2, 0]);
+  Assert.equal(histogramValueCount(histSendTimeSuccess.snapshot()), 2);
+  Assert.equal(histogramValueCount(histSendTimeFail.snapshot()), 0);
+});
+
+// Test that the current, non-persisted pending pings are properly saved on shutdown.
+add_task(function* test_persistCurrentPingsOnShutdown() {
+  const TEST_TYPE = "test-persistCurrentPingsOnShutdown";
+  const PING_COUNT = 5;
+  yield TelemetrySend.reset();
+  PingServer.stop();
+  Assert.equal(TelemetrySend.pendingPingCount, 0, "Should have no pending pings yet");
+
+  // Submit new pings that shouldn't be persisted yet.
+  let ids = [];
+  for (let i=0; i<5; ++i) {
+    ids.push(fakePingId("f", i));
+    TelemetryController.submitExternalPing(TEST_TYPE, {});
+  }
+
+  Assert.equal(TelemetrySend.pendingPingCount, PING_COUNT, "Should have the correct pending ping count");
+
+  // Triggering a shutdown should persist the pings.
+  yield TelemetrySend.shutdown();
+  Assert.ok((yield checkPingsSaved(ids)), "All pending pings should have been persisted");
+
+  // After a restart the pings should have been found when scanning.
+  yield TelemetrySend.reset();
+  Assert.equal(TelemetrySend.pendingPingCount, PING_COUNT, "Should have the correct pending ping count");
+});

toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js

@@ -0,0 +1,547 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/
+
+/**
+ * This test case populates the profile with some fake stored
+ * pings, and checks that pending pings are immediatlely sent
+ * after delayed init.
+ */
+
+"use strict"
+
+Cu.import("resource://gre/modules/osfile.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/Promise.jsm", this);
+Cu.import("resource://gre/modules/TelemetryStorage.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySend.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+var {OS: {File, Path, Constants}} = Cu.import("resource://gre/modules/osfile.jsm", {});
+
+// We increment TelemetryStorage's MAX_PING_FILE_AGE and
+// OVERDUE_PING_FILE_AGE by 1 minute so that our test pings exceed
+// those points in time, even taking into account file system imprecision.
+const ONE_MINUTE_MS = 60 * 1000;
+const OVERDUE_PING_FILE_AGE = TelemetrySend.OVERDUE_PING_FILE_AGE + ONE_MINUTE_MS;
+
+const PING_SAVE_FOLDER = "saved-telemetry-pings";
+const PING_TIMEOUT_LENGTH = 5000;
+const OVERDUE_PINGS = 6;
+const OLD_FORMAT_PINGS = 4;
+const RECENT_PINGS = 4;
+
+const TOTAL_EXPECTED_PINGS = OVERDUE_PINGS + RECENT_PINGS + OLD_FORMAT_PINGS;
+
+const PREF_FHR_UPLOAD = "datareporting.healthreport.uploadEnabled";
+
+var gCreatedPings = 0;
+var gSeenPings = 0;
+
+/**
+ * Creates some Telemetry pings for the and saves them to disk. Each ping gets a
+ * unique ID based on an incrementor.
+ *
+ * @param {Array} aPingInfos An array of ping type objects. Each entry must be an
+ *                object containing a "num" field for the number of pings to create and
+ *                an "age" field. The latter representing the age in milliseconds to offset
+ *                from now. A value of 10 would make the ping 10ms older than now, for
+ *                example.
+ * @returns Promise
+ * @resolve an Array with the created pings ids.
+ */
+var createSavedPings = Task.async(function* (aPingInfos) {
+  let pingIds = [];
+  let now = Date.now();
+
+  for (let type in aPingInfos) {
+    let num = aPingInfos[type].num;
+    let age = now - (aPingInfos[type].age || 0);
+    for (let i = 0; i < num; ++i) {
+      let pingId = yield TelemetryController.addPendingPing("test-ping", {}, { overwrite: true });
+      if (aPingInfos[type].age) {
+        // savePing writes to the file synchronously, so we're good to
+        // modify the lastModifedTime now.
+        let filePath = getSavePathForPingId(pingId);
+        yield File.setDates(filePath, null, age);
+      }
+      gCreatedPings++;
+      pingIds.push(pingId);
+    }
+  }
+
+  return pingIds;
+});
+
+/**
+ * Deletes locally saved pings if they exist.
+ *
+ * @param aPingIds an Array of ping ids to delete.
+ * @returns Promise
+ */
+var clearPings = Task.async(function* (aPingIds) {
+  for (let pingId of aPingIds) {
+    yield TelemetryStorage.removePendingPing(pingId);
+  }
+});
+
+/**
+ * Fakes the pending pings storage quota.
+ * @param {Integer} aPendingQuota The new quota, in bytes.
+ */
+function fakePendingPingsQuota(aPendingQuota) {
+  let storage = Cu.import("resource://gre/modules/TelemetryStorage.jsm");
+  storage.Policy.getPendingPingsQuota = () => aPendingQuota;
+}
+
+/**
+ * Returns a handle for the file that a ping should be
+ * stored in locally.
+ *
+ * @returns path
+ */
+function getSavePathForPingId(aPingId) {
+  return Path.join(Constants.Path.profileDir, PING_SAVE_FOLDER, aPingId);
+}
+
+/**
+ * Check if the number of Telemetry pings received by the HttpServer is not equal
+ * to aExpectedNum.
+ *
+ * @param aExpectedNum the number of pings we expect to receive.
+ */
+function assertReceivedPings(aExpectedNum) {
+  do_check_eq(gSeenPings, aExpectedNum);
+}
+
+/**
+ * Throws if any pings with the id in aPingIds is saved locally.
+ *
+ * @param aPingIds an Array of pings ids to check.
+ * @returns Promise
+ */
+var assertNotSaved = Task.async(function* (aPingIds) {
+  let saved = 0;
+  for (let id of aPingIds) {
+    let filePath = getSavePathForPingId(id);
+    if (yield File.exists(filePath)) {
+      saved++;
+    }
+  }
+  if (saved > 0) {
+    do_throw("Found " + saved + " unexpected saved pings.");
+  }
+});
+
+/**
+ * Our handler function for the HttpServer that simply
+ * increments the gSeenPings global when it successfully
+ * receives and decodes a Telemetry payload.
+ *
+ * @param aRequest the HTTP request sent from HttpServer.
+ */
+function pingHandler(aRequest) {
+  gSeenPings++;
+}
+
+add_task(function* test_setup() {
+  PingServer.start();
+  PingServer.registerPingHandler(pingHandler);
+  do_get_profile();
+  loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true);
+  Services.prefs.setCharPref(TelemetryController.Constants.PREF_SERVER,
+                             "http://localhost:" + PingServer.port);
+});
+
+/**
+ * Setup the tests by making sure the ping storage directory is available, otherwise
+ * |TelemetryController.testSaveDirectoryToFile| could fail.
+ */
+add_task(function* setupEnvironment() {
+  // The following tests assume this pref to be true by default.
+  Services.prefs.setBoolPref(PREF_FHR_UPLOAD, true);
+
+  yield TelemetryController.testSetup();
+
+  let directory = TelemetryStorage.pingDirectoryPath;
+  yield File.makeDir(directory, { ignoreExisting: true, unixMode: OS.Constants.S_IRWXU });
+
+  yield TelemetryStorage.testClearPendingPings();
+});
+
+/**
+ * Test that really recent pings are sent on Telemetry initialization.
+ */
+add_task(function* test_recent_pings_sent() {
+  let pingTypes = [{ num: RECENT_PINGS }];
+  yield createSavedPings(pingTypes);
+
+  yield TelemetryController.testReset();
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  assertReceivedPings(RECENT_PINGS);
+
+  yield TelemetryStorage.testClearPendingPings();
+});
+
+/**
+ * Create an overdue ping in the old format and try to send it.
+ */
+add_task(function* test_overdue_old_format() {
+  // A test ping in the old, standard format.
+  const PING_OLD_FORMAT = {
+    slug: "1234567abcd",
+    reason: "test-ping",
+    payload: {
+      info: {
+        reason: "test-ping",
+        OS: "XPCShell",
+        appID: "SomeId",
+        appVersion: "1.0",
+        appName: "XPCShell",
+        appBuildID: "123456789",
+        appUpdateChannel: "Test",
+        platformBuildID: "987654321",
+      },
+    },
+  };
+
+  // A ping with no info section, but with a slug.
+  const PING_NO_INFO = {
+    slug: "1234-no-info-ping",
+    reason: "test-ping",
+    payload: {}
+  };
+
+  // A ping with no payload.
+  const PING_NO_PAYLOAD = {
+    slug: "5678-no-payload",
+    reason: "test-ping",
+  };
+
+  // A ping with no info and no slug.
+  const PING_NO_SLUG = {
+    reason: "test-ping",
+    payload: {}
+  };
+
+  const PING_FILES_PATHS = [
+    getSavePathForPingId(PING_OLD_FORMAT.slug),
+    getSavePathForPingId(PING_NO_INFO.slug),
+    getSavePathForPingId(PING_NO_PAYLOAD.slug),
+    getSavePathForPingId("no-slug-file"),
+  ];
+
+  // Write the ping to file and make it overdue.
+  yield TelemetryStorage.savePing(PING_OLD_FORMAT, true);
+  yield TelemetryStorage.savePing(PING_NO_INFO, true);
+  yield TelemetryStorage.savePing(PING_NO_PAYLOAD, true);
+  yield TelemetryStorage.savePingToFile(PING_NO_SLUG, PING_FILES_PATHS[3], true);
+
+  for (let f in PING_FILES_PATHS) {
+    yield File.setDates(PING_FILES_PATHS[f], null, Date.now() - OVERDUE_PING_FILE_AGE);
+  }
+
+  gSeenPings = 0;
+  yield TelemetryController.testReset();
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  assertReceivedPings(OLD_FORMAT_PINGS);
+
+  // |TelemetryStorage.cleanup| doesn't know how to remove a ping with no slug or id,
+  // so remove it manually so that the next test doesn't fail.
+  yield OS.File.remove(PING_FILES_PATHS[3]);
+
+  yield TelemetryStorage.testClearPendingPings();
+});
+
+add_task(function* test_corrupted_pending_pings() {
+  const TEST_TYPE = "test_corrupted";
+
+  Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").clear();
+  Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").clear();
+
+  // Save a pending ping and get its id.
+  let pendingPingId = yield TelemetryController.addPendingPing(TEST_TYPE, {}, {});
+
+  // Try to load it: there should be no error.
+  yield TelemetryStorage.loadPendingPing(pendingPingId);
+
+  let h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must not report a pending ping load failure");
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must not report a pending ping parse failure");
+
+  // Delete it from the disk, so that its id will be kept in the cache but it will
+  // fail loading the file.
+  yield OS.File.remove(getSavePathForPingId(pendingPingId));
+
+  // Try to load a pending ping which isn't there anymore.
+  yield Assert.rejects(TelemetryStorage.loadPendingPing(pendingPingId),
+                       "Telemetry must fail loading a ping which isn't there");
+
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();
+  Assert.equal(h.sum, 1, "Telemetry must report a pending ping load failure");
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must not report a pending ping parse failure");
+
+  // Save a new ping, so that it gets in the pending pings cache.
+  pendingPingId = yield TelemetryController.addPendingPing(TEST_TYPE, {}, {});
+  // Overwrite it with a corrupted JSON file and then try to load it.
+  const INVALID_JSON = "{ invalid,JSON { {1}";
+  yield OS.File.writeAtomic(getSavePathForPingId(pendingPingId), INVALID_JSON, { encoding: "utf-8" });
+
+  // Try to load the ping with the corrupted JSON content.
+  yield Assert.rejects(TelemetryStorage.loadPendingPing(pendingPingId),
+                       "Telemetry must fail loading a corrupted ping");
+
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").snapshot();
+  Assert.equal(h.sum, 1, "Telemetry must report a pending ping load failure");
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").snapshot();
+  Assert.equal(h.sum, 1, "Telemetry must report a pending ping parse failure");
+
+  let exists = yield OS.File.exists(getSavePathForPingId(pendingPingId));
+  Assert.ok(!exists, "The unparseable ping should have been removed");
+
+  yield TelemetryStorage.testClearPendingPings();
+});
+
+/**
+ * Create some recent and overdue pings and verify that they get sent.
+ */
+add_task(function* test_overdue_pings_trigger_send() {
+  let pingTypes = [
+    { num: RECENT_PINGS },
+    { num: OVERDUE_PINGS, age: OVERDUE_PING_FILE_AGE },
+  ];
+  let pings = yield createSavedPings(pingTypes);
+  let recentPings = pings.slice(0, RECENT_PINGS);
+  let overduePings = pings.slice(-OVERDUE_PINGS);
+
+  yield TelemetryController.testReset();
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  assertReceivedPings(TOTAL_EXPECTED_PINGS);
+
+  yield assertNotSaved(recentPings);
+  yield assertNotSaved(overduePings);
+
+  Assert.equal(TelemetrySend.overduePingsCount, overduePings.length,
+               "Should have tracked the correct amount of overdue pings");
+
+  yield TelemetryStorage.testClearPendingPings();
+});
+
+/**
+ * Create a ping in the old format, send it, and make sure the request URL contains
+ * the correct version query parameter.
+ */
+add_task(function* test_overdue_old_format() {
+  // A test ping in the old, standard format.
+  const PING_OLD_FORMAT = {
+    slug: "1234567abcd",
+    reason: "test-ping",
+    payload: {
+      info: {
+        reason: "test-ping",
+        OS: "XPCShell",
+        appID: "SomeId",
+        appVersion: "1.0",
+        appName: "XPCShell",
+        appBuildID: "123456789",
+        appUpdateChannel: "Test",
+        platformBuildID: "987654321",
+      },
+    },
+  };
+
+  const filePath =
+    Path.join(Constants.Path.profileDir, PING_SAVE_FOLDER, PING_OLD_FORMAT.slug);
+
+  // Write the ping to file and make it overdue.
+  yield TelemetryStorage.savePing(PING_OLD_FORMAT, true);
+  yield File.setDates(filePath, null, Date.now() - OVERDUE_PING_FILE_AGE);
+
+  let receivedPings = 0;
+  // Register a new prefix handler to validate the URL.
+  PingServer.registerPingHandler(request => {
+    // Check that we have a version query parameter in the URL.
+    Assert.notEqual(request.queryString, "");
+
+    // Make sure the version in the query string matches the old ping format version.
+    let params = request.queryString.split("&");
+    Assert.ok(params.find(p => p == "v=1"));
+
+    receivedPings++;
+  });
+
+  yield TelemetryController.testReset();
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  Assert.equal(receivedPings, 1, "We must receive a ping in the old format.");
+
+  yield TelemetryStorage.testClearPendingPings();
+  PingServer.resetPingHandler();
+});
+
+add_task(function* test_pendingPingsQuota() {
+  const PING_TYPE = "foo";
+
+  // Disable upload so pings don't get sent and removed from the pending pings directory.
+  Services.prefs.setBoolPref(PREF_FHR_UPLOAD, false);
+
+  // Remove all the pending pings then startup and wait for the cleanup task to complete.
+  // There should be nothing to remove.
+  yield TelemetryStorage.testClearPendingPings();
+  yield TelemetryController.testReset();
+  yield TelemetrySend.testWaitOnOutgoingPings();
+  yield TelemetryStorage.testPendingQuotaTaskPromise();
+
+  // Remove the pending deletion ping generated when flipping FHR upload off.
+  yield TelemetryStorage.testClearPendingPings();
+
+  let expectedPrunedPings = [];
+  let expectedNotPrunedPings = [];
+
+  let checkPendingPings = Task.async(function*() {
+    // Check that the pruned pings are not on disk anymore.
+    for (let prunedPingId of expectedPrunedPings) {
+      yield Assert.rejects(TelemetryStorage.loadPendingPing(prunedPingId),
+                           "Ping " + prunedPingId + " should have been pruned.");
+      const pingPath = getSavePathForPingId(prunedPingId);
+      Assert.ok(!(yield OS.File.exists(pingPath)), "The ping should not be on the disk anymore.");
+    }
+
+    // Check that the expected pings are there.
+    for (let expectedPingId of expectedNotPrunedPings) {
+      Assert.ok((yield TelemetryStorage.loadPendingPing(expectedPingId)),
+                "Ping" + expectedPingId + " should be among the pending pings.");
+    }
+  });
+
+  let pendingPingsInfo = [];
+  let pingsSizeInBytes = 0;
+
+  // Create 10 pings to test the pending pings quota.
+  for (let days = 1; days < 11; days++) {
+    const date = fakeNow(2010, 1, days, 1, 1, 0);
+    const pingId = yield TelemetryController.addPendingPing(PING_TYPE, {}, {});
+
+    // Find the size of the ping.
+    const pingFilePath = getSavePathForPingId(pingId);
+    const pingSize = (yield OS.File.stat(pingFilePath)).size;
+    // Add the info at the beginning of the array, so that most recent pings come first.
+    pendingPingsInfo.unshift({id: pingId, size: pingSize, timestamp: date.getTime() });
+
+    // Set the last modification date.
+    yield OS.File.setDates(pingFilePath, null, date.getTime());
+
+    // Add it to the pending ping directory size.
+    pingsSizeInBytes += pingSize;
+  }
+
+  // We need to test the pending pings size before we hit the quota, otherwise a special
+  // value is recorded.
+  Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").clear();
+  Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").clear();
+  Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").clear();
+
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testPendingQuotaTaskPromise();
+
+  // Check that the correct values for quota probes are reported when no quota is hit.
+  let h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").snapshot();
+  Assert.equal(h.sum, Math.round(pingsSizeInBytes / 1024 / 1024),
+               "Telemetry must report the correct pending pings directory size.");
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must report 0 evictions if quota is not hit.");
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").snapshot();
+  Assert.equal(h.sum, 0, "Telemetry must report a null elapsed time if quota is not hit.");
+
+  // Set the quota to 80% of the space.
+  const testQuotaInBytes = pingsSizeInBytes * 0.8;
+  fakePendingPingsQuota(testQuotaInBytes);
+
+  // The storage prunes pending pings until we reach 90% of the requested storage quota.
+  // Based on that, find how many pings should be kept.
+  const safeQuotaSize = Math.round(testQuotaInBytes * 0.9);
+  let sizeInBytes = 0;
+  let pingsWithinQuota = [];
+  let pingsOutsideQuota = [];
+
+  for (let pingInfo of pendingPingsInfo) {
+    sizeInBytes += pingInfo.size;
+    if (sizeInBytes >= safeQuotaSize) {
+      pingsOutsideQuota.push(pingInfo.id);
+      continue;
+    }
+    pingsWithinQuota.push(pingInfo.id);
+  }
+
+  expectedNotPrunedPings = pingsWithinQuota;
+  expectedPrunedPings = pingsOutsideQuota;
+
+  // Reset TelemetryController to start the pending pings cleanup.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testPendingQuotaTaskPromise();
+  yield checkPendingPings();
+
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").snapshot();
+  Assert.equal(h.sum, pingsOutsideQuota.length,
+               "Telemetry must correctly report the over quota pings evicted from the pending pings directory.");
+  h = Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").snapshot();
+  Assert.equal(h.sum, 17, "Pending pings quota was hit, a special size must be reported.");
+
+  // Trigger a cleanup again and make sure we're not removing anything.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testPendingQuotaTaskPromise();
+  yield checkPendingPings();
+
+  const OVERSIZED_PING_ID = "9b21ec8f-f762-4d28-a2c1-44e1c4694f24";
+  // Create a pending oversized ping.
+  const OVERSIZED_PING = {
+    id: OVERSIZED_PING_ID,
+    type: PING_TYPE,
+    creationDate: (new Date()).toISOString(),
+    // Generate a 2MB string to use as the ping payload.
+    payload: generateRandomString(2 * 1024 * 1024),
+  };
+  yield TelemetryStorage.savePendingPing(OVERSIZED_PING);
+
+  // Reset the histograms.
+  Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").clear();
+  Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").clear();
+
+  // Try to manually load the oversized ping.
+  yield Assert.rejects(TelemetryStorage.loadPendingPing(OVERSIZED_PING_ID),
+                       "The oversized ping should have been pruned.");
+  Assert.ok(!(yield OS.File.exists(getSavePathForPingId(OVERSIZED_PING_ID))),
+            "The ping should not be on the disk anymore.");
+
+  // Make sure we're correctly updating the related histograms.
+  h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").snapshot();
+  Assert.equal(h.sum, 1, "Telemetry must report 1 oversized ping in the pending pings directory.");
+  h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").snapshot();
+  Assert.equal(h.counts[2], 1, "Telemetry must report a 2MB, oversized, ping.");
+
+  // Save the ping again to check if it gets pruned when scanning the pings directory.
+  yield TelemetryStorage.savePendingPing(OVERSIZED_PING);
+  expectedPrunedPings.push(OVERSIZED_PING_ID);
+
+  // Scan the pending pings directory.
+  yield TelemetryController.testReset();
+  yield TelemetryStorage.testPendingQuotaTaskPromise();
+  yield checkPendingPings();
+
+  // Make sure we're correctly updating the related histograms.
+  h = Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").snapshot();
+  Assert.equal(h.sum, 2, "Telemetry must report 1 oversized ping in the pending pings directory.");
+  h = Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB").snapshot();
+  Assert.equal(h.counts[2], 2, "Telemetry must report two 2MB, oversized, pings.");
+
+  Services.prefs.setBoolPref(PREF_FHR_UPLOAD, true);
+});
+
+add_task(function* teardown() {
+  yield PingServer.stop();
+});

toolkit/components/telemetry/tests/unit/test_TelemetryStopwatch.js

@@ -0,0 +1,156 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var tmpScope = {};
+Cu.import("resource://gre/modules/TelemetryStopwatch.jsm", tmpScope);
+var TelemetryStopwatch = tmpScope.TelemetryStopwatch;
+
+const HIST_NAME = "TELEMETRY_SEND_SUCCESS";
+const HIST_NAME2 = "RANGE_CHECKSUM_ERRORS";
+const KEYED_HIST = { id: "TELEMETRY_INVALID_PING_TYPE_SUBMITTED", key: "TEST" };
+
+var refObj = {}, refObj2 = {};
+
+var originalCount1, originalCount2;
+
+function run_test() {
+  let histogram = Telemetry.getHistogramById(HIST_NAME);
+  let snapshot = histogram.snapshot();
+  originalCount1 = snapshot.counts.reduce((a, b) => a += b);
+
+  histogram = Telemetry.getHistogramById(HIST_NAME2);
+  snapshot = histogram.snapshot();
+  originalCount2 = snapshot.counts.reduce((a, b) => a += b);
+
+  histogram = Telemetry.getKeyedHistogramById(KEYED_HIST.id);
+  snapshot = histogram.snapshot(KEYED_HIST.key);
+  originalCount3 = snapshot.counts.reduce((a, b) => a += b);
+
+  do_check_false(TelemetryStopwatch.start(3));
+  do_check_false(TelemetryStopwatch.start({}));
+  do_check_false(TelemetryStopwatch.start("", 3));
+  do_check_false(TelemetryStopwatch.start("", ""));
+  do_check_false(TelemetryStopwatch.start({}, {}));
+
+  do_check_true(TelemetryStopwatch.start("mark1"));
+  do_check_true(TelemetryStopwatch.start("mark2"));
+
+  do_check_true(TelemetryStopwatch.start("mark1", refObj));
+  do_check_true(TelemetryStopwatch.start("mark2", refObj));
+
+  // Same timer can't be re-started before being stopped
+  do_check_false(TelemetryStopwatch.start("mark1"));
+  do_check_false(TelemetryStopwatch.start("mark1", refObj));
+
+  // Can't stop a timer that was accidentaly started twice
+  do_check_false(TelemetryStopwatch.finish("mark1"));
+  do_check_false(TelemetryStopwatch.finish("mark1", refObj));
+
+  do_check_true(TelemetryStopwatch.start("NON-EXISTENT_HISTOGRAM"));
+  do_check_false(TelemetryStopwatch.finish("NON-EXISTENT_HISTOGRAM"));
+
+  do_check_true(TelemetryStopwatch.start("NON-EXISTENT_HISTOGRAM", refObj));
+  do_check_false(TelemetryStopwatch.finish("NON-EXISTENT_HISTOGRAM", refObj));
+
+  do_check_true(TelemetryStopwatch.start(HIST_NAME));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME2));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME, refObj));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME2, refObj));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME, refObj2));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME2, refObj2));
+
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME2));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME, refObj));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME2, refObj));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME, refObj2));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME2, refObj2));
+
+  // Verify that TS.finish deleted the timers
+  do_check_false(TelemetryStopwatch.finish(HIST_NAME));
+  do_check_false(TelemetryStopwatch.finish(HIST_NAME, refObj));
+
+  // Verify that they can be used again
+  do_check_true(TelemetryStopwatch.start(HIST_NAME));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME, refObj));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME));
+  do_check_true(TelemetryStopwatch.finish(HIST_NAME, refObj));
+
+  do_check_false(TelemetryStopwatch.finish("unknown-mark")); // Unknown marker
+  do_check_false(TelemetryStopwatch.finish("unknown-mark", {})); // Unknown object
+  do_check_false(TelemetryStopwatch.finish(HIST_NAME, {})); // Known mark on unknown object
+
+  // Test cancel
+  do_check_true(TelemetryStopwatch.start(HIST_NAME));
+  do_check_true(TelemetryStopwatch.start(HIST_NAME, refObj));
+  do_check_true(TelemetryStopwatch.cancel(HIST_NAME));
+  do_check_true(TelemetryStopwatch.cancel(HIST_NAME, refObj));
+
+  // Verify that can not cancel twice
+  do_check_false(TelemetryStopwatch.cancel(HIST_NAME));
+  do_check_false(TelemetryStopwatch.cancel(HIST_NAME, refObj));
+
+  // Verify that cancel removes the timers
+  do_check_false(TelemetryStopwatch.finish(HIST_NAME));
+  do_check_false(TelemetryStopwatch.finish(HIST_NAME, refObj));
+
+  // Verify that keyed stopwatch reject invalid keys.
+  for (let key of [3, {}, ""]) {
+    do_check_false(TelemetryStopwatch.startKeyed(KEYED_HIST.id, key));
+  }
+
+  // Verify that keyed histograms can be started.
+  do_check_true(TelemetryStopwatch.startKeyed("HISTOGRAM", "KEY1"));
+  do_check_true(TelemetryStopwatch.startKeyed("HISTOGRAM", "KEY2"));
+  do_check_true(TelemetryStopwatch.startKeyed("HISTOGRAM", "KEY1", refObj));
+  do_check_true(TelemetryStopwatch.startKeyed("HISTOGRAM", "KEY2", refObj));
+
+  // Restarting keyed histograms should fail.
+  do_check_false(TelemetryStopwatch.startKeyed("HISTOGRAM", "KEY1"));
+  do_check_false(TelemetryStopwatch.startKeyed("HISTOGRAM", "KEY1", refObj));
+
+  // Finishing a stopwatch of a non existing histogram should return false.
+  do_check_false(TelemetryStopwatch.finishKeyed("HISTOGRAM", "KEY2"));
+  do_check_false(TelemetryStopwatch.finishKeyed("HISTOGRAM", "KEY2", refObj));
+
+  // Starting & finishing a keyed stopwatch for an existing histogram should work.
+  do_check_true(TelemetryStopwatch.startKeyed(KEYED_HIST.id, KEYED_HIST.key));
+  do_check_true(TelemetryStopwatch.finishKeyed(KEYED_HIST.id, KEYED_HIST.key));
+  // Verify that TS.finish deleted the timers
+  do_check_false(TelemetryStopwatch.finishKeyed(KEYED_HIST.id, KEYED_HIST.key));
+
+  // Verify that they can be used again
+  do_check_true(TelemetryStopwatch.startKeyed(KEYED_HIST.id, KEYED_HIST.key));
+  do_check_true(TelemetryStopwatch.finishKeyed(KEYED_HIST.id, KEYED_HIST.key));
+
+  do_check_false(TelemetryStopwatch.finishKeyed("unknown-mark", "unknown-key"));
+  do_check_false(TelemetryStopwatch.finishKeyed(KEYED_HIST.id, "unknown-key"));
+
+  // Verify that keyed histograms can only be canceled through "keyed" API.
+  do_check_true(TelemetryStopwatch.startKeyed(KEYED_HIST.id, KEYED_HIST.key));
+  do_check_false(TelemetryStopwatch.cancel(KEYED_HIST.id, KEYED_HIST.key));
+  do_check_true(TelemetryStopwatch.cancelKeyed(KEYED_HIST.id, KEYED_HIST.key));
+  do_check_false(TelemetryStopwatch.cancelKeyed(KEYED_HIST.id, KEYED_HIST.key));
+
+  finishTest();
+}
+
+function finishTest() {
+  let histogram = Telemetry.getHistogramById(HIST_NAME);
+  let snapshot = histogram.snapshot();
+  let newCount = snapshot.counts.reduce((a, b) => a += b);
+
+  do_check_eq(newCount - originalCount1, 5, "The correct number of histograms were added for histogram 1.");
+
+  histogram = Telemetry.getHistogramById(HIST_NAME2);
+  snapshot = histogram.snapshot();
+  newCount = snapshot.counts.reduce((a, b) => a += b);
+
+  do_check_eq(newCount - originalCount2, 3, "The correct number of histograms were added for histogram 2.");
+
+  histogram = Telemetry.getKeyedHistogramById(KEYED_HIST.id);
+  snapshot = histogram.snapshot(KEYED_HIST.key);
+  newCount = snapshot.counts.reduce((a, b) => a += b);
+
+  do_check_eq(newCount - originalCount3, 2, "The correct number of histograms were added for histogram 3.");
+}

toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js

@@ -0,0 +1,77 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+var Cu = Components.utils;
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/TelemetryController.jsm", this);
+Cu.import("resource://gre/modules/TelemetrySession.jsm", this);
+Cu.import('resource://gre/modules/XPCOMUtils.jsm');
+
+// The @mozilla/xre/app-info;1 XPCOM object provided by the xpcshell test harness doesn't
+// implement the nsIXULAppInfo interface, which is needed by Services.jsm and
+// TelemetrySession.jsm. updateAppInfo() creates and registers a minimal mock app-info.
+Cu.import("resource://testing-common/AppInfo.jsm");
+updateAppInfo();
+
+var gGlobalScope = this;
+
+function getSimpleMeasurementsFromTelemetryController() {
+  return TelemetrySession.getPayload().simpleMeasurements;
+}
+
+add_task(function* test_setup() {
+  // Telemetry needs the AddonManager.
+  loadAddonManager();
+  // Make profile available for |TelemetryController.testShutdown()|.
+  do_get_profile();
+
+  // Make sure we don't generate unexpected pings due to pref changes.
+  yield setEmptyPrefWatchlist();
+
+  yield new Promise(resolve =>
+    Services.telemetry.asyncFetchTelemetryData(resolve));
+});
+
+add_task(function* actualTest() {
+  yield TelemetryController.testSetup();
+
+  // Test the module logic
+  let tmp = {};
+  Cu.import("resource://gre/modules/TelemetryTimestamps.jsm", tmp);
+  let TelemetryTimestamps = tmp.TelemetryTimestamps;
+  let now = Date.now();
+  TelemetryTimestamps.add("foo");
+  do_check_true(TelemetryTimestamps.get().foo != null); // foo was added
+  do_check_true(TelemetryTimestamps.get().foo >= now); // foo has a reasonable value
+
+  // Add timestamp with value
+  // Use a value far in the future since TelemetryController substracts the time of
+  // process initialization.
+  const YEAR_4000_IN_MS = 64060588800000;
+  TelemetryTimestamps.add("bar", YEAR_4000_IN_MS);
+  do_check_eq(TelemetryTimestamps.get().bar, YEAR_4000_IN_MS); // bar has the right value
+
+  // Can't add the same timestamp twice
+  TelemetryTimestamps.add("bar", 2);
+  do_check_eq(TelemetryTimestamps.get().bar, YEAR_4000_IN_MS); // bar wasn't overwritten
+
+  let threw = false;
+  try {
+    TelemetryTimestamps.add("baz", "this isn't a number");
+  } catch (ex) {
+    threw = true;
+  }
+  do_check_true(threw); // adding non-number threw
+  do_check_null(TelemetryTimestamps.get().baz); // no baz was added
+
+  // Test that the data gets added to the telemetry ping properly
+  let simpleMeasurements = getSimpleMeasurementsFromTelemetryController();
+  do_check_true(simpleMeasurements != null); // got simple measurements from ping data
+  do_check_true(simpleMeasurements.foo > 1); // foo was included
+  do_check_true(simpleMeasurements.bar > 1); // bar was included
+  do_check_eq(undefined, simpleMeasurements.baz); // baz wasn't included since it wasn't added
+
+  yield TelemetryController.testShutdown();
+});

toolkit/components/telemetry/tests/unit/test_ThreadHangStats.js

@@ -0,0 +1,102 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+Cu.import("resource://gre/modules/Services.jsm");
+
+function getMainThreadHangStats() {
+  let threads = Services.telemetry.threadHangStats;
+  return threads.find((thread) => (thread.name === "Gecko"));
+}
+
+function run_test() {
+  let startHangs = getMainThreadHangStats();
+
+  // We disable hang reporting in several situations (e.g. debug builds,
+  // official releases). In those cases, we don't have hang stats available
+  // and should exit the test early.
+  if (!startHangs) {
+    ok("Hang reporting not enabled.");
+    return;
+  }
+
+  if (Services.appinfo.OS === 'Linux' || Services.appinfo.OS === 'Android') {
+    // We use the rt_tgsigqueueinfo syscall on Linux which requires a
+    // certain kernel version. It's not an error if the system running
+    // the test is older than that.
+    let kernel = Services.sysinfo.get('kernel_version') ||
+                 Services.sysinfo.get('version');
+    if (Services.vc.compare(kernel, '2.6.31') < 0) {
+      ok("Hang reporting not supported for old kernel.");
+      return;
+    }
+  }
+
+  // Run three events in the event loop:
+  // the first event causes a transient hang;
+  // the second event causes a permanent hang;
+  // the third event checks results from previous events.
+
+  do_execute_soon(() => {
+    // Cause a hang lasting 1 second (transient hang).
+    let startTime = Date.now();
+    while ((Date.now() - startTime) < 1000);
+  });
+
+  do_execute_soon(() => {
+    // Cause a hang lasting 10 seconds (permanent hang).
+    let startTime = Date.now();
+    while ((Date.now() - startTime) < 10000);
+  });
+
+  do_execute_soon(() => {
+    do_test_pending();
+
+    let check_results = () => {
+      let endHangs = getMainThreadHangStats();
+
+      // Because hangs are recorded asynchronously, if we don't see new hangs,
+      // we should wait for pending hangs to be recorded. On the other hand,
+      // if hang monitoring is broken, this test will time out.
+      if (endHangs.hangs.length === startHangs.hangs.length) {
+        do_timeout(100, check_results);
+        return;
+      }
+
+      let check_histogram = (histogram) => {
+        equal(typeof histogram, "object");
+        equal(histogram.histogram_type, 0);
+        equal(typeof histogram.min, "number");
+        equal(typeof histogram.max, "number");
+        equal(typeof histogram.sum, "number");
+        ok(Array.isArray(histogram.ranges));
+        ok(Array.isArray(histogram.counts));
+        equal(histogram.counts.length, histogram.ranges.length);
+      };
+
+      // Make sure the hang stats structure is what we expect.
+      equal(typeof endHangs, "object");
+      check_histogram(endHangs.activity);
+
+      ok(Array.isArray(endHangs.hangs));
+      notEqual(endHangs.hangs.length, 0);
+
+      ok(Array.isArray(endHangs.hangs[0].stack));
+      notEqual(endHangs.hangs[0].stack.length, 0);
+      equal(typeof endHangs.hangs[0].stack[0], "string");
+
+      // Make sure one of the hangs is a permanent
+      // hang containing a native stack.
+      ok(endHangs.hangs.some((hang) => (
+        Array.isArray(hang.nativeStack) &&
+        hang.nativeStack.length !== 0 &&
+        typeof hang.nativeStack[0] === "string"
+      )));
+
+      check_histogram(endHangs.hangs[0].histogram);
+
+      do_test_finished();
+    };
+
+    check_results();
+  });
+}

toolkit/components/telemetry/tests/unit/test_nsITelemetry.js

@@ -0,0 +1,883 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const INT_MAX = 0x7FFFFFFF;
+
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);
+
+// Return an array of numbers from lower up to, excluding, upper
+function numberRange(lower, upper)
+{
+  let a = [];
+  for (let i=lower; i<upper; ++i) {
+    a.push(i);
+  }
+  return a;
+}
+
+function expect_fail(f) {
+  let failed = false;
+  try {
+    f();
+    failed = false;
+  } catch (e) {
+    failed = true;
+  }
+  do_check_true(failed);
+}
+
+function expect_success(f) {
+  let succeeded = false;
+  try {
+    f();
+    succeeded = true;
+  } catch (e) {
+    succeeded = false;
+  }
+  do_check_true(succeeded);
+}
+
+function compareHistograms(h1, h2) {
+  let s1 = h1.snapshot();
+  let s2 = h2.snapshot();
+
+  do_check_eq(s1.histogram_type, s2.histogram_type);
+  do_check_eq(s1.min, s2.min);
+  do_check_eq(s1.max, s2.max);
+  do_check_eq(s1.sum, s2.sum);
+
+  do_check_eq(s1.counts.length, s2.counts.length);
+  for (let i = 0; i < s1.counts.length; i++)
+    do_check_eq(s1.counts[i], s2.counts[i]);
+
+  do_check_eq(s1.ranges.length, s2.ranges.length);
+  for (let i = 0; i < s1.ranges.length; i++)
+    do_check_eq(s1.ranges[i], s2.ranges[i]);
+}
+
+function check_histogram(histogram_type, name, min, max, bucket_count) {
+  var h = Telemetry.getHistogramById(name);
+  var r = h.snapshot().ranges;
+  var sum = 0;
+  for (let i=0;i<r.length;i++) {
+    var v = r[i];
+    sum += v;
+    h.add(v);
+  }
+  var s = h.snapshot();
+  // verify properties
+  do_check_eq(sum, s.sum);
+
+  // there should be exactly one element per bucket
+  for (let i of s.counts) {
+    do_check_eq(i, 1);
+  }
+  var hgrams = Telemetry.histogramSnapshots
+  let gh = hgrams[name]
+  do_check_eq(gh.histogram_type, histogram_type);
+
+  do_check_eq(gh.min, min)
+  do_check_eq(gh.max, max)
+
+  // Check that booleans work with nonboolean histograms
+  h.add(false);
+  h.add(true);
+  s = h.snapshot().counts;
+  do_check_eq(s[0], 2)
+  do_check_eq(s[1], 2)
+
+  // Check that clearing works.
+  h.clear();
+  s = h.snapshot();
+  for (var i of s.counts) {
+    do_check_eq(i, 0);
+  }
+  do_check_eq(s.sum, 0);
+
+  h.add(0);
+  h.add(1);
+  var c = h.snapshot().counts;
+  do_check_eq(c[0], 1);
+  do_check_eq(c[1], 1);
+}
+
+// This MUST be the very first test of this file.
+add_task({
+  skip_if: () => gIsAndroid
+},
+function* test_instantiate() {
+  const ID = "TELEMETRY_TEST_COUNT";
+  let h = Telemetry.getHistogramById(ID);
+
+  // Instantiate the subsession histogram through |add| and make sure they match.
+  // This MUST be the first use of "TELEMETRY_TEST_COUNT" in this file, otherwise
+  // |add| will not instantiate the histogram.
+  h.add(1);
+  let snapshot = h.snapshot();
+  let subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.equal(snapshot.sum, subsession[ID].sum,
+               "Histogram and subsession histogram sum must match.");
+  // Clear the histogram, so we don't void the assumptions from the other tests.
+  h.clear();
+});
+
+add_task(function* test_parameterChecks() {
+  let kinds = [Telemetry.HISTOGRAM_EXPONENTIAL, Telemetry.HISTOGRAM_LINEAR]
+  let testNames = ["TELEMETRY_TEST_EXPONENTIAL", "TELEMETRY_TEST_LINEAR"]
+  for (let i = 0; i < kinds.length; i++) {
+    let histogram_type = kinds[i];
+    let test_type = testNames[i];
+    let [min, max, bucket_count] = [1, INT_MAX - 1, 10]
+    check_histogram(histogram_type, test_type, min, max, bucket_count);
+  }
+});
+
+add_task(function* test_noSerialization() {
+  // Instantiate the storage for this histogram and make sure it doesn't
+  // get reflected into JS, as it has no interesting data in it.
+  Telemetry.getHistogramById("NEWTAB_PAGE_PINNED_SITES_COUNT");
+  do_check_false("NEWTAB_PAGE_PINNED_SITES_COUNT" in Telemetry.histogramSnapshots);
+});
+
+add_task(function* test_boolean_histogram() {
+  var h = Telemetry.getHistogramById("TELEMETRY_TEST_BOOLEAN");
+  var r = h.snapshot().ranges;
+  // boolean histograms ignore numeric parameters
+  do_check_eq(uneval(r), uneval([0, 1, 2]))
+  for (var i=0;i<r.length;i++) {
+    var v = r[i];
+    h.add(v);
+  }
+  h.add(true);
+  h.add(false);
+  var s = h.snapshot();
+  do_check_eq(s.histogram_type, Telemetry.HISTOGRAM_BOOLEAN);
+  // last bucket should always be 0 since .add parameters are normalized to either 0 or 1
+  do_check_eq(s.counts[2], 0);
+  do_check_eq(s.sum, 3);
+  do_check_eq(s.counts[0], 2);
+});
+
+add_task(function* test_flag_histogram() {
+  var h = Telemetry.getHistogramById("TELEMETRY_TEST_FLAG");
+  var r = h.snapshot().ranges;
+  // Flag histograms ignore numeric parameters.
+  do_check_eq(uneval(r), uneval([0, 1, 2]));
+  // Should already have a 0 counted.
+  var c = h.snapshot().counts;
+  var s = h.snapshot().sum;
+  do_check_eq(uneval(c), uneval([1, 0, 0]));
+  do_check_eq(s, 0);
+  // Should switch counts.
+  h.add(1);
+  var c2 = h.snapshot().counts;
+  var s2 = h.snapshot().sum;
+  do_check_eq(uneval(c2), uneval([0, 1, 0]));
+  do_check_eq(s2, 1);
+  // Should only switch counts once.
+  h.add(1);
+  var c3 = h.snapshot().counts;
+  var s3 = h.snapshot().sum;
+  do_check_eq(uneval(c3), uneval([0, 1, 0]));
+  do_check_eq(s3, 1);
+  do_check_eq(h.snapshot().histogram_type, Telemetry.HISTOGRAM_FLAG);
+});
+
+add_task(function* test_count_histogram() {
+  let h = Telemetry.getHistogramById("TELEMETRY_TEST_COUNT2");
+  let s = h.snapshot();
+  do_check_eq(uneval(s.ranges), uneval([0, 1, 2]));
+  do_check_eq(uneval(s.counts), uneval([0, 0, 0]));
+  do_check_eq(s.sum, 0);
+  h.add();
+  s = h.snapshot();
+  do_check_eq(uneval(s.counts), uneval([1, 0, 0]));
+  do_check_eq(s.sum, 1);
+  h.add();
+  s = h.snapshot();
+  do_check_eq(uneval(s.counts), uneval([2, 0, 0]));
+  do_check_eq(s.sum, 2);
+});
+
+add_task(function* test_categorical_histogram()
+{
+  let h1 = Telemetry.getHistogramById("TELEMETRY_TEST_CATEGORICAL");
+  for (let v of ["CommonLabel", "Label2", "Label3", "Label3", 0, 0, 1]) {
+    h1.add(v);
+  }
+  for (let s of ["", "Label4", "1234"]) {
+    Assert.throws(() => h1.add(s));
+  }
+
+  let snapshot = h1.snapshot();
+  Assert.equal(snapshot.sum, 6);
+  Assert.deepEqual(snapshot.ranges, [0, 1, 2, 3]);
+  Assert.deepEqual(snapshot.counts, [3, 2, 2, 0]);
+
+  let h2 = Telemetry.getHistogramById("TELEMETRY_TEST_CATEGORICAL_OPTOUT");
+  for (let v of ["CommonLabel", "CommonLabel", "Label4", "Label5", "Label6", 0, 1]) {
+    h2.add(v);
+  }
+  for (let s of ["", "Label3", "1234"]) {
+    Assert.throws(() => h2.add(s));
+  }
+
+  snapshot = h2.snapshot();
+  Assert.equal(snapshot.sum, 7);
+  Assert.deepEqual(snapshot.ranges, [0, 1, 2, 3, 4]);
+  Assert.deepEqual(snapshot.counts, [3, 2, 1, 1, 0]);
+});
+
+add_task(function* test_getHistogramById() {
+  try {
+    Telemetry.getHistogramById("nonexistent");
+    do_throw("This can't happen");
+  } catch (e) {
+
+  }
+  var h = Telemetry.getHistogramById("CYCLE_COLLECTOR");
+  var s = h.snapshot();
+  do_check_eq(s.histogram_type, Telemetry.HISTOGRAM_EXPONENTIAL);
+  do_check_eq(s.min, 1);
+  do_check_eq(s.max, 10000);
+});
+
+add_task(function* test_getSlowSQL() {
+  var slow = Telemetry.slowSQL;
+  do_check_true(("mainThread" in slow) && ("otherThreads" in slow));
+});
+
+add_task(function* test_getWebrtc() {
+  var webrtc = Telemetry.webrtcStats;
+  do_check_true("IceCandidatesStats" in webrtc);
+  var icestats = webrtc.IceCandidatesStats;
+  do_check_true("webrtc" in icestats);
+});
+
+// Check that telemetry doesn't record in private mode
+add_task(function* test_privateMode() {
+  var h = Telemetry.getHistogramById("TELEMETRY_TEST_BOOLEAN");
+  var orig = h.snapshot();
+  Telemetry.canRecordExtended = false;
+  h.add(1);
+  do_check_eq(uneval(orig), uneval(h.snapshot()));
+  Telemetry.canRecordExtended = true;
+  h.add(1);
+  do_check_neq(uneval(orig), uneval(h.snapshot()));
+});
+
+// Check that telemetry records only when it is suppose to.
+add_task(function* test_histogramRecording() {
+  // Check that no histogram is recorded if both base and extended recording are off.
+  Telemetry.canRecordBase = false;
+  Telemetry.canRecordExtended = false;
+
+  let h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTOUT");
+  h.clear();
+  let orig = h.snapshot();
+  h.add(1);
+  Assert.equal(orig.sum, h.snapshot().sum);
+
+  // Check that only base histograms are recorded.
+  Telemetry.canRecordBase = true;
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+               "Histogram value should have incremented by 1 due to recording.");
+
+  // Extended histograms should not be recorded.
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTIN");
+  orig = h.snapshot();
+  h.add(1);
+  Assert.equal(orig.sum, h.snapshot().sum,
+               "Histograms should be equal after recording.");
+
+  // Runtime created histograms should not be recorded.
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_BOOLEAN");
+  orig = h.snapshot();
+  h.add(1);
+  Assert.equal(orig.sum, h.snapshot().sum,
+               "Histograms should be equal after recording.");
+
+  // Check that extended histograms are recorded when required.
+  Telemetry.canRecordExtended = true;
+
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+               "Runtime histogram value should have incremented by 1 due to recording.");
+
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTIN");
+  orig = h.snapshot();
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+               "Histogram value should have incremented by 1 due to recording.");
+
+  // Check that base histograms are still being recorded.
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTOUT");
+  h.clear();
+  orig = h.snapshot();
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+               "Histogram value should have incremented by 1 due to recording.");
+});
+
+add_task(function* test_addons() {
+  var addon_id = "testing-addon";
+  var fake_addon_id = "fake-addon";
+  var name1 = "testing-histogram1";
+  var register = Telemetry.registerAddonHistogram;
+  expect_success(() =>
+                 register(addon_id, name1, Telemetry.HISTOGRAM_LINEAR, 1, 5, 6));
+  // Can't register the same histogram multiple times.
+  expect_fail(() =>
+              register(addon_id, name1, Telemetry.HISTOGRAM_LINEAR, 1, 5, 6));
+  // Make sure we can't get at it with another name.
+  expect_fail(() => Telemetry.getAddonHistogram(fake_addon_id, name1));
+
+  // Check for reflection capabilities.
+  var h1 = Telemetry.getAddonHistogram(addon_id, name1);
+  // Verify that although we've created storage for it, we don't reflect it into JS.
+  var snapshots = Telemetry.addonHistogramSnapshots;
+  do_check_false(name1 in snapshots[addon_id]);
+  h1.add(1);
+  h1.add(3);
+  var s1 = h1.snapshot();
+  do_check_eq(s1.histogram_type, Telemetry.HISTOGRAM_LINEAR);
+  do_check_eq(s1.min, 1);
+  do_check_eq(s1.max, 5);
+  do_check_eq(s1.counts[1], 1);
+  do_check_eq(s1.counts[3], 1);
+
+  var name2 = "testing-histogram2";
+  expect_success(() =>
+                 register(addon_id, name2, Telemetry.HISTOGRAM_LINEAR, 2, 4, 4));
+
+  var h2 = Telemetry.getAddonHistogram(addon_id, name2);
+  h2.add(2);
+  h2.add(3);
+  var s2 = h2.snapshot();
+  do_check_eq(s2.histogram_type, Telemetry.HISTOGRAM_LINEAR);
+  do_check_eq(s2.min, 2);
+  do_check_eq(s2.max, 4);
+  do_check_eq(s2.counts[1], 1);
+  do_check_eq(s2.counts[2], 1);
+
+  // Check that we can register histograms for a different addon with
+  // identical names.
+  var extra_addon = "testing-extra-addon";
+  expect_success(() =>
+                 register(extra_addon, name1, Telemetry.HISTOGRAM_BOOLEAN));
+
+  // Check that we can register flag histograms.
+  var flag_addon = "testing-flag-addon";
+  var flag_histogram = "flag-histogram";
+  expect_success(() =>
+                 register(flag_addon, flag_histogram, Telemetry.HISTOGRAM_FLAG));
+  expect_success(() =>
+                 register(flag_addon, name2, Telemetry.HISTOGRAM_LINEAR, 2, 4, 4));
+
+  // Check that we reflect registered addons and histograms.
+  snapshots = Telemetry.addonHistogramSnapshots;
+  do_check_true(addon_id in snapshots)
+  do_check_true(extra_addon in snapshots);
+  do_check_true(flag_addon in snapshots);
+
+  // Check that we have data for our created histograms.
+  do_check_true(name1 in snapshots[addon_id]);
+  do_check_true(name2 in snapshots[addon_id]);
+  var s1_alt = snapshots[addon_id][name1];
+  var s2_alt = snapshots[addon_id][name2];
+  do_check_eq(s1_alt.min, s1.min);
+  do_check_eq(s1_alt.max, s1.max);
+  do_check_eq(s1_alt.histogram_type, s1.histogram_type);
+  do_check_eq(s2_alt.min, s2.min);
+  do_check_eq(s2_alt.max, s2.max);
+  do_check_eq(s2_alt.histogram_type, s2.histogram_type);
+
+  // Even though we've registered it, it shouldn't show up until data is added to it.
+  do_check_false(name1 in snapshots[extra_addon]);
+
+  // Flag histograms should show up automagically.
+  do_check_true(flag_histogram in snapshots[flag_addon]);
+  do_check_false(name2 in snapshots[flag_addon]);
+
+  // Check that we can remove addon histograms.
+  Telemetry.unregisterAddonHistograms(addon_id);
+  snapshots = Telemetry.addonHistogramSnapshots;
+  do_check_false(addon_id in snapshots);
+  // Make sure other addons are unaffected.
+  do_check_true(extra_addon in snapshots);
+});
+
+add_task(function* test_expired_histogram() {
+  var test_expired_id = "TELEMETRY_TEST_EXPIRED";
+  var dummy = Telemetry.getHistogramById(test_expired_id);
+  var rh = Telemetry.registeredHistograms(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN, []);
+  Assert.ok(!!rh);
+
+  dummy.add(1);
+
+  do_check_eq(Telemetry.histogramSnapshots["__expired__"], undefined);
+  do_check_eq(Telemetry.histogramSnapshots[test_expired_id], undefined);
+  do_check_eq(rh[test_expired_id], undefined);
+});
+
+add_task(function* test_keyed_histogram() {
+  // Check that invalid names get rejected.
+
+  let threw = false;
+  try {
+    Telemetry.getKeyedHistogramById("test::unknown histogram", "never", Telemetry.HISTOGRAM_BOOLEAN);
+  } catch (e) {
+    // This should throw as it is an unknown ID
+    threw = true;
+  }
+  Assert.ok(threw, "getKeyedHistogramById should have thrown");
+});
+
+add_task(function* test_keyed_boolean_histogram() {
+  const KEYED_ID = "TELEMETRY_TEST_KEYED_BOOLEAN";
+  let KEYS = numberRange(0, 2).map(i => "key" + (i + 1));
+  KEYS.push("漢語");
+  let histogramBase = {
+    "min": 1,
+    "max": 2,
+    "histogram_type": 2,
+    "sum": 1,
+    "ranges": [0, 1, 2],
+    "counts": [0, 1, 0]
+  };
+  let testHistograms = numberRange(0, 3).map(i => JSON.parse(JSON.stringify(histogramBase)));
+  let testKeys = [];
+  let testSnapShot = {};
+
+  let h = Telemetry.getKeyedHistogramById(KEYED_ID);
+  for (let i=0; i<2; ++i) {
+    let key = KEYS[i];
+    h.add(key, true);
+    testSnapShot[key] = testHistograms[i];
+    testKeys.push(key);
+
+    Assert.deepEqual(h.keys().sort(), testKeys);
+    Assert.deepEqual(h.snapshot(), testSnapShot);
+  }
+
+  h = Telemetry.getKeyedHistogramById(KEYED_ID);
+  Assert.deepEqual(h.keys().sort(), testKeys);
+  Assert.deepEqual(h.snapshot(), testSnapShot);
+
+  let key = KEYS[2];
+  h.add(key, false);
+  testKeys.push(key);
+  testSnapShot[key] = testHistograms[2];
+  testSnapShot[key].sum = 0;
+  testSnapShot[key].counts = [1, 0, 0];
+  Assert.deepEqual(h.keys().sort(), testKeys);
+  Assert.deepEqual(h.snapshot(), testSnapShot);
+
+  let allSnapshots = Telemetry.keyedHistogramSnapshots;
+  Assert.deepEqual(allSnapshots[KEYED_ID], testSnapShot);
+
+  h.clear();
+  Assert.deepEqual(h.keys(), []);
+  Assert.deepEqual(h.snapshot(), {});
+});
+
+add_task(function* test_keyed_count_histogram() {
+  const KEYED_ID = "TELEMETRY_TEST_KEYED_COUNT";
+  const KEYS = numberRange(0, 5).map(i => "key" + (i + 1));
+  let histogramBase = {
+    "min": 1,
+    "max": 2,
+    "histogram_type": 4,
+    "sum": 0,
+    "ranges": [0, 1, 2],
+    "counts": [1, 0, 0]
+  };
+  let testHistograms = numberRange(0, 5).map(i => JSON.parse(JSON.stringify(histogramBase)));
+  let testKeys = [];
+  let testSnapShot = {};
+
+  let h = Telemetry.getKeyedHistogramById(KEYED_ID);
+  for (let i=0; i<4; ++i) {
+    let key = KEYS[i];
+    let value = i*2 + 1;
+
+    for (let k=0; k<value; ++k) {
+      h.add(key);
+    }
+    testHistograms[i].counts[0] = value;
+    testHistograms[i].sum = value;
+    testSnapShot[key] = testHistograms[i];
+    testKeys.push(key);
+
+    Assert.deepEqual(h.keys().sort(), testKeys);
+    Assert.deepEqual(h.snapshot(key), testHistograms[i]);
+    Assert.deepEqual(h.snapshot(), testSnapShot);
+  }
+
+  h = Telemetry.getKeyedHistogramById(KEYED_ID);
+  Assert.deepEqual(h.keys().sort(), testKeys);
+  Assert.deepEqual(h.snapshot(), testSnapShot);
+
+  let key = KEYS[4];
+  h.add(key);
+  testKeys.push(key);
+  testHistograms[4].counts[0] = 1;
+  testHistograms[4].sum = 1;
+  testSnapShot[key] = testHistograms[4];
+
+  Assert.deepEqual(h.keys().sort(), testKeys);
+  Assert.deepEqual(h.snapshot(), testSnapShot);
+
+  let allSnapshots = Telemetry.keyedHistogramSnapshots;
+  Assert.deepEqual(allSnapshots[KEYED_ID], testSnapShot);
+
+  h.clear();
+  Assert.deepEqual(h.keys(), []);
+  Assert.deepEqual(h.snapshot(), {});
+});
+
+add_task(function* test_keyed_flag_histogram() {
+  const KEYED_ID = "TELEMETRY_TEST_KEYED_FLAG";
+  let h = Telemetry.getKeyedHistogramById(KEYED_ID);
+
+  const KEY = "default";
+  h.add(KEY, true);
+
+  let testSnapshot = {};
+  testSnapshot[KEY] = {
+    "min": 1,
+    "max": 2,
+    "histogram_type": 3,
+    "sum": 1,
+    "ranges": [0, 1, 2],
+    "counts": [0, 1, 0]
+  };
+
+  Assert.deepEqual(h.keys().sort(), [KEY]);
+  Assert.deepEqual(h.snapshot(), testSnapshot);
+
+  let allSnapshots = Telemetry.keyedHistogramSnapshots;
+  Assert.deepEqual(allSnapshots[KEYED_ID], testSnapshot);
+
+  h.clear();
+  Assert.deepEqual(h.keys(), []);
+  Assert.deepEqual(h.snapshot(), {});
+});
+
+add_task(function* test_keyed_histogram_recording() {
+  // Check that no histogram is recorded if both base and extended recording are off.
+  Telemetry.canRecordBase = false;
+  Telemetry.canRecordExtended = false;
+
+  const TEST_KEY = "record_foo";
+  let h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT");
+  h.clear();
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 0);
+
+  // Check that only base histograms are recorded.
+  Telemetry.canRecordBase = true;
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+               "The keyed histogram should record the correct value.");
+
+  // Extended set keyed histograms should not be recorded.
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTIN");
+  h.clear();
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 0,
+               "The keyed histograms should not record any data.");
+
+  // Check that extended histograms are recorded when required.
+  Telemetry.canRecordExtended = true;
+
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+                  "The runtime keyed histogram should record the correct value.");
+
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTIN");
+  h.clear();
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+               "The keyed histogram should record the correct value.");
+
+  // Check that base histograms are still being recorded.
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT");
+  h.clear();
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1);
+});
+
+add_task(function* test_histogram_recording_enabled() {
+  Telemetry.canRecordBase = true;
+  Telemetry.canRecordExtended = true;
+
+  // Check that a "normal" histogram respects recording-enabled on/off
+  var h = Telemetry.getHistogramById("TELEMETRY_TEST_COUNT");
+  var orig = h.snapshot();
+
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+              "add should record by default.");
+
+  // Check that when recording is disabled - add is ignored
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_COUNT", false);
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+              "When recording is disabled add should not record.");
+
+  // Check that we're back to normal after recording is enabled
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_COUNT", true);
+  h.add(1);
+  Assert.equal(orig.sum + 2, h.snapshot().sum,
+               "When recording is re-enabled add should record.");
+
+  // Check that we're correctly accumulating values other than 1.
+  h.clear();
+  h.add(3);
+  Assert.equal(3, h.snapshot().sum, "Recording counts greater than 1 should work.");
+
+  // Check that a histogram with recording disabled by default behaves correctly
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_COUNT_INIT_NO_RECORD");
+  orig = h.snapshot();
+
+  h.add(1);
+  Assert.equal(orig.sum, h.snapshot().sum,
+               "When recording is disabled by default, add should not record by default.");
+
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_COUNT_INIT_NO_RECORD", true);
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+               "When recording is enabled add should record.");
+
+  // Restore to disabled
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_COUNT_INIT_NO_RECORD", false);
+  h.add(1);
+  Assert.equal(orig.sum + 1, h.snapshot().sum,
+               "When recording is disabled add should not record.");
+});
+
+add_task(function* test_keyed_histogram_recording_enabled() {
+  Telemetry.canRecordBase = true;
+  Telemetry.canRecordExtended = true;
+
+  // Check RecordingEnabled for keyed histograms which are recording by default
+  const TEST_KEY = "record_foo";
+  let h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT");
+
+  h.clear();
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+    "Keyed histogram add should record by default");
+
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT", false);
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+    "Keyed histogram add should not record when recording is disabled");
+
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT", true);
+  h.clear();
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+    "Keyed histogram add should record when recording is re-enabled");
+
+  // Check that a histogram with recording disabled by default behaves correctly
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_COUNT_INIT_NO_RECORD");
+  h.clear();
+
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 0,
+    "Keyed histogram add should not record by default for histograms which don't record by default");
+
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_KEYED_COUNT_INIT_NO_RECORD", true);
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+    "Keyed histogram add should record when recording is enabled");
+
+  // Restore to disabled
+  Telemetry.setHistogramRecordingEnabled("TELEMETRY_TEST_KEYED_COUNT_INIT_NO_RECORD", false);
+  h.add(TEST_KEY, 1);
+  Assert.equal(h.snapshot(TEST_KEY).sum, 1,
+    "Keyed histogram add should not record when recording is disabled");
+});
+
+add_task(function* test_datasets() {
+  // Check that datasets work as expected.
+
+  const RELEASE_CHANNEL_OPTOUT = Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTOUT;
+  const RELEASE_CHANNEL_OPTIN  = Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN;
+
+  // Histograms should default to the extended dataset
+  let h = Telemetry.getHistogramById("TELEMETRY_TEST_FLAG");
+  Assert.equal(h.dataset(), RELEASE_CHANNEL_OPTIN);
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_FLAG");
+  Assert.equal(h.dataset(), RELEASE_CHANNEL_OPTIN);
+
+  // Check test histograms with explicit dataset definitions
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTIN");
+  Assert.equal(h.dataset(), RELEASE_CHANNEL_OPTIN);
+  h = Telemetry.getHistogramById("TELEMETRY_TEST_RELEASE_OPTOUT");
+  Assert.equal(h.dataset(), RELEASE_CHANNEL_OPTOUT);
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTIN");
+  Assert.equal(h.dataset(), RELEASE_CHANNEL_OPTIN);
+  h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT");
+  Assert.equal(h.dataset(), RELEASE_CHANNEL_OPTOUT);
+
+  // Check that registeredHistogram works properly
+  let registered = Telemetry.registeredHistograms(RELEASE_CHANNEL_OPTIN, []);
+  registered = new Set(registered);
+  Assert.ok(registered.has("TELEMETRY_TEST_FLAG"));
+  Assert.ok(registered.has("TELEMETRY_TEST_RELEASE_OPTIN"));
+  Assert.ok(registered.has("TELEMETRY_TEST_RELEASE_OPTOUT"));
+  registered = Telemetry.registeredHistograms(RELEASE_CHANNEL_OPTOUT, []);
+  registered = new Set(registered);
+  Assert.ok(!registered.has("TELEMETRY_TEST_FLAG"));
+  Assert.ok(!registered.has("TELEMETRY_TEST_RELEASE_OPTIN"));
+  Assert.ok(registered.has("TELEMETRY_TEST_RELEASE_OPTOUT"));
+
+  // Check that registeredKeyedHistograms works properly
+  registered = Telemetry.registeredKeyedHistograms(RELEASE_CHANNEL_OPTIN, []);
+  registered = new Set(registered);
+  Assert.ok(registered.has("TELEMETRY_TEST_KEYED_FLAG"));
+  Assert.ok(registered.has("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT"));
+  registered = Telemetry.registeredKeyedHistograms(RELEASE_CHANNEL_OPTOUT, []);
+  registered = new Set(registered);
+  Assert.ok(!registered.has("TELEMETRY_TEST_KEYED_FLAG"));
+  Assert.ok(registered.has("TELEMETRY_TEST_KEYED_RELEASE_OPTOUT"));
+});
+
+add_task({
+  skip_if: () => gIsAndroid
+},
+function* test_subsession() {
+  const ID = "TELEMETRY_TEST_COUNT";
+  const FLAG = "TELEMETRY_TEST_FLAG";
+  let h = Telemetry.getHistogramById(ID);
+  let flag = Telemetry.getHistogramById(FLAG);
+
+  // Both original and duplicate should start out the same.
+  h.clear();
+  let snapshot = Telemetry.histogramSnapshots;
+  let subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.ok(!(ID in snapshot));
+  Assert.ok(!(ID in subsession));
+
+  // They should instantiate and pick-up the count.
+  h.add(1);
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.ok(ID in snapshot);
+  Assert.ok(ID in subsession);
+  Assert.equal(snapshot[ID].sum, 1);
+  Assert.equal(subsession[ID].sum, 1);
+
+  // They should still reset properly.
+  h.clear();
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.ok(!(ID in snapshot));
+  Assert.ok(!(ID in subsession));
+
+  // Both should instantiate and pick-up the count.
+  h.add(1);
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.equal(snapshot[ID].sum, 1);
+  Assert.equal(subsession[ID].sum, 1);
+
+  // Check that we are able to only reset the duplicate histogram.
+  h.clear(true);
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.ok(ID in snapshot);
+  Assert.ok(ID in subsession);
+  Assert.equal(snapshot[ID].sum, 1);
+  Assert.equal(subsession[ID].sum, 0);
+
+  // Both should register the next count.
+  h.add(1);
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.equal(snapshot[ID].sum, 2);
+  Assert.equal(subsession[ID].sum, 1);
+
+  // Retrieve a subsession snapshot and pass the flag to
+  // clear subsession histograms too.
+  h.clear();
+  flag.clear();
+  h.add(1);
+  flag.add(1);
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms(true);
+  Assert.ok(ID in snapshot);
+  Assert.ok(ID in subsession);
+  Assert.ok(FLAG in snapshot);
+  Assert.ok(FLAG in subsession);
+  Assert.equal(snapshot[ID].sum, 1);
+  Assert.equal(subsession[ID].sum, 1);
+  Assert.equal(snapshot[FLAG].sum, 1);
+  Assert.equal(subsession[FLAG].sum, 1);
+
+  // The next subsesssion snapshot should show the histograms
+  // got reset.
+  snapshot = Telemetry.histogramSnapshots;
+  subsession = Telemetry.snapshotSubsessionHistograms();
+  Assert.ok(ID in snapshot);
+  Assert.ok(ID in subsession);
+  Assert.ok(FLAG in snapshot);
+  Assert.ok(FLAG in subsession);
+  Assert.equal(snapshot[ID].sum, 1);
+  Assert.equal(subsession[ID].sum, 0);
+  Assert.equal(snapshot[FLAG].sum, 1);
+  Assert.equal(subsession[FLAG].sum, 0);
+});
+
+add_task({
+  skip_if: () => gIsAndroid
+},
+function* test_keyed_subsession() {
+  let h = Telemetry.getKeyedHistogramById("TELEMETRY_TEST_KEYED_FLAG");
+  const KEY = "foo";
+
+  // Both original and subsession should start out the same.
+  h.clear();
+  Assert.ok(!(KEY in h.snapshot()));
+  Assert.ok(!(KEY in h.subsessionSnapshot()));
+  Assert.equal(h.snapshot(KEY).sum, 0);
+  Assert.equal(h.subsessionSnapshot(KEY).sum, 0);
+
+  // Both should register the flag.
+  h.add(KEY, 1);
+  Assert.ok(KEY in h.snapshot());
+  Assert.ok(KEY in h.subsessionSnapshot());
+  Assert.equal(h.snapshot(KEY).sum, 1);
+  Assert.equal(h.subsessionSnapshot(KEY).sum, 1);
+
+  // Check that we are able to only reset the subsession histogram.
+  h.clear(true);
+  Assert.ok(KEY in h.snapshot());
+  Assert.ok(!(KEY in h.subsessionSnapshot()));
+  Assert.equal(h.snapshot(KEY).sum, 1);
+  Assert.equal(h.subsessionSnapshot(KEY).sum, 0);
+
+  // Setting the flag again should make both match again.
+  h.add(KEY, 1);
+  Assert.ok(KEY in h.snapshot());
+  Assert.ok(KEY in h.subsessionSnapshot());
+  Assert.equal(h.snapshot(KEY).sum, 1);
+  Assert.equal(h.subsessionSnapshot(KEY).sum, 1);
+
+  // Check that "snapshot and clear" works properly.
+  let snapshot = h.snapshot();
+  let subsession = h.snapshotSubsessionAndClear();
+  Assert.ok(KEY in snapshot);
+  Assert.ok(KEY in subsession);
+  Assert.equal(snapshot[KEY].sum, 1);
+  Assert.equal(subsession[KEY].sum, 1);
+
+  subsession = h.subsessionSnapshot();
+  Assert.ok(!(KEY in subsession));
+  Assert.equal(h.subsessionSnapshot(KEY).sum, 0);
+});

toolkit/components/telemetry/tests/unit/xpcshell.ini

@@ -0,0 +1,63 @@
+[DEFAULT]
+head = head.js
+tail =
+firefox-appdir = browser
+# The *.xpi files are only needed for test_TelemetryEnvironment.js, but
+# xpcshell fails to install tests if we move them under the test entry.
+support-files =
+  ../search/chrome.manifest
+  ../search/searchTest.jar
+  dictionary.xpi
+  experiment.xpi
+  extension.xpi
+  extension-2.xpi
+  engine.xml
+  system.xpi
+  restartless.xpi
+  theme.xpi
+  !/toolkit/mozapps/extensions/test/xpcshell/head_addons.js
+generated-files =
+  dictionary.xpi
+  experiment.xpi
+  extension.xpi
+  extension-2.xpi
+  system.xpi
+  restartless.xpi
+  theme.xpi
+
+[test_nsITelemetry.js]
+[test_SubsessionChaining.js]
+tags = addons
+[test_TelemetryEnvironment.js]
+skip-if = os == "android"
+tags = addons
+[test_PingAPI.js]
+skip-if = os == "android"
+[test_TelemetryFlagClear.js]
+[test_TelemetryLateWrites.js]
+[test_TelemetryLockCount.js]
+[test_TelemetryLog.js]
+[test_TelemetryController.js]
+tags = addons
+[test_TelemetryController_idle.js]
+[test_TelemetryControllerShutdown.js]
+tags = addons
+[test_TelemetryStopwatch.js]
+[test_TelemetryControllerBuildID.js]
+[test_TelemetrySendOldPings.js]
+skip-if = os == "android" # Disabled due to intermittent orange on Android
+tags = addons
+[test_TelemetrySession.js]
+tags = addons
+[test_ThreadHangStats.js]
+run-sequentially = Bug 1046307, test can fail intermittently when CPU load is high
+[test_TelemetrySend.js]
+[test_ChildHistograms.js]
+skip-if = os == "android"
+tags = addons
+[test_TelemetryReportingPolicy.js]
+tags = addons
+[test_TelemetryScalars.js]
+[test_TelemetryTimestamps.js]
+skip-if = toolkit == 'android'
+[test_TelemetryEvents.js]