wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-10-10 01:07:31 +09:00
Code Issues Projects Releases Packages Wiki Activity

Issue #21 - Remove Telemetry plumbing and fix build.

Browse source 
Note this won't give working applications. Requires FE changes and
additional js module changes (next part).
This commit is contained in:
Moonchild 2021-10-29 11:02:27 +00:00 • committed by roytam1
commit 8d800b1cb0
205 changed files with 45 additions and 40519 deletions
Download patch file Download diff file Expand all files Collapse all files

1
toolkit/components/moz.build
View file

@ -51,7 +51,6 @@ DIRS += [
'reflect',
'startup',
'statusfilter',
'telemetry',
'thumbnails',
'timermanager',
'tooltiptext',

1
toolkit/components/startup/StartupTimeline.cpp
View file

@ -3,7 +3,6 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "StartupTimeline.h"
#include "mozilla/Telemetry.h"
#include "mozilla/TimeStamp.h"
#include "nsXULAppAPI.h"

3
toolkit/components/startup/StartupTimeline.h
View file

@ -46,9 +46,6 @@ NS_VISIBILITY_DEFAULT __attribute__((weak));
namespace mozilla {
void RecordShutdownEndTimeStamp();
void RecordShutdownStartTimeStamp();
class StartupTimeline {
public:
enum Event {

2
toolkit/components/startup/nsAppStartup.cpp
View file

@ -48,7 +48,6 @@
#endif
#include "mozilla/IOInterposer.h"
#include "mozilla/Telemetry.h"
#include "mozilla/StartupTimeline.h"
static NS_DEFINE_CID(kAppShellCID, NS_APPSHELL_CID);
@ -344,7 +343,6 @@ nsAppStartup::Quit(uint32_t aMode)
}
PROFILER_MARKER("Shutdown start");
mozilla::RecordShutdownStartTimeStamp();
mShuttingDown = true;
if (!mRestart) {
mRestart = (aMode & eRestart) != 0;

52
toolkit/components/telemetry/EventInfo.h
View file

@ -1,52 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TelemetryEventInfo_h__
#define TelemetryEventInfo_h__
// This module is internal to Telemetry. The structures here hold data that
// describe events.
// It should only be used by TelemetryEventData.h and TelemetryEvent.cpp.
//
// For the public interface to Telemetry functionality, see Telemetry.h.
namespace {
struct CommonEventInfo {
// Indices for the category and expiration strings.
uint32_t category_offset;
uint32_t expiration_version_offset;
// The index and count for the extra key offsets in the extra table.
uint32_t extra_index;
uint32_t extra_count;
// The day since UNIX epoch that this probe expires on.
uint32_t expiration_day;
// The dataset this event is recorded in.
uint32_t dataset;
// Convenience functions for accessing event strings.
const char* expiration_version() const;
const char* category() const;
const char* extra_key(uint32_t index) const;
};
struct EventInfo {
// The corresponding CommonEventInfo.
const CommonEventInfo& common_info;
// Indices for the method & object strings.
uint32_t method_offset;
uint32_t object_offset;
const char* method() const;
const char* object() const;
};
} // namespace
#endif // TelemetryEventInfo_h__

68
toolkit/components/telemetry/Events.yaml
View file

@ -1,68 +0,0 @@
navigation:
- methods: ["search"]
objects: ["about_home", "about_newtab", "contextmenu", "oneoff",
"suggestion", "alias", "enter", "searchbar", "urlbar"]
release_channel_collection: opt-in
description: >
This is recorded on each search navigation.
The value field records the action used to trigger the search:
"enter", "oneoff", "suggestion", "alias", null (for contextmenu)
bug_numbers: [1316281]
notification_emails: ["past@mozilla.com"]
expiry_version: "58.0"
extra_keys:
engine: The id of the search engine used.
# This category contains event entries used for Telemetry tests.
# They will not be sent out with any pings.
telemetry.test:
- methods: ["test1", "test2"]
objects: ["object1", "object2"]
bug_numbers: [1286606]
notification_emails: ["telemetry-client-dev@mozilla.com"]
description: This is a test entry for Telemetry.
expiry_date: never
extra_keys:
key1: This is just a test description.
key2: This is another test description.
- methods: ["optout"]
objects: ["object1", "object2"]
bug_numbers: [1286606]
notification_emails: ["telemetry-client-dev@mozilla.com"]
description: This is an opt-out test entry.
expiry_date: never
release_channel_collection: opt-out
extra_keys:
key1: This is just a test description.
- methods: ["expired_version"]
objects: ["object1", "object2"]
bug_numbers: [1286606]
notification_emails: ["telemetry-client-dev@mozilla.com"]
description: This is a test entry with an expired version.
expiry_version: "3.6"
- methods: ["expired_date"]
objects: ["object1", "object2"]
bug_numbers: [1286606]
notification_emails: ["telemetry-client-dev@mozilla.com"]
description: This is a test entry with an expired date.
expiry_date: 2014-01-28
- methods: ["not_expired_optout"]
objects: ["object1"]
bug_numbers: [1286606]
notification_emails: ["telemetry-client-dev@mozilla.com"]
description: This is an opt-out test entry with unexpired date and version.
release_channel_collection: opt-out
expiry_date: 2099-01-01
expiry_version: "999.0"
# This is a secondary category used for Telemetry tests.
# The events here will not be sent out with any pings.
telemetry.test.second:
- methods: ["test"]
objects: ["object1", "object2", "object3"]
bug_numbers: [1286606]
notification_emails: ["telemetry-client-dev@mozilla.com"]
description: This is a test entry for Telemetry.
expiry_date: never
extra_keys:
key1: This is just a test description.

216
toolkit/components/telemetry/GCTelemetry.jsm
View file

@ -1,216 +0,0 @@
/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
/**
* This module records detailed timing information about selected
* GCs. The data is sent back in the telemetry session ping. To avoid
* bloating the ping, only a few GCs are included. There are two
* selection strategies. We always save the five GCs with the worst
* max_pause time. Additionally, five collections are selected at
* random. If a GC runs for C milliseconds and the total time for all
* GCs since the session began is T milliseconds, then the GC has a
* 5*C/T probablility of being selected (the factor of 5 is because we
* save 5 of them).
*
* GCs from both the main process and all content processes are
* recorded. The data is cleared for each new subsession.
*/
const Cu = Components.utils;
Cu.import("resource://gre/modules/Services.jsm", this);
this.EXPORTED_SYMBOLS = ["GCTelemetry"];
// Names of processes where we record GCs.
const PROCESS_NAMES = ["main", "content"];
// Should be the time we started up in milliseconds since the epoch.
const BASE_TIME = Date.now() - Services.telemetry.msSinceProcessStart();
// Records selected GCs. There is one instance per process type.
class GCData {
constructor(kind) {
let numRandom = {main: 0, content: 2};
let numWorst = {main: 2, content: 2};
this.totalGCTime = 0;
this.randomlySelected = Array(numRandom[kind]).fill(null);
this.worst = Array(numWorst[kind]).fill(null);
}
// Turn absolute timestamps (in microseconds since the epoch) into
// milliseconds since startup.
rebaseTimes(data) {
function fixup(t) {
return t / 1000.0 - BASE_TIME;
}
data.timestamp = fixup(data.timestamp);
for (let i = 0; i < data.slices.length; i++) {
let slice = data.slices[i];
slice.start_timestamp = fixup(slice.start_timestamp);
slice.end_timestamp = fixup(slice.end_timestamp);
}
}
// Records a GC (represented by |data|) in the randomlySelected or
// worst batches depending on the criteria above.
record(data) {
this.rebaseTimes(data);
let time = data.total_time;
this.totalGCTime += time;
// Probability that we will replace any one of our
// current randomlySelected GCs with |data|.
let prob = time / this.totalGCTime;
// Note that we may replace multiple GCs in
// randomlySelected. It's easier to reason about the
// probabilities this way, and it's unlikely to have any effect in
// practice.
for (let i = 0; i < this.randomlySelected.length; i++) {
let r = Math.random();
if (r <= prob) {
this.randomlySelected[i] = data;
}
}
// Save the 5 worst GCs based on max_pause. A GC may appear in
// both worst and randomlySelected.
for (let i = 0; i < this.worst.length; i++) {
if (!this.worst[i]) {
this.worst[i] = data;
break;
}
if (this.worst[i].max_pause < data.max_pause) {
this.worst.splice(i, 0, data);
this.worst.length--;
break;
}
}
}
entries() {
return {
random: this.randomlySelected.filter(e => e !== null),
worst: this.worst.filter(e => e !== null),
};
}
}
// If you adjust any of the constants here (slice limit, number of keys, etc.)
// make sure to update the JSON schema at:
// https://github.com/mozilla-services/mozilla-pipeline-schemas/blob/master/telemetry/main.schema.json
// You should also adjust browser_TelemetryGC.js.
const MAX_GC_KEYS = 25;
const MAX_SLICES = 4;
const MAX_SLICE_KEYS = 15;
const MAX_PHASES = 65;
function limitProperties(obj, count) {
// If there are too many properties, just delete them all. We don't
// expect this ever to happen.
if (Object.keys(obj).length > count) {
for (let key of Object.keys(obj)) {
delete obj[key];
}
}
}
function limitSize(data) {
// Store the number of slices so we know if we lost any at the end.
data.num_slices = data.slices.length;
data.slices.sort((a, b) => b.pause - a.pause);
if (data.slices.length > MAX_SLICES) {
// Make sure we always keep the first slice since it has the
// reason the GC was started.
let firstSliceIndex = data.slices.findIndex(s => s.slice == 0);
if (firstSliceIndex >= MAX_SLICES) {
data.slices[MAX_SLICES - 1] = data.slices[firstSliceIndex];
}
data.slices.length = MAX_SLICES;
}
data.slices.sort((a, b) => a.slice - b.slice);
limitProperties(data, MAX_GC_KEYS);
for (let slice of data.slices) {
limitProperties(slice, MAX_SLICE_KEYS);
limitProperties(slice.times, MAX_PHASES);
}
limitProperties(data.totals, MAX_PHASES);
}
let processData = new Map();
for (let name of PROCESS_NAMES) {
processData.set(name, new GCData(name));
}
var GCTelemetry = {
initialized: false,
init() {
if (this.initialized) {
return false;
}
this.initialized = true;
Services.obs.addObserver(this, "garbage-collection-statistics", false);
if (Services.appinfo.processType == Services.appinfo.PROCESS_TYPE_DEFAULT) {
Services.ppmm.addMessageListener("Telemetry:GCStatistics", this);
}
return true;
},
shutdown() {
if (!this.initialized) {
return;
}
Services.obs.removeObserver(this, "garbage-collection-statistics");
if (Services.appinfo.processType == Services.appinfo.PROCESS_TYPE_DEFAULT) {
Services.ppmm.removeMessageListener("Telemetry:GCStatistics", this);
}
this.initialized = false;
},
observe(subject, topic, arg) {
let data = JSON.parse(arg);
limitSize(data);
if (Services.appinfo.processType == Services.appinfo.PROCESS_TYPE_DEFAULT) {
processData.get("main").record(data);
} else {
Services.cpmm.sendAsyncMessage("Telemetry:GCStatistics", data);
}
},
receiveMessage(msg) {
processData.get("content").record(msg.data);
},
entries(kind, clear) {
let result = processData.get(kind).entries();
if (clear) {
processData.set(kind, new GCData(kind));
}
return result;
},
};

10817
toolkit/components/telemetry/Histograms.json
View file

File diff suppressed because it is too large Load diff

63
toolkit/components/telemetry/ProcessedStack.h
View file

@ -1,63 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef ProcessedStack_h__
#define ProcessedStack_h__
#include <string>
#include <vector>
namespace mozilla {
namespace Telemetry {
// This class represents a stack trace and the modules referenced in that trace.
// It is designed to be easy to read and write to disk or network and doesn't
// include any logic on how to collect or read the information it stores.
class ProcessedStack
{
public:
ProcessedStack();
size_t GetStackSize() const;
size_t GetNumModules() const;
struct Frame
{
// The offset of this program counter in its module or an absolute pc.
uintptr_t mOffset;
// The index to pass to GetModule to get the module this program counter
// was in.
uint16_t mModIndex;
};
struct Module
{
// The file name, /foo/bar/libxul.so for example.
std::string mName;
std::string mBreakpadId;
bool operator==(const Module& other) const;
};
const Frame &GetFrame(unsigned aIndex) const;
void AddFrame(const Frame& aFrame);
const Module &GetModule(unsigned aIndex) const;
void AddModule(const Module& aFrame);
void Clear();
private:
std::vector<Module> mModules;
std::vector<Frame> mStack;
};
// Get the current list of loaded modules, filter and pair it to the provided
// stack. We let the caller collect the stack since different callers have
// different needs (current thread X main thread, stopping the thread, etc).
ProcessedStack
GetStackAndModules(const std::vector<uintptr_t> &aPCs);
} // namespace Telemetry
} // namespace mozilla
#endif // ProcessedStack_h__

27
toolkit/components/telemetry/ScalarInfo.h
View file

@ -1,27 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TelemetryScalarInfo_h__
#define TelemetryScalarInfo_h__
// This module is internal to Telemetry. It defines a structure that holds the
// scalar info. It should only be used by TelemetryScalarData.h automatically
// generated file and TelemetryScalar.cpp. This should not be used anywhere else.
// For the public interface to Telemetry functionality, see Telemetry.h.
namespace {
struct ScalarInfo {
uint32_t kind;
uint32_t name_offset;
uint32_t expiration_offset;
uint32_t dataset;
bool keyed;
const char *name() const;
const char *expiration() const;
};
} // namespace
#endif // TelemetryScalarInfo_h__

298
toolkit/components/telemetry/Scalars.yaml
View file

@ -1,298 +0,0 @@
# This file contains a definition of the scalar probes that are recorded in Telemetry.
# They are submitted with the "main" pings and can be inspected in about:telemetry.
# The following section contains the aushelper system add-on scalars.
aushelper:
websense_reg_version:
bug_numbers:
- 1305847
description: The Websense version from the Windows registry.
expires: "60"
kind: string
notification_emails:
- application-update-telemetry-alerts@mozilla.com
release_channel_collection: opt-out
# The following section contains the browser engagement scalars.
browser.engagement:
max_concurrent_tab_count:
bug_numbers:
- 1271304
description: >
The count of maximum number of tabs open during a subsession,
across all windows, including tabs in private windows and restored
at startup.
expires: "55"
kind: uint
notification_emails:
- rweiss@mozilla.com
release_channel_collection: opt-out
tab_open_event_count:
bug_numbers:
- 1271304
description: >
The count of tab open events per subsession, across all windows, after the
session has been restored. This includes tab open events from private windows
and from manual session restorations (i.e. after crashes and from about:home).
expires: "55"
kind: uint
notification_emails:
- rweiss@mozilla.com
release_channel_collection: opt-out
max_concurrent_window_count:
bug_numbers:
- 1271304
description: >
The count of maximum number of browser windows open during a subsession. This
includes private windows and the ones opened when starting the browser.
expires: "55"
kind: uint
notification_emails:
- rweiss@mozilla.com
release_channel_collection: opt-out
window_open_event_count:
bug_numbers:
- 1271304
description: >
The count of browser window open events per subsession, after the session
has been restored. The count includes private windows and the ones from manual
session restorations (i.e. after crashes and from about:home).
expires: "55"
kind: uint
notification_emails:
- rweiss@mozilla.com
release_channel_collection: opt-out
total_uri_count:
bug_numbers:
- 1271313
description: >
The count of the total non-unique http(s) URIs visited in a subsession, including
page reloads, after the session has been restored. This does not include background
page requests and URIs from embedded pages or private browsing.
expires: "55"
kind: uint
notification_emails:
- rweiss@mozilla.com
release_channel_collection: opt-out
unfiltered_uri_count:
bug_numbers:
- 1304647
description: >
The count of the total non-unique URIs visited in a subsession, not restricted to
a specific protocol, including page reloads and about:* pages (other than initial
pages such as about:blank, ...), after the session has been restored. This does
not include background page requests and URIs from embedded pages or private browsing.
expires: "55"
kind: uint
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
unique_domains_count:
bug_numbers:
- 1271310
description: >
The count of the unique domains visited in a subsession, after the session
has been restored. Subdomains under eTLD are aggregated after the first level
(i.e. test.example.com and other.example.com are only counted once).
This does not include background page requests and domains from embedded pages
or private browsing. The count is limited to 100 unique domains.
expires: "55"
kind: uint
notification_emails:
- rweiss@mozilla.com
release_channel_collection: opt-out
# The following section contains the browser engagement scalars.
browser.engagement.navigation:
urlbar:
bug_numbers:
- 1271313
description: >
The count URI loads triggered in a subsession from the urlbar (awesomebar),
broken down by the originating action.
expires: "55"
kind: uint
keyed: true
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
searchbar:
bug_numbers:
- 1271313
description: >
The count URI loads triggered in a subsession from the searchbar,
broken down by the originating action.
expires: "55"
kind: uint
keyed: true
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
about_home:
bug_numbers:
- 1271313
description: >
The count URI loads triggered in a subsession from about:home,
broken down by the originating action.
expires: "55"
kind: uint
keyed: true
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
about_newtab:
bug_numbers:
- 1271313
description: >
The count URI loads triggered in a subsession from about:newtab,
broken down by the originating action.
expires: "55"
kind: uint
keyed: true
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
contextmenu:
bug_numbers:
- 1271313
description: >
The count URI loads triggered in a subsession from the contextmenu,
broken down by the originating action.
expires: "55"
kind: uint
keyed: true
notification_emails:
- bcolloran@mozilla.com
release_channel_collection: opt-out
# The following section is for probes testing the Telemetry system. They will not be
# submitted in pings and are only used for testing.
telemetry.test:
unsigned_int_kind:
bug_numbers:
- 1276190
description: >
This is a test uint type with a really long description, maybe spanning even multiple
lines, to just prove a point: everything works just fine.
expires: never
kind: uint
notification_emails:
- telemetry-client-dev@mozilla.com
string_kind:
bug_numbers:
- 1276190
description: A string test type with a one line comment that works just fine!
expires: never
kind: string
notification_emails:
- telemetry-client-dev@mozilla.com
boolean_kind:
bug_numbers:
- 1281214
description: A boolean test type with a one line comment that works just fine!
expires: never
kind: boolean
notification_emails:
- telemetry-client-dev@mozilla.com
expired:
bug_numbers:
- 1276190
description: This is an expired testing scalar; not meant to be touched.
expires: 4.0a1
kind: uint
notification_emails:
- telemetry-client-dev@mozilla.com
unexpired:
bug_numbers:
- 1276190
description: This is an unexpired testing scalar; not meant to be touched.
expires: "375.0"
kind: uint
notification_emails:
- telemetry-client-dev@mozilla.com
release_optin:
bug_numbers:
- 1276190
description: A testing scalar; not meant to be touched.
expires: never
kind: uint
notification_emails:
- telemetry-client-dev@mozilla.com
release_channel_collection: opt-in
release_optout:
bug_numbers:
- 1276190
description: A testing scalar; not meant to be touched.
expires: never
kind: uint
notification_emails:
- telemetry-client-dev@mozilla.com
release_channel_collection: opt-out
keyed_release_optin:
bug_numbers:
- 1277806
description: A testing scalar; not meant to be touched.
expires: never
kind: uint
keyed: true
notification_emails:
- telemetry-client-dev@mozilla.com
release_channel_collection: opt-in
keyed_release_optout:
bug_numbers:
- 1277806
description: A testing scalar; not meant to be touched.
expires: never
kind: uint
keyed: true
notification_emails:
- telemetry-client-dev@mozilla.com
release_channel_collection: opt-out
keyed_expired:
bug_numbers:
- 1277806
description: This is an expired testing scalar; not meant to be touched.
expires: 4.0a1
kind: uint
keyed: true
notification_emails:
- telemetry-client-dev@mozilla.com
keyed_unsigned_int:
bug_numbers:
- 1277806
description: A testing keyed uint scalar; not meant to be touched.
expires: never
kind: uint
keyed: true
notification_emails:
- telemetry-client-dev@mozilla.com
keyed_boolean_kind:
bug_numbers:
- 1277806
description: A testing keyed boolean scalar; not meant to be touched.
expires: never
kind: boolean
keyed: true
notification_emails:
- telemetry-client-dev@mozilla.com

2935
toolkit/components/telemetry/Telemetry.cpp
View file

File diff suppressed because it is too large Load diff

417
toolkit/components/telemetry/Telemetry.h
View file

@ -1,417 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef Telemetry_h__
#define Telemetry_h__
#include "mozilla/GuardObjects.h"
#include "mozilla/TimeStamp.h"
#include "mozilla/StartupTimeline.h"
#include "nsTArray.h"
#include "nsStringGlue.h"
#include "nsXULAppAPI.h"
#include "mozilla/TelemetryHistogramEnums.h"
#include "mozilla/TelemetryScalarEnums.h"
/******************************************************************************
* This implements the Telemetry system.
* It allows recording into histograms as well some more specialized data
* points and gives access to the data.
*
* For documentation on how to add and use new Telemetry probes, see:
* https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Adding_a_new_Telemetry_probe
*
* For more general information on Telemetry see:
* https://wiki.mozilla.org/Telemetry
*****************************************************************************/
namespace mozilla {
namespace HangMonitor {
class HangAnnotations;
} // namespace HangMonitor
namespace Telemetry {
struct Accumulation;
struct KeyedAccumulation;
enum TimerResolution {
Millisecond,
Microsecond
};
/**
* Create and destroy the underlying base::StatisticsRecorder singleton.
* Creation has to be done very early in the startup sequence.
*/
void CreateStatisticsRecorder();
void DestroyStatisticsRecorder();
/**
* Initialize the Telemetry service on the main thread at startup.
*/
void Init();
/**
* Adds sample to a histogram defined in TelemetryHistogramEnums.h
*
* @param id - histogram id
* @param sample - value to record.
*/
void Accumulate(ID id, uint32_t sample);
/**
* Adds sample to a keyed histogram defined in TelemetryHistogramEnums.h
*
* @param id - keyed histogram id
* @param key - the string key
* @param sample - (optional) value to record, defaults to 1.
*/
void Accumulate(ID id, const nsCString& key, uint32_t sample = 1);
/**
* Adds a sample to a histogram defined in TelemetryHistogramEnums.h.
* This function is here to support telemetry measurements from Java,
* where we have only names and not numeric IDs. You should almost
* certainly be using the by-enum-id version instead of this one.
*
* @param name - histogram name
* @param sample - value to record
*/
void Accumulate(const char* name, uint32_t sample);
/**
* Adds a sample to a histogram defined in TelemetryHistogramEnums.h.
* This function is here to support telemetry measurements from Java,
* where we have only names and not numeric IDs. You should almost
* certainly be using the by-enum-id version instead of this one.
*
* @param name - histogram name
* @param key - the string key
* @param sample - sample - (optional) value to record, defaults to 1.
*/
void Accumulate(const char *name, const nsCString& key, uint32_t sample = 1);
/**
* Adds sample to a categorical histogram defined in TelemetryHistogramEnums.h
* This is the typesafe - and preferred - way to use the categorical histograms
* by passing values from the corresponding Telemetry::LABELS_* enum.
*
* @param enumValue - Label value from one of the Telemetry::LABELS_* enums.
*/
template<class E>
void AccumulateCategorical(E enumValue) {
static_assert(IsCategoricalLabelEnum<E>::value,
"Only categorical label enum types are supported.");
Accumulate(static_cast<ID>(CategoricalLabelId<E>::value),
static_cast<uint32_t>(enumValue));
};
/**
* Adds sample to a categorical histogram defined in TelemetryHistogramEnums.h
* This string will be matched against the labels defined in Histograms.json.
* If the string does not match a label defined for the histogram, nothing will
* be recorded.
*
* @param id - The histogram id.
* @param label - A string label value that is defined in Histograms.json for this histogram.
*/
void AccumulateCategorical(ID id, const nsCString& label);
/**
* Adds time delta in milliseconds to a histogram defined in TelemetryHistogramEnums.h
*
* @param id - histogram id
* @param start - start time
* @param end - end time
*/
void AccumulateTimeDelta(ID id, TimeStamp start, TimeStamp end = TimeStamp::Now());
/**
* Accumulate child process data into histograms for the given process type.
*
* @param aAccumulations - accumulation actions to perform
*/
void AccumulateChild(GeckoProcessType aProcessType, const nsTArray<Accumulation>& aAccumulations);
/**
* Accumulate child process data into keyed histograms for the given process type.
*
* @param aAccumulations - accumulation actions to perform
*/
void AccumulateChildKeyed(GeckoProcessType aProcessType, const nsTArray<KeyedAccumulation>& aAccumulations);
/**
* Enable/disable recording for this histogram at runtime.
* Recording is enabled by default, unless listed at kRecordingInitiallyDisabledIDs[].
* id must be a valid telemetry enum, otherwise an assertion is triggered.
*
* @param id - histogram id
* @param enabled - whether or not to enable recording from now on.
*/
void SetHistogramRecordingEnabled(ID id, bool enabled);
const char* GetHistogramName(ID id);
/**
* Those wrappers are needed because the VS versions we use do not support free
* functions with default template arguments.
*/
template<TimerResolution res>
struct AccumulateDelta_impl
{
static void compute(ID id, TimeStamp start, TimeStamp end = TimeStamp::Now());
static void compute(ID id, const nsCString& key, TimeStamp start, TimeStamp end = TimeStamp::Now());
};
template<>
struct AccumulateDelta_impl<Millisecond>
{
static void compute(ID id, TimeStamp start, TimeStamp end = TimeStamp::Now()) {
Accumulate(id, static_cast<uint32_t>((end - start).ToMilliseconds()));
}
static void compute(ID id, const nsCString& key, TimeStamp start, TimeStamp end = TimeStamp::Now()) {
Accumulate(id, key, static_cast<uint32_t>((end - start).ToMilliseconds()));
}
};
template<>
struct AccumulateDelta_impl<Microsecond>
{
static void compute(ID id, TimeStamp start, TimeStamp end = TimeStamp::Now()) {
Accumulate(id, static_cast<uint32_t>((end - start).ToMicroseconds()));
}
static void compute(ID id, const nsCString& key, TimeStamp start, TimeStamp end = TimeStamp::Now()) {
Accumulate(id, key, static_cast<uint32_t>((end - start).ToMicroseconds()));
}
};
template<ID id, TimerResolution res = Millisecond>
class MOZ_RAII AutoTimer {
public:
explicit AutoTimer(TimeStamp aStart = TimeStamp::Now() MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
: start(aStart)
{
MOZ_GUARD_OBJECT_NOTIFIER_INIT;
}
explicit AutoTimer(const nsCString& aKey, TimeStamp aStart = TimeStamp::Now() MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
: start(aStart)
, key(aKey)
{
MOZ_GUARD_OBJECT_NOTIFIER_INIT;
}
~AutoTimer() {
if (key.IsEmpty()) {
AccumulateDelta_impl<res>::compute(id, start);
} else {
AccumulateDelta_impl<res>::compute(id, key, start);
}
}
private:
const TimeStamp start;
const nsCString key;
MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER
};
template<ID id>
class MOZ_RAII AutoCounter {
public:
explicit AutoCounter(uint32_t counterStart = 0 MOZ_GUARD_OBJECT_NOTIFIER_PARAM)
: counter(counterStart)
{
MOZ_GUARD_OBJECT_NOTIFIER_INIT;
}
~AutoCounter() {
Accumulate(id, counter);
}
// Prefix increment only, to encourage good habits.
void operator++() {
++counter;
}
// Chaining doesn't make any sense, don't return anything.
void operator+=(int increment) {
counter += increment;
}
private:
uint32_t counter;
MOZ_DECL_USE_GUARD_OBJECT_NOTIFIER
};
/**
* Indicates whether Telemetry base data recording is turned on. Added for future uses.
*/
bool CanRecordBase();
/**
* Indicates whether Telemetry extended data recording is turned on. This is intended
* to guard calls to Accumulate when the statistic being recorded is expensive to compute.
*/
bool CanRecordExtended();
/**
* Records slow SQL statements for Telemetry reporting.
*
* @param statement - offending SQL statement to record
* @param dbName - DB filename
* @param delay - execution time in milliseconds
*/
void RecordSlowSQLStatement(const nsACString &statement,
const nsACString &dbName,
uint32_t delay);
/**
* Record Webrtc ICE candidate type combinations in a 17bit bitmask
*
* @param iceCandidateBitmask - the bitmask representing local and remote ICE
* candidate types present for the connection
* @param success - did the peer connection connected
*/
void
RecordWebrtcIceCandidates(const uint32_t iceCandidateBitmask,
const bool success);
/**
* Initialize I/O Reporting
* Initially this only records I/O for files in the binary directory.
*
* @param aXreDir - XRE directory
*/
void InitIOReporting(nsIFile* aXreDir);
/**
* Set the profile directory. Once called, files in the profile directory will
* be included in I/O reporting. We can't use the directory
* service to obtain this information because it isn't running yet.
*/
void SetProfileDir(nsIFile* aProfD);
/**
* Called to inform Telemetry that startup has completed.
*/
void LeavingStartupStage();
/**
* Called to inform Telemetry that shutdown is commencing.
*/
void EnteringShutdownStage();
/**
* Thresholds for a statement to be considered slow, in milliseconds
*/
const uint32_t kSlowSQLThresholdForMainThread = 50;
const uint32_t kSlowSQLThresholdForHelperThreads = 100;
class ProcessedStack;
class ThreadHangStats;
/**
* Move a ThreadHangStats to Telemetry storage. Normally Telemetry queries
* for active ThreadHangStats through BackgroundHangMonitor, but once a
* thread exits, the thread's copy of ThreadHangStats needs to be moved to
* inside Telemetry using this function.
*
* @param aStats ThreadHangStats to save; the data inside aStats
* will be moved and aStats should be treated as
* invalid after this function returns
*/
void RecordThreadHangStats(ThreadHangStats& aStats);
/**
* Record a failed attempt at locking the user's profile.
*
* @param aProfileDir The profile directory whose lock attempt failed
*/
void WriteFailedProfileLock(nsIFile* aProfileDir);
/**
* Adds the value to the given scalar.
*
* @param aId The scalar enum id.
* @param aValue The value to add to the scalar.
*/
void ScalarAdd(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
/**
* Sets the scalar to the given value.
*
* @param aId The scalar enum id.
* @param aValue The value to set the scalar to.
*/
void ScalarSet(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
/**
* Sets the scalar to the given value.
*
* @param aId The scalar enum id.
* @param aValue The value to set the scalar to.
*/
void ScalarSet(mozilla::Telemetry::ScalarID aId, bool aValue);
/**
* Sets the scalar to the given value.
*
* @param aId The scalar enum id.
* @param aValue The value to set the scalar to, truncated to
* 50 characters if exceeding that length.
*/
void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aValue);
/**
* Sets the scalar to the maximum of the current and the passed value.
*
* @param aId The scalar enum id.
* @param aValue The value the scalar is set to if its greater
* than the current value.
*/
void ScalarSetMaximum(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
/**
* Adds the value to the given scalar.
*
* @param aId The scalar enum id.
* @param aKey The scalar key.
* @param aValue The value to add to the scalar.
*/
void ScalarAdd(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
/**
* Sets the scalar to the given value.
*
* @param aId The scalar enum id.
* @param aKey The scalar key.
* @param aValue The value to set the scalar to.
*/
void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
/**
* Sets the scalar to the given value.
*
* @param aId The scalar enum id.
* @param aKey The scalar key.
* @param aValue The value to set the scalar to.
*/
void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, bool aValue);
/**
* Sets the scalar to the maximum of the current and the passed value.
*
* @param aId The scalar enum id.
* @param aKey The scalar key.
* @param aValue The value the scalar is set to if its greater
* than the current value.
*/
void ScalarSetMaximum(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
} // namespace Telemetry
} // namespace mozilla
#endif // Telemetry_h__

125
toolkit/components/telemetry/TelemetryArchive.jsm
View file

@ -1,125 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = [
"TelemetryArchive"
];
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/Preferences.jsm", this);
Cu.import("resource://gre/modules/Task.jsm", this);
Cu.import("resource://gre/modules/osfile.jsm", this);
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryArchive::";
const PREF_BRANCH = "toolkit.telemetry.";
const PREF_ARCHIVE_ENABLED = PREF_BRANCH + "archive.enabled";
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryStorage",
"resource://gre/modules/TelemetryStorage.jsm");
this.TelemetryArchive = {
/**
* Get a list of the archived pings, sorted by the creation date.
* Note that scanning the archived pings on disk is delayed on startup,
* use promizeInitialized() to access this after scanning.
*
* @return {Promise<sequence<Object>>}
* A list of the archived ping info in the form:
* { id: <string>,
* timestampCreated: <number>,
* type: <string> }
*/
promiseArchivedPingList: function() {
return TelemetryArchiveImpl.promiseArchivedPingList();
},
/**
* Load an archived ping from disk by id, asynchronously.
*
* @param id {String} The pings UUID.
* @return {Promise<PingData>} A promise resolved with the pings data on success.
*/
promiseArchivedPingById: function(id) {
return TelemetryArchiveImpl.promiseArchivedPingById(id);
},
/**
* Archive a ping and persist it to disk.
*
* @param {object} ping The ping data to archive.
* @return {promise} Promise that is resolved when the ping is successfully archived.
*/
promiseArchivePing: function(ping) {
return TelemetryArchiveImpl.promiseArchivePing(ping);
},
};
/**
* Checks if pings can be archived. Some products (e.g. Thunderbird) might not want
* to do that.
* @return {Boolean} True if pings should be archived, false otherwise.
*/
function shouldArchivePings() {
return Preferences.get(PREF_ARCHIVE_ENABLED, false);
}
var TelemetryArchiveImpl = {
_logger: null,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
promiseArchivePing: function(ping) {
if (!shouldArchivePings()) {
this._log.trace("promiseArchivePing - archiving is disabled");
return Promise.resolve();
}
for (let field of ["creationDate", "id", "type"]) {
if (!(field in ping)) {
this._log.warn("promiseArchivePing - missing field " + field)
return Promise.reject(new Error("missing field " + field));
}
}
return TelemetryStorage.saveArchivedPing(ping);
},
_buildArchivedPingList: function(archivedPingsMap) {
let list = Array.from(archivedPingsMap, p => ({
id: p[0],
timestampCreated: p[1].timestampCreated,
type: p[1].type,
}));
list.sort((a, b) => a.timestampCreated - b.timestampCreated);
return list;
},
promiseArchivedPingList: function() {
this._log.trace("promiseArchivedPingList");
return TelemetryStorage.loadArchivedPingList().then(loadedInfo => {
return this._buildArchivedPingList(loadedInfo);
});
},
promiseArchivedPingById: function(id) {
this._log.trace("promiseArchivedPingById - id: " + id);
return TelemetryStorage.loadArchivedPing(id);
},
};

104
toolkit/components/telemetry/TelemetryCommon.cpp
View file

@ -1,104 +0,0 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsITelemetry.h"
#include "nsVersionComparator.h"
#include "mozilla/TimeStamp.h"
#include "nsIConsoleService.h"
#include "nsThreadUtils.h"
#include "TelemetryCommon.h"
#include <cstring>
namespace mozilla {
namespace Telemetry {
namespace Common {
bool
IsExpiredVersion(const char* aExpiration)
{
MOZ_ASSERT(aExpiration);
// Note: We intentionally don't construct a static Version object here as we
// saw odd crashes around this (see bug 1334105).
return strcmp(aExpiration, "never") && strcmp(aExpiration, "default") &&
(mozilla::Version(aExpiration) <= MOZ_APP_VERSION);
}
bool
IsInDataset(uint32_t aDataset, uint32_t aContainingDataset)
{
if (aDataset == aContainingDataset) {
return true;
}
// The "optin on release channel" dataset is a superset of the
// "optout on release channel one".
if (aContainingDataset == nsITelemetry::DATASET_RELEASE_CHANNEL_OPTIN &&
aDataset == nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT) {
return true;
}
return false;
}
bool
CanRecordDataset(uint32_t aDataset, bool aCanRecordBase, bool aCanRecordExtended)
{
// If we are extended telemetry is enabled, we are allowed to record
// regardless of the dataset.
if (aCanRecordExtended) {
return true;
}
// If base telemetry data is enabled and we're trying to record base
// telemetry, allow it.
if (aCanRecordBase &&
IsInDataset(aDataset, nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT)) {
return true;
}
// We're not recording extended telemetry or this is not the base
// dataset. Bail out.
return false;
}
nsresult
MsSinceProcessStart(double* aResult)
{
bool error;
*aResult = (TimeStamp::NowLoRes() -
TimeStamp::ProcessCreation(error)).ToMilliseconds();
if (error) {
return NS_ERROR_NOT_AVAILABLE;
}
return NS_OK;
}
void
LogToBrowserConsole(uint32_t aLogLevel, const nsAString& aMsg)
{
if (!NS_IsMainThread()) {
nsString msg(aMsg);
nsCOMPtr<nsIRunnable> task =
NS_NewRunnableFunction([aLogLevel, msg]() { LogToBrowserConsole(aLogLevel, msg); });
NS_DispatchToMainThread(task.forget(), NS_DISPATCH_NORMAL);
return;
}
nsCOMPtr<nsIConsoleService> console(do_GetService("@mozilla.org/consoleservice;1"));
if (!console) {
NS_WARNING("Failed to log message to console.");
return;
}
nsCOMPtr<nsIScriptError> error(do_CreateInstance(NS_SCRIPTERROR_CONTRACTID));
error->Init(aMsg, EmptyString(), EmptyString(), 0, 0, aLogLevel, "chrome javascript");
console->LogMessage(error);
}
} // namespace Common
} // namespace Telemetry
} // namespace mozilla

75
toolkit/components/telemetry/TelemetryCommon.h
View file

@ -1,75 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TelemetryCommon_h__
#define TelemetryCommon_h__
#include "nsTHashtable.h"
#include "jsapi.h"
#include "nsIScriptError.h"
namespace mozilla {
namespace Telemetry {
namespace Common {
template<class EntryType>
class AutoHashtable : public nsTHashtable<EntryType>
{
public:
explicit AutoHashtable(uint32_t initLength =
PLDHashTable::kDefaultInitialLength);
typedef bool (*ReflectEntryFunc)(EntryType *entry, JSContext *cx, JS::Handle<JSObject*> obj);
bool ReflectIntoJS(ReflectEntryFunc entryFunc, JSContext *cx, JS::Handle<JSObject*> obj);
};
template<class EntryType>
AutoHashtable<EntryType>::AutoHashtable(uint32_t initLength)
: nsTHashtable<EntryType>(initLength)
{
}
/**
* Reflect the individual entries of table into JS, usually by defining
* some property and value of obj. entryFunc is called for each entry.
*/
template<typename EntryType>
bool
AutoHashtable<EntryType>::ReflectIntoJS(ReflectEntryFunc entryFunc,
JSContext *cx, JS::Handle<JSObject*> obj)
{
for (auto iter = this->Iter(); !iter.Done(); iter.Next()) {
if (!entryFunc(iter.Get(), cx, obj)) {
return false;
}
}
return true;
}
bool IsExpiredVersion(const char* aExpiration);
bool IsInDataset(uint32_t aDataset, uint32_t aContainingDataset);
bool CanRecordDataset(uint32_t aDataset, bool aCanRecordBase, bool aCanRecordExtended);
/**
* Return the number of milliseconds since process start using monotonic
* timestamps (unaffected by system clock changes).
*
* @return NS_OK on success, NS_ERROR_NOT_AVAILABLE if TimeStamp doesn't have the data.
*/
nsresult MsSinceProcessStart(double* aResult);
/**
* Dumps a log message to the Browser Console using the provided level.
*
* @param aLogLevel The level to use when displaying the message in the browser console
* (e.g. nsIScriptError::warningFlag, ...).
* @param aMsg The text message to print to the console.
*/
void LogToBrowserConsole(uint32_t aLogLevel, const nsAString& aMsg);
} // namespace Common
} // namespace Telemetry
} // namespace mozilla
#endif // TelemetryCommon_h__

84
toolkit/components/telemetry/TelemetryComms.h
View file

@ -1,84 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
*/
#ifndef Telemetry_Comms_h__
#define Telemetry_Comms_h__
#include "ipc/IPCMessageUtils.h"
namespace mozilla {
namespace Telemetry {
enum ID : uint32_t;
struct Accumulation
{
mozilla::Telemetry::ID mId;
uint32_t mSample;
};
struct KeyedAccumulation
{
mozilla::Telemetry::ID mId;
uint32_t mSample;
nsCString mKey;
};
} // namespace Telemetry
} // namespace mozilla
namespace IPC {
template<>
struct
ParamTraits<mozilla::Telemetry::Accumulation>
{
typedef mozilla::Telemetry::Accumulation paramType;
static void Write(Message* aMsg, const paramType& aParam)
{
aMsg->WriteUInt32(aParam.mId);
WriteParam(aMsg, aParam.mSample);
}
static bool Read(const Message* aMsg, PickleIterator* aIter, paramType* aResult)
{
if (!aMsg->ReadUInt32(aIter, reinterpret_cast<uint32_t*>(&(aResult->mId))) ||
!ReadParam(aMsg, aIter, &(aResult->mSample))) {
return false;
}
return true;
}
};
template<>
struct
ParamTraits<mozilla::Telemetry::KeyedAccumulation>
{
typedef mozilla::Telemetry::KeyedAccumulation paramType;
static void Write(Message* aMsg, const paramType& aParam)
{
aMsg->WriteUInt32(aParam.mId);
WriteParam(aMsg, aParam.mSample);
WriteParam(aMsg, aParam.mKey);
}
static bool Read(const Message* aMsg, PickleIterator* aIter, paramType* aResult)
{
if (!aMsg->ReadUInt32(aIter, reinterpret_cast<uint32_t*>(&(aResult->mId))) ||
!ReadParam(aMsg, aIter, &(aResult->mSample)) ||
!ReadParam(aMsg, aIter, &(aResult->mKey))) {
return false;
}
return true;
}
};
} // namespace IPC
#endif // Telemetry_Comms_h__

954
toolkit/components/telemetry/TelemetryController.jsm
View file

@ -1,954 +0,0 @@
/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;
const myScope = this;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://gre/modules/debug.js", this);
Cu.import("resource://gre/modules/Services.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/osfile.jsm", this);
Cu.import("resource://gre/modules/Promise.jsm", this);
Cu.import("resource://gre/modules/PromiseUtils.jsm", this);
Cu.import("resource://gre/modules/Task.jsm", this);
Cu.import("resource://gre/modules/DeferredTask.jsm", this);
Cu.import("resource://gre/modules/Preferences.jsm");
Cu.import("resource://gre/modules/Timer.jsm");
Cu.import("resource://gre/modules/TelemetryUtils.jsm", this);
Cu.import("resource://gre/modules/AppConstants.jsm");
const Utils = TelemetryUtils;
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryController::";
const PREF_BRANCH = "toolkit.telemetry.";
const PREF_BRANCH_LOG = PREF_BRANCH + "log.";
const PREF_SERVER = PREF_BRANCH + "server";
const PREF_LOG_LEVEL = PREF_BRANCH_LOG + "level";
const PREF_LOG_DUMP = PREF_BRANCH_LOG + "dump";
const PREF_CACHED_CLIENTID = PREF_BRANCH + "cachedClientID";
const PREF_FHR_UPLOAD_ENABLED = "datareporting.healthreport.uploadEnabled";
const PREF_SESSIONS_BRANCH = "datareporting.sessions.";
const PREF_UNIFIED = PREF_BRANCH + "unified";
// Whether the FHR/Telemetry unification features are enabled.
// Changing this pref requires a restart.
const IS_UNIFIED_TELEMETRY = Preferences.get(PREF_UNIFIED, false);
const PING_FORMAT_VERSION = 4;
// Delay before intializing telemetry (ms)
const TELEMETRY_DELAY = Preferences.get("toolkit.telemetry.initDelay", 60) * 1000;
// Delay before initializing telemetry if we're testing (ms)
const TELEMETRY_TEST_DELAY = 1;
// Ping types.
const PING_TYPE_MAIN = "main";
const PING_TYPE_DELETION = "deletion";
// Session ping reasons.
const REASON_GATHER_PAYLOAD = "gather-payload";
const REASON_GATHER_SUBSESSION_PAYLOAD = "gather-subsession-payload";
XPCOMUtils.defineLazyModuleGetter(this, "ClientID",
"resource://gre/modules/ClientID.jsm");
XPCOMUtils.defineLazyServiceGetter(this, "Telemetry",
"@mozilla.org/base/telemetry;1",
"nsITelemetry");
XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown",
"resource://gre/modules/AsyncShutdown.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryStorage",
"resource://gre/modules/TelemetryStorage.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "ThirdPartyCookieProbe",
"resource://gre/modules/ThirdPartyCookieProbe.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryEnvironment",
"resource://gre/modules/TelemetryEnvironment.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "SessionRecorder",
"resource://gre/modules/SessionRecorder.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils",
"resource://gre/modules/UpdateUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryArchive",
"resource://gre/modules/TelemetryArchive.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetrySession",
"resource://gre/modules/TelemetrySession.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetrySend",
"resource://gre/modules/TelemetrySend.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryReportingPolicy",
"resource://gre/modules/TelemetryReportingPolicy.jsm");
/**
* Setup Telemetry logging. This function also gets called when loggin related
* preferences change.
*/
var gLogger = null;
var gLogAppenderDump = null;
function configureLogging() {
if (!gLogger) {
gLogger = Log.repository.getLogger(LOGGER_NAME);
// Log messages need to go to the browser console.
let consoleAppender = new Log.ConsoleAppender(new Log.BasicFormatter());
gLogger.addAppender(consoleAppender);
Preferences.observe(PREF_BRANCH_LOG, configureLogging);
}
// Make sure the logger keeps up with the logging level preference.
gLogger.level = Log.Level[Preferences.get(PREF_LOG_LEVEL, "Warn")];
// If enabled in the preferences, add a dump appender.
let logDumping = Preferences.get(PREF_LOG_DUMP, false);
if (logDumping != !!gLogAppenderDump) {
if (logDumping) {
gLogAppenderDump = new Log.DumpAppender(new Log.BasicFormatter());
gLogger.addAppender(gLogAppenderDump);
} else {
gLogger.removeAppender(gLogAppenderDump);
gLogAppenderDump = null;
}
}
}
/**
* This is a policy object used to override behavior for testing.
*/
var Policy = {
now: () => new Date(),
generatePingId: () => Utils.generateUUID(),
getCachedClientID: () => ClientID.getCachedClientID(),
}
this.EXPORTED_SYMBOLS = ["TelemetryController"];
this.TelemetryController = Object.freeze({
Constants: Object.freeze({
PREF_LOG_LEVEL: PREF_LOG_LEVEL,
PREF_LOG_DUMP: PREF_LOG_DUMP,
PREF_SERVER: PREF_SERVER,
}),
/**
* Used only for testing purposes.
*/
testInitLogging: function() {
configureLogging();
},
/**
* Used only for testing purposes.
*/
testReset: function() {
return Impl.reset();
},
/**
* Used only for testing purposes.
*/
testSetup: function() {
return Impl.setupTelemetry(true);
},
/**
* Used only for testing purposes.
*/
testShutdown: function() {
return Impl.shutdown();
},
/**
* Used only for testing purposes.
*/
testSetupContent: function() {
return Impl.setupContentTelemetry(true);
},
/**
* Send a notification.
*/
observe: function (aSubject, aTopic, aData) {
return Impl.observe(aSubject, aTopic, aData);
},
/**
* Submit ping payloads to Telemetry. This will assemble a complete ping, adding
* environment data, client id and some general info.
* Depending on configuration, the ping will be sent to the server (immediately or later)
* and archived locally.
*
* To identify the different pings and to be able to query them pings have a type.
* A type is a string identifier that should be unique to the type ping that is being submitted,
* it should only contain alphanumeric characters and '-' for separation, i.e. satisfy:
* /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {Object} [aOptions] Options object.
* @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
* id, false otherwise.
* @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
* environment data.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
* @returns {Promise} Test-only - a promise that resolves with the ping id once the ping is stored or sent.
*/
submitExternalPing: function(aType, aPayload, aOptions = {}) {
aOptions.addClientId = aOptions.addClientId || false;
aOptions.addEnvironment = aOptions.addEnvironment || false;
return Impl.submitExternalPing(aType, aPayload, aOptions);
},
/**
* Get the current session ping data as it would be sent out or stored.
*
* @param {bool} aSubsession Whether to get subsession data. Optional, defaults to false.
* @return {object} The current ping data if Telemetry is enabled, null otherwise.
*/
getCurrentPingData: function(aSubsession = false) {
return Impl.getCurrentPingData(aSubsession);
},
/**
* Save a ping to disk.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {Object} [aOptions] Options object.
* @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
* id, false otherwise.
* @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
* environment data.
* @param {Boolean} [aOptions.overwrite=false] true overwrites a ping with the same name,
* if found.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
*
* @returns {Promise} A promise that resolves with the ping id when the ping is saved to
* disk.
*/
addPendingPing: function(aType, aPayload, aOptions = {}) {
let options = aOptions;
options.addClientId = aOptions.addClientId || false;
options.addEnvironment = aOptions.addEnvironment || false;
options.overwrite = aOptions.overwrite || false;
return Impl.addPendingPing(aType, aPayload, options);
},
/**
* Check if we have an aborted-session ping from a previous session.
* If so, submit and then remove it.
*
* @return {Promise} Promise that is resolved when the ping is saved.
*/
checkAbortedSessionPing: function() {
return Impl.checkAbortedSessionPing();
},
/**
* Save an aborted-session ping to disk without adding it to the pending pings.
*
* @param {Object} aPayload The ping payload data.
* @return {Promise} Promise that is resolved when the ping is saved.
*/
saveAbortedSessionPing: function(aPayload) {
return Impl.saveAbortedSessionPing(aPayload);
},
/**
* Remove the aborted-session ping if any exists.
*
* @return {Promise} Promise that is resolved when the ping was removed.
*/
removeAbortedSessionPing: function() {
return Impl.removeAbortedSessionPing();
},
/**
* Write a ping to a specified location on the disk. Does not add the ping to the
* pending pings.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {String} aFilePath The path to save the ping to.
* @param {Object} [aOptions] Options object.
* @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
* id, false otherwise.
* @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
* environment data.
* @param {Boolean} [aOptions.overwrite=false] true overwrites a ping with the same name,
* if found.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
*
* @returns {Promise} A promise that resolves with the ping id when the ping is saved to
* disk.
*/
savePing: function(aType, aPayload, aFilePath, aOptions = {}) {
let options = aOptions;
options.addClientId = aOptions.addClientId || false;
options.addEnvironment = aOptions.addEnvironment || false;
options.overwrite = aOptions.overwrite || false;
return Impl.savePing(aType, aPayload, aFilePath, options);
},
/**
* The session recorder instance managed by Telemetry.
* @return {Object} The active SessionRecorder instance or null if not available.
*/
getSessionRecorder: function() {
return Impl._sessionRecorder;
},
/**
* Allows waiting for TelemetryControllers delayed initialization to complete.
* The returned promise is guaranteed to resolve before TelemetryController is shutting down.
* @return {Promise} Resolved when delayed TelemetryController initialization completed.
*/
promiseInitialized: function() {
return Impl.promiseInitialized();
},
});
var Impl = {
_initialized: false,
_initStarted: false, // Whether we started setting up TelemetryController.
_logger: null,
_prevValues: {},
// The previous build ID, if this is the first run with a new build.
// Undefined if this is not the first run, or the previous build ID is unknown.
_previousBuildID: undefined,
_clientID: null,
// A task performing delayed initialization
_delayedInitTask: null,
// The deferred promise resolved when the initialization task completes.
_delayedInitTaskDeferred: null,
// The session recorder, shared with FHR and the Data Reporting Service.
_sessionRecorder: null,
// This is a public barrier Telemetry clients can use to add blockers to the shutdown
// of TelemetryController.
// After this barrier, clients can not submit Telemetry pings anymore.
_shutdownBarrier: new AsyncShutdown.Barrier("TelemetryController: Waiting for clients."),
// This is a private barrier blocked by pending async ping activity (sending & saving).
_connectionsBarrier: new AsyncShutdown.Barrier("TelemetryController: Waiting for pending ping activity"),
// This is true when running in the test infrastructure.
_testMode: false,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
/**
* Get the data for the "application" section of the ping.
*/
_getApplicationSection: function() {
// Querying architecture and update channel can throw. Make sure to recover and null
// those fields.
let arch = null;
try {
arch = Services.sysinfo.get("arch");
} catch (e) {
this._log.trace("_getApplicationSection - Unable to get system architecture.", e);
}
let updateChannel = null;
try {
updateChannel = UpdateUtils.UpdateChannel;
} catch (e) {
this._log.trace("_getApplicationSection - Unable to get update channel.", e);
}
return {
architecture: arch,
buildId: Services.appinfo.appBuildID,
name: Services.appinfo.name,
version: Services.appinfo.version,
displayVersion: AppConstants.MOZ_APP_VERSION_DISPLAY,
vendor: Services.appinfo.vendor,
platformVersion: Services.appinfo.platformVersion,
xpcomAbi: Services.appinfo.XPCOMABI,
channel: updateChannel,
};
},
/**
* Assemble a complete ping following the common ping format specification.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {Object} aOptions Options object.
* @param {Boolean} aOptions.addClientId true if the ping should contain the client
* id, false otherwise.
* @param {Boolean} aOptions.addEnvironment true if the ping should contain the
* environment data.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
*
* @returns {Object} An object that contains the assembled ping data.
*/
assemblePing: function assemblePing(aType, aPayload, aOptions = {}) {
this._log.trace("assemblePing - Type " + aType + ", aOptions " + JSON.stringify(aOptions));
// Clone the payload data so we don't race against unexpected changes in subobjects that are
// still referenced by other code.
// We can't trust all callers to do this properly on their own.
let payload = Cu.cloneInto(aPayload, myScope);
// Fill the common ping fields.
let pingData = {
type: aType,
id: Policy.generatePingId(),
creationDate: (Policy.now()).toISOString(),
version: PING_FORMAT_VERSION,
application: this._getApplicationSection(),
payload: payload,
};
if (aOptions.addClientId) {
pingData.clientId = this._clientID;
}
if (aOptions.addEnvironment) {
pingData.environment = aOptions.overrideEnvironment || TelemetryEnvironment.currentEnvironment;
}
return pingData;
},
/**
* Track any pending ping send and save tasks through the promise passed here.
* This is needed to block shutdown on any outstanding ping activity.
*/
_trackPendingPingTask: function (aPromise) {
this._connectionsBarrier.client.addBlocker("Waiting for ping task", aPromise);
},
/**
* Internal function to assemble a complete ping, adding environment data, client id
* and some general info. This waits on the client id to be loaded/generated if it's
* not yet available. Note that this function is synchronous unless we need to load
* the client id.
* Depending on configuration, the ping will be sent to the server (immediately or later)
* and archived locally.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {Object} [aOptions] Options object.
* @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
* id, false otherwise.
* @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
* environment data.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
* @returns {Promise} Test-only - a promise that is resolved with the ping id once the ping is stored or sent.
*/
_submitPingLogic: Task.async(function* (aType, aPayload, aOptions) {
// Make sure to have a clientId if we need one. This cover the case of submitting
// a ping early during startup, before Telemetry is initialized, if no client id was
// cached.
if (!this._clientID && aOptions.addClientId) {
Telemetry.getHistogramById("TELEMETRY_PING_SUBMISSION_WAITING_CLIENTID").add();
// We can safely call |getClientID| here and during initialization: we would still
// spawn and return one single loading task.
this._clientID = yield ClientID.getClientID();
}
const pingData = this.assemblePing(aType, aPayload, aOptions);
this._log.trace("submitExternalPing - ping assembled, id: " + pingData.id);
// Always persist the pings if we are allowed to. We should not yield on any of the
// following operations to keep this function synchronous for the majority of the calls.
let archivePromise = TelemetryArchive.promiseArchivePing(pingData)
.catch(e => this._log.error("submitExternalPing - Failed to archive ping " + pingData.id, e));
let p = [ archivePromise ];
p.push(TelemetrySend.submitPing(pingData));
return Promise.all(p).then(() => pingData.id);
}),
/**
* Submit ping payloads to Telemetry.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {Object} [aOptions] Options object.
* @param {Boolean} [aOptions.addClientId=false] true if the ping should contain the client
* id, false otherwise.
* @param {Boolean} [aOptions.addEnvironment=false] true if the ping should contain the
* environment data.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
* @returns {Promise} Test-only - a promise that is resolved with the ping id once the ping is stored or sent.
*/
submitExternalPing: function send(aType, aPayload, aOptions) {
this._log.trace("submitExternalPing - type: " + aType + ", aOptions: " + JSON.stringify(aOptions));
// Enforce the type string to only contain sane characters.
const typeUuid = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i;
if (!typeUuid.test(aType)) {
this._log.error("submitExternalPing - invalid ping type: " + aType);
let histogram = Telemetry.getKeyedHistogramById("TELEMETRY_INVALID_PING_TYPE_SUBMITTED");
histogram.add(aType, 1);
return Promise.reject(new Error("Invalid type string submitted."));
}
// Enforce that the payload is an object.
if (aPayload === null || typeof aPayload !== 'object' || Array.isArray(aPayload)) {
this._log.error("submitExternalPing - invalid payload type: " + typeof aPayload);
let histogram = Telemetry.getHistogramById("TELEMETRY_INVALID_PAYLOAD_SUBMITTED");
histogram.add(1);
return Promise.reject(new Error("Invalid payload type submitted."));
}
let promise = this._submitPingLogic(aType, aPayload, aOptions);
this._trackPendingPingTask(promise);
return promise;
},
/**
* Save a ping to disk.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {Object} aOptions Options object.
* @param {Boolean} aOptions.addClientId true if the ping should contain the client id,
* false otherwise.
* @param {Boolean} aOptions.addEnvironment true if the ping should contain the
* environment data.
* @param {Boolean} aOptions.overwrite true overwrites a ping with the same name, if found.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
*
* @returns {Promise} A promise that resolves with the ping id when the ping is saved to
* disk.
*/
addPendingPing: function addPendingPing(aType, aPayload, aOptions) {
this._log.trace("addPendingPing - Type " + aType + ", aOptions " + JSON.stringify(aOptions));
let pingData = this.assemblePing(aType, aPayload, aOptions);
let savePromise = TelemetryStorage.savePendingPing(pingData);
let archivePromise = TelemetryArchive.promiseArchivePing(pingData).catch(e => {
this._log.error("addPendingPing - Failed to archive ping " + pingData.id, e);
});
// Wait for both the archiving and ping persistence to complete.
let promises = [
savePromise,
archivePromise,
];
return Promise.all(promises).then(() => pingData.id);
},
/**
* Write a ping to a specified location on the disk. Does not add the ping to the
* pending pings.
*
* @param {String} aType The type of the ping.
* @param {Object} aPayload The actual data payload for the ping.
* @param {String} aFilePath The path to save the ping to.
* @param {Object} aOptions Options object.
* @param {Boolean} aOptions.addClientId true if the ping should contain the client id,
* false otherwise.
* @param {Boolean} aOptions.addEnvironment true if the ping should contain the
* environment data.
* @param {Boolean} aOptions.overwrite true overwrites a ping with the same name, if found.
* @param {Object} [aOptions.overrideEnvironment=null] set to override the environment data.
*
* @returns {Promise} A promise that resolves with the ping id when the ping is saved to
* disk.
*/
savePing: function savePing(aType, aPayload, aFilePath, aOptions) {
this._log.trace("savePing - Type " + aType + ", File Path " + aFilePath +
", aOptions " + JSON.stringify(aOptions));
let pingData = this.assemblePing(aType, aPayload, aOptions);
return TelemetryStorage.savePingToFile(pingData, aFilePath, aOptions.overwrite)
.then(() => pingData.id);
},
/**
* Check whether we have an aborted-session ping. If so add it to the pending pings and archive it.
*
* @return {Promise} Promise that is resolved when the ping is submitted and archived.
*/
checkAbortedSessionPing: Task.async(function*() {
let ping = yield TelemetryStorage.loadAbortedSessionPing();
this._log.trace("checkAbortedSessionPing - found aborted-session ping: " + !!ping);
if (!ping) {
return;
}
try {
yield TelemetryStorage.addPendingPing(ping);
yield TelemetryArchive.promiseArchivePing(ping);
} catch (e) {
this._log.error("checkAbortedSessionPing - Unable to add the pending ping", e);
} finally {
yield TelemetryStorage.removeAbortedSessionPing();
}
}),
/**
* Save an aborted-session ping to disk without adding it to the pending pings.
*
* @param {Object} aPayload The ping payload data.
* @return {Promise} Promise that is resolved when the ping is saved.
*/
saveAbortedSessionPing: function(aPayload) {
this._log.trace("saveAbortedSessionPing");
const options = {addClientId: true, addEnvironment: true};
const pingData = this.assemblePing(PING_TYPE_MAIN, aPayload, options);
return TelemetryStorage.saveAbortedSessionPing(pingData);
},
removeAbortedSessionPing: function() {
return TelemetryStorage.removeAbortedSessionPing();
},
/**
* Perform telemetry initialization for either chrome or content process.
* @return {Boolean} True if Telemetry is allowed to record at least base (FHR) data,
* false otherwise.
*/
enableTelemetryRecording: function enableTelemetryRecording() {
// The thumbnail service also runs in a content process, even with e10s off.
// We need to check if e10s is on so we don't submit child payloads for it.
// We still need xpcshell child tests to work, so we skip this if test mode is enabled.
if (Utils.isContentProcess && !this._testMode && !Services.appinfo.browserTabsRemoteAutostart) {
this._log.config("enableTelemetryRecording - not enabling Telemetry for non-e10s child process");
Telemetry.canRecordBase = false;
Telemetry.canRecordExtended = false;
return false;
}
// Configure base Telemetry recording.
// Unified Telemetry makes it opt-out. If extended Telemetry is enabled, base recording
// is always on as well.
const enabled = Utils.isTelemetryEnabled;
Telemetry.canRecordBase = enabled || IS_UNIFIED_TELEMETRY;
Telemetry.canRecordExtended = enabled;
this._log.config("enableTelemetryRecording - canRecordBase:" + Telemetry.canRecordBase +
", canRecordExtended: " + Telemetry.canRecordExtended);
return Telemetry.canRecordBase;
},
/**
* This triggers basic telemetry initialization and schedules a full initialized for later
* for performance reasons.
*
* This delayed initialization means TelemetryController init can be in the following states:
* 1) setupTelemetry was never called
* or it was called and
* 2) _delayedInitTask was scheduled, but didn't run yet.
* 3) _delayedInitTask is currently running.
* 4) _delayedInitTask finished running and is nulled out.
*
* @return {Promise} Resolved when TelemetryController and TelemetrySession are fully
* initialized. This is only used in tests.
*/
setupTelemetry: function setupTelemetry(testing) {
this._initStarted = true;
this._testMode = testing;
this._log.trace("setupTelemetry");
if (this._delayedInitTask) {
this._log.error("setupTelemetry - init task already running");
return this._delayedInitTaskDeferred.promise;
}
if (this._initialized && !this._testMode) {
this._log.error("setupTelemetry - already initialized");
return Promise.resolve();
}
// This will trigger displaying the datachoices infobar.
TelemetryReportingPolicy.setup();
if (!this.enableTelemetryRecording()) {
this._log.config("setupChromeProcess - Telemetry recording is disabled, skipping Chrome process setup.");
return Promise.resolve();
}
// Initialize the session recorder.
if (!this._sessionRecorder) {
this._sessionRecorder = new SessionRecorder(PREF_SESSIONS_BRANCH);
this._sessionRecorder.onStartup();
}
this._attachObservers();
// Perform a lightweight, early initialization for the component, just registering
// a few observers and initializing the session.
TelemetrySession.earlyInit(this._testMode);
// For very short session durations, we may never load the client
// id from disk.
// We try to cache it in prefs to avoid this, even though this may
// lead to some stale client ids.
this._clientID = ClientID.getCachedClientID();
// Delay full telemetry initialization to give the browser time to
// run various late initializers. Otherwise our gathered memory
// footprint and other numbers would be too optimistic.
this._delayedInitTaskDeferred = Promise.defer();
this._delayedInitTask = new DeferredTask(function* () {
try {
// TODO: This should probably happen after all the delayed init here.
this._initialized = true;
TelemetryEnvironment.delayedInit();
yield TelemetrySend.setup(this._testMode);
// Load the ClientID.
this._clientID = yield ClientID.getClientID();
// Perform TelemetrySession delayed init.
yield TelemetrySession.delayedInit();
// Purge the pings archive by removing outdated pings. We don't wait for
// this task to complete, but TelemetryStorage blocks on it during
// shutdown.
TelemetryStorage.runCleanPingArchiveTask();
// Now that FHR/healthreporter is gone, make sure to remove FHR's DB from
// the profile directory. This is a temporary measure that we should drop
// in the future.
TelemetryStorage.removeFHRDatabase();
this._delayedInitTaskDeferred.resolve();
} catch (e) {
this._delayedInitTaskDeferred.reject(e);
} finally {
this._delayedInitTask = null;
}
}.bind(this), this._testMode ? TELEMETRY_TEST_DELAY : TELEMETRY_DELAY);
AsyncShutdown.sendTelemetry.addBlocker("TelemetryController: shutting down",
() => this.shutdown(),
() => this._getState());
this._delayedInitTask.arm();
return this._delayedInitTaskDeferred.promise;
},
/**
* This triggers basic telemetry initialization for content processes.
* @param {Boolean} [testing=false] True if we are in test mode, false otherwise.
*/
setupContentTelemetry: function (testing = false) {
this._testMode = testing;
// We call |enableTelemetryRecording| here to make sure that Telemetry.canRecord* flags
// are in sync between chrome and content processes.
if (!this.enableTelemetryRecording()) {
this._log.trace("setupContentTelemetry - Content process recording disabled.");
return;
}
TelemetrySession.setupContent(testing);
},
// Do proper shutdown waiting and cleanup.
_cleanupOnShutdown: Task.async(function*() {
if (!this._initialized) {
return;
}
Preferences.ignore(PREF_BRANCH_LOG, configureLogging);
this._detachObservers();
// Now do an orderly shutdown.
try {
// Stop the datachoices infobar display.
TelemetryReportingPolicy.shutdown();
TelemetryEnvironment.shutdown();
// Stop any ping sending.
yield TelemetrySend.shutdown();
yield TelemetrySession.shutdown();
// First wait for clients processing shutdown.
yield this._shutdownBarrier.wait();
// ... and wait for any outstanding async ping activity.
yield this._connectionsBarrier.wait();
// Perform final shutdown operations.
yield TelemetryStorage.shutdown();
} finally {
// Reset state.
this._initialized = false;
this._initStarted = false;
}
}),
shutdown: function() {
this._log.trace("shutdown");
// We can be in one the following states here:
// 1) setupTelemetry was never called
// or it was called and
// 2) _delayedInitTask was scheduled, but didn't run yet.
// 3) _delayedInitTask is running now.
// 4) _delayedInitTask finished running already.
// This handles 1).
if (!this._initStarted) {
return Promise.resolve();
}
// This handles 4).
if (!this._delayedInitTask) {
// We already ran the delayed initialization.
return this._cleanupOnShutdown();
}
// This handles 2) and 3).
return this._delayedInitTask.finalize().then(() => this._cleanupOnShutdown());
},
/**
* This observer drives telemetry.
*/
observe: function (aSubject, aTopic, aData) {
// The logger might still be not available at this point.
if (aTopic == "profile-after-change" || aTopic == "app-startup") {
// If we don't have a logger, we need to make sure |Log.repository.getLogger()| is
// called before |getLoggerWithMessagePrefix|. Otherwise logging won't work.
configureLogging();
}
this._log.trace("observe - " + aTopic + " notified.");
switch (aTopic) {
case "profile-after-change":
// profile-after-change is only registered for chrome processes.
return this.setupTelemetry();
case "app-startup":
// app-startup is only registered for content processes.
return this.setupContentTelemetry();
}
return undefined;
},
/**
* Get an object describing the current state of this module for AsyncShutdown diagnostics.
*/
_getState: function() {
return {
initialized: this._initialized,
initStarted: this._initStarted,
haveDelayedInitTask: !!this._delayedInitTask,
shutdownBarrier: this._shutdownBarrier.state,
connectionsBarrier: this._connectionsBarrier.state,
sendModule: TelemetrySend.getShutdownState(),
};
},
/**
* Called whenever the FHR Upload preference changes (e.g. when user disables FHR from
* the preferences panel), this triggers sending the deletion ping.
*/
_onUploadPrefChange: function() {
const uploadEnabled = Preferences.get(PREF_FHR_UPLOAD_ENABLED, false);
if (uploadEnabled) {
// There's nothing we should do if we are enabling upload.
return;
}
let p = Task.spawn(function*() {
try {
// Clear the current pings.
yield TelemetrySend.clearCurrentPings();
// Remove all the pending pings, but not the deletion ping.
yield TelemetryStorage.runRemovePendingPingsTask();
} catch (e) {
this._log.error("_onUploadPrefChange - error clearing pending pings", e);
} finally {
// Always send the deletion ping.
this._log.trace("_onUploadPrefChange - Sending deletion ping.");
this.submitExternalPing(PING_TYPE_DELETION, {}, { addClientId: true });
}
}.bind(this));
this._shutdownBarrier.client.addBlocker(
"TelemetryController: removing pending pings after data upload was disabled", p);
},
_attachObservers: function() {
if (IS_UNIFIED_TELEMETRY) {
// Watch the FHR upload setting to trigger deletion pings.
Preferences.observe(PREF_FHR_UPLOAD_ENABLED, this._onUploadPrefChange, this);
}
},
/**
* Remove the preference observer to avoid leaks.
*/
_detachObservers: function() {
if (IS_UNIFIED_TELEMETRY) {
Preferences.ignore(PREF_FHR_UPLOAD_ENABLED, this._onUploadPrefChange, this);
}
},
/**
* Allows waiting for TelemetryControllers delayed initialization to complete.
* This will complete before TelemetryController is shutting down.
* @return {Promise} Resolved when delayed TelemetryController initialization completed.
*/
promiseInitialized: function() {
return this._delayedInitTaskDeferred.promise;
},
getCurrentPingData: function(aSubsession) {
this._log.trace("getCurrentPingData - subsession: " + aSubsession)
// Telemetry is disabled, don't gather any data.
if (!Telemetry.canRecordBase) {
return null;
}
const reason = aSubsession ? REASON_GATHER_SUBSESSION_PAYLOAD : REASON_GATHER_PAYLOAD;
const type = PING_TYPE_MAIN;
const payload = TelemetrySession.getPayload(reason);
const options = { addClientId: true, addEnvironment: true };
const ping = this.assemblePing(type, payload, options);
return ping;
},
reset: Task.async(function*() {
this._clientID = null;
this._detachObservers();
yield TelemetrySession.testReset();
this._connectionsBarrier = new AsyncShutdown.Barrier(
"TelemetryController: Waiting for pending ping activity"
);
this._shutdownBarrier = new AsyncShutdown.Barrier(
"TelemetryController: Waiting for clients."
);
// We need to kick of the controller setup first for tests that check the
// cached client id.
let controllerSetup = this.setupTelemetry(true);
yield TelemetrySend.reset();
yield TelemetryStorage.reset();
yield TelemetryEnvironment.testReset();
yield controllerSetup;
}),
};

1413
toolkit/components/telemetry/TelemetryEnvironment.jsm
View file

File diff suppressed because it is too large Load diff

686
toolkit/components/telemetry/TelemetryEvent.cpp
View file

@ -1,686 +0,0 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include <prtime.h>
#include "nsITelemetry.h"
#include "nsHashKeys.h"
#include "nsDataHashtable.h"
#include "nsClassHashtable.h"
#include "nsTArray.h"
#include "mozilla/StaticMutex.h"
#include "mozilla/Unused.h"
#include "mozilla/Maybe.h"
#include "mozilla/StaticPtr.h"
#include "jsapi.h"
#include "nsJSUtils.h"
#include "nsXULAppAPI.h"
#include "nsUTF8Utils.h"
#include "TelemetryCommon.h"
#include "TelemetryEvent.h"
#include "TelemetryEventData.h"
using mozilla::StaticMutex;
using mozilla::StaticMutexAutoLock;
using mozilla::ArrayLength;
using mozilla::Maybe;
using mozilla::Nothing;
using mozilla::Pair;
using mozilla::StaticAutoPtr;
using mozilla::Telemetry::Common::AutoHashtable;
using mozilla::Telemetry::Common::IsExpiredVersion;
using mozilla::Telemetry::Common::CanRecordDataset;
using mozilla::Telemetry::Common::IsInDataset;
using mozilla::Telemetry::Common::MsSinceProcessStart;
using mozilla::Telemetry::Common::LogToBrowserConsole;
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
//
// Naming: there are two kinds of functions in this file:
//
// * Functions taking a StaticMutexAutoLock: these can only be reached via
// an interface function (TelemetryEvent::*). They expect the interface
// function to have acquired |gTelemetryEventsMutex|, so they do not
// have to be thread-safe.
//
// * Functions named TelemetryEvent::*. This is the external interface.
// Entries and exits to these functions are serialised using
// |gTelemetryEventsMutex|.
//
// Avoiding races and deadlocks:
//
// All functions in the external interface (TelemetryEvent::*) are
// serialised using the mutex |gTelemetryEventsMutex|. This means
// that the external interface is thread-safe, and the internal
// functions can ignore thread safety. But it also brings a danger
// of deadlock if any function in the external interface can get back
// to that interface. That is, we will deadlock on any call chain like
// this:
//
// TelemetryEvent::* -> .. any functions .. -> TelemetryEvent::*
//
// To reduce the danger of that happening, observe the following rules:
//
// * No function in TelemetryEvent::* may directly call, nor take the
// address of, any other function in TelemetryEvent::*.
//
// * No internal function may call, nor take the address
// of, any function in TelemetryEvent::*.
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
//
// PRIVATE TYPES
namespace {
const uint32_t kEventCount = mozilla::Telemetry::EventID::EventCount;
// This is a special event id used to mark expired events, to make expiry checks
// faster at runtime.
const uint32_t kExpiredEventId = kEventCount + 1;
static_assert(kEventCount < kExpiredEventId, "Should not overflow.");
// This is the hard upper limit on the number of event records we keep in storage.
// If we cross this limit, we will drop any further event recording until elements
// are removed from storage.
const uint32_t kMaxEventRecords = 1000;
// Maximum length of any passed value string, in UTF8 byte sequence length.
const uint32_t kMaxValueByteLength = 80;
// Maximum length of any string value in the extra dictionary, in UTF8 byte sequence length.
const uint32_t kMaxExtraValueByteLength = 80;
typedef nsDataHashtable<nsCStringHashKey, uint32_t> EventMapType;
typedef nsClassHashtable<nsCStringHashKey, nsCString> StringMap;
enum class RecordEventResult {
Ok,
UnknownEvent,
InvalidExtraKey,
StorageLimitReached,
};
struct ExtraEntry {
const nsCString key;
const nsCString value;
};
typedef nsTArray<ExtraEntry> ExtraArray;
class EventRecord {
public:
EventRecord(double timestamp, uint32_t eventId, const Maybe<nsCString>& value,
const ExtraArray& extra)
: mTimestamp(timestamp)
, mEventId(eventId)
, mValue(value)
, mExtra(extra)
{}
EventRecord(const EventRecord& other)
: mTimestamp(other.mTimestamp)
, mEventId(other.mEventId)
, mValue(other.mValue)
, mExtra(other.mExtra)
{}
EventRecord& operator=(const EventRecord& other) = delete;
double Timestamp() const { return mTimestamp; }
uint32_t EventId() const { return mEventId; }
const Maybe<nsCString>& Value() const { return mValue; }
const ExtraArray& Extra() const { return mExtra; }
size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
private:
const double mTimestamp;
const uint32_t mEventId;
const Maybe<nsCString> mValue;
const ExtraArray mExtra;
};
// Implements the methods for EventInfo.
const char*
EventInfo::method() const
{
return &gEventsStringTable[this->method_offset];
}
const char*
EventInfo::object() const
{
return &gEventsStringTable[this->object_offset];
}
// Implements the methods for CommonEventInfo.
const char*
CommonEventInfo::category() const
{
return &gEventsStringTable[this->category_offset];
}
const char*
CommonEventInfo::expiration_version() const
{
return &gEventsStringTable[this->expiration_version_offset];
}
const char*
CommonEventInfo::extra_key(uint32_t index) const
{
MOZ_ASSERT(index < this->extra_count);
uint32_t key_index = gExtraKeysTable[this->extra_index + index];
return &gEventsStringTable[key_index];
}
// Implementation for the EventRecord class.
size_t
EventRecord::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
{
size_t n = 0;
if (mValue) {
n += mValue.value().SizeOfExcludingThisIfUnshared(aMallocSizeOf);
}
n += mExtra.ShallowSizeOfExcludingThis(aMallocSizeOf);
for (uint32_t i = 0; i < mExtra.Length(); ++i) {
n += mExtra[i].key.SizeOfExcludingThisIfUnshared(aMallocSizeOf);
n += mExtra[i].value.SizeOfExcludingThisIfUnshared(aMallocSizeOf);
}
return n;
}
nsCString
UniqueEventName(const nsACString& category, const nsACString& method, const nsACString& object)
{
nsCString name;
name.Append(category);
name.AppendLiteral("#");
name.Append(method);
name.AppendLiteral("#");
name.Append(object);
return name;
}
nsCString
UniqueEventName(const EventInfo& info)
{
return UniqueEventName(nsDependentCString(info.common_info.category()),
nsDependentCString(info.method()),
nsDependentCString(info.object()));
}
bool
IsExpiredDate(uint32_t expires_days_since_epoch) {
if (expires_days_since_epoch == 0) {
return false;
}
const uint32_t days_since_epoch = PR_Now() / (PRTime(PR_USEC_PER_SEC) * 24 * 60 * 60);
return expires_days_since_epoch <= days_since_epoch;
}
void
TruncateToByteLength(nsCString& str, uint32_t length)
{
// last will be the index of the first byte of the current multi-byte sequence.
uint32_t last = RewindToPriorUTF8Codepoint(str.get(), length);
str.Truncate(last);
}
} // anonymous namespace
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
//
// PRIVATE STATE, SHARED BY ALL THREADS
namespace {
// Set to true once this global state has been initialized.
bool gInitDone = false;
bool gCanRecordBase;
bool gCanRecordExtended;
// The Name -> ID cache map.
EventMapType gEventNameIDMap(kEventCount);
// The main event storage. Events are inserted here in recording order.
StaticAutoPtr<nsTArray<EventRecord>> gEventRecords;
} // namespace
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
//
// PRIVATE: thread-unsafe helpers for event recording.
namespace {
bool
CanRecordEvent(const StaticMutexAutoLock& lock, const CommonEventInfo& info)
{
if (!gCanRecordBase) {
return false;
}
return CanRecordDataset(info.dataset, gCanRecordBase, gCanRecordExtended);
}
RecordEventResult
RecordEvent(const StaticMutexAutoLock& lock, double timestamp,
const nsACString& category, const nsACString& method,
const nsACString& object, const Maybe<nsCString>& value,
const ExtraArray& extra)
{
// Apply hard limit on event count in storage.
if (gEventRecords->Length() >= kMaxEventRecords) {
return RecordEventResult::StorageLimitReached;
}
// Look up the event id.
const nsCString& name = UniqueEventName(category, method, object);
uint32_t eventId;
if (!gEventNameIDMap.Get(name, &eventId)) {
return RecordEventResult::UnknownEvent;
}
// If the event is expired, silently drop this call.
// We don't want recording for expired probes to be an error so code doesn't
// have to be removed at a specific time or version.
// Even logging warnings would become very noisy.
if (eventId == kExpiredEventId) {
return RecordEventResult::Ok;
}
// Check whether we can record this event.
const CommonEventInfo& common = gEventInfo[eventId].common_info;
if (!CanRecordEvent(lock, common)) {
return RecordEventResult::Ok;
}
// Check whether the extra keys passed are valid.
nsTHashtable<nsCStringHashKey> validExtraKeys;
for (uint32_t i = 0; i < common.extra_count; ++i) {
validExtraKeys.PutEntry(nsDependentCString(common.extra_key(i)));
}
for (uint32_t i = 0; i < extra.Length(); ++i) {
if (!validExtraKeys.GetEntry(extra[i].key)) {
return RecordEventResult::InvalidExtraKey;
}
}
// Add event record.
gEventRecords->AppendElement(EventRecord(timestamp, eventId, value, extra));
return RecordEventResult::Ok;
}
} // anonymous namespace
////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////
//
// EXTERNALLY VISIBLE FUNCTIONS in namespace TelemetryEvents::
// This is a StaticMutex rather than a plain Mutex (1) so that
// it gets initialised in a thread-safe manner the first time
// it is used, and (2) because it is never de-initialised, and
// a normal Mutex would show up as a leak in BloatView. StaticMutex
// also has the "OffTheBooks" property, so it won't show as a leak
// in BloatView.
// Another reason to use a StaticMutex instead of a plain Mutex is
// that, due to the nature of Telemetry, we cannot rely on having a
// mutex initialized in InitializeGlobalState. Unfortunately, we
// cannot make sure that no other function is called before this point.
static StaticMutex gTelemetryEventsMutex;
void
TelemetryEvent::InitializeGlobalState(bool aCanRecordBase, bool aCanRecordExtended)
{
StaticMutexAutoLock locker(gTelemetryEventsMutex);
MOZ_ASSERT(!gInitDone, "TelemetryEvent::InitializeGlobalState "
"may only be called once");
gCanRecordBase = aCanRecordBase;
gCanRecordExtended = aCanRecordExtended;
gEventRecords = new nsTArray<EventRecord>();
// Populate the static event name->id cache. Note that the event names are
// statically allocated and come from the automatically generated TelemetryEventData.h.
const uint32_t eventCount = static_cast<uint32_t>(mozilla::Telemetry::EventID::EventCount);
for (uint32_t i = 0; i < eventCount; ++i) {
const EventInfo& info = gEventInfo[i];
uint32_t eventId = i;
// If this event is expired, mark it with a special event id.
// This avoids doing expensive expiry checks at runtime.
if (IsExpiredVersion(info.common_info.expiration_version()) ||
IsExpiredDate(info.common_info.expiration_day)) {
eventId = kExpiredEventId;
}
gEventNameIDMap.Put(UniqueEventName(info), eventId);
}
#ifdef DEBUG
gEventNameIDMap.MarkImmutable();
#endif
gInitDone = true;
}
void
TelemetryEvent::DeInitializeGlobalState()
{
StaticMutexAutoLock locker(gTelemetryEventsMutex);
MOZ_ASSERT(gInitDone);
gCanRecordBase = false;
gCanRecordExtended = false;
gEventNameIDMap.Clear();
gEventRecords->Clear();
gEventRecords = nullptr;
gInitDone = false;
}
void
TelemetryEvent::SetCanRecordBase(bool b)
{
StaticMutexAutoLock locker(gTelemetryEventsMutex);
gCanRecordBase = b;
}
void
TelemetryEvent::SetCanRecordExtended(bool b) {
StaticMutexAutoLock locker(gTelemetryEventsMutex);
gCanRecordExtended = b;
}
nsresult
TelemetryEvent::RecordEvent(const nsACString& aCategory, const nsACString& aMethod,
const nsACString& aObject, JS::HandleValue aValue,
JS::HandleValue aExtra, JSContext* cx,
uint8_t optional_argc)
{
// Currently only recording in the parent process is supported.
if (!XRE_IsParentProcess()) {
return NS_OK;
}
// Get the current time.
double timestamp = -1;
nsresult rv = MsSinceProcessStart(&timestamp);
if (NS_FAILED(rv)) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Failed to get time since process start."));
return NS_OK;
}
// Check value argument.
if ((optional_argc > 0) && !aValue.isNull() && !aValue.isString()) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Invalid type for value parameter."));
return NS_OK;
}
// Extract value parameter.
Maybe<nsCString> value;
if (aValue.isString()) {
nsAutoJSString jsStr;
if (!jsStr.init(cx, aValue)) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Invalid string value for value parameter."));
return NS_OK;
}
nsCString str = NS_ConvertUTF16toUTF8(jsStr);
if (str.Length() > kMaxValueByteLength) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Value parameter exceeds maximum string length, truncating."));
TruncateToByteLength(str, kMaxValueByteLength);
}
value = mozilla::Some(str);
}
// Check extra argument.
if ((optional_argc > 1) && !aExtra.isNull() && !aExtra.isObject()) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Invalid type for extra parameter."));
return NS_OK;
}
// Extract extra dictionary.
ExtraArray extra;
if (aExtra.isObject()) {
JS::RootedObject obj(cx, &aExtra.toObject());
JS::Rooted<JS::IdVector> ids(cx, JS::IdVector(cx));
if (!JS_Enumerate(cx, obj, &ids)) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Failed to enumerate object."));
return NS_OK;
}
for (size_t i = 0, n = ids.length(); i < n; i++) {
nsAutoJSString key;
if (!key.init(cx, ids[i])) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Extra dictionary should only contain string keys."));
return NS_OK;
}
JS::Rooted<JS::Value> value(cx);
if (!JS_GetPropertyById(cx, obj, ids[i], &value)) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Failed to get extra property."));
return NS_OK;
}
nsAutoJSString jsStr;
if (!value.isString() || !jsStr.init(cx, value)) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Extra properties should have string values."));
return NS_OK;
}
nsCString str = NS_ConvertUTF16toUTF8(jsStr);
if (str.Length() > kMaxExtraValueByteLength) {
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Extra value exceeds maximum string length, truncating."));
TruncateToByteLength(str, kMaxExtraValueByteLength);
}
extra.AppendElement(ExtraEntry{NS_ConvertUTF16toUTF8(key), str});
}
}
// Lock for accessing internal data.
// While the lock is being held, no complex calls like JS calls can be made,
// as all of these could record Telemetry, which would result in deadlock.
RecordEventResult res;
{
StaticMutexAutoLock lock(gTelemetryEventsMutex);
if (!gInitDone) {
return NS_ERROR_FAILURE;
}
res = ::RecordEvent(lock, timestamp, aCategory, aMethod, aObject, value, extra);
}
// Trigger warnings or errors where needed.
switch (res) {
case RecordEventResult::UnknownEvent: {
JS_ReportErrorASCII(cx, R"(Unknown event: ["%s", "%s", "%s"])",
PromiseFlatCString(aCategory).get(),
PromiseFlatCString(aMethod).get(),
PromiseFlatCString(aObject).get());
return NS_ERROR_INVALID_ARG;
}
case RecordEventResult::InvalidExtraKey:
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Invalid extra key for event."));
return NS_OK;
case RecordEventResult::StorageLimitReached:
LogToBrowserConsole(nsIScriptError::warningFlag,
NS_LITERAL_STRING("Event storage limit reached."));
return NS_OK;
default:
return NS_OK;
}
}
nsresult
TelemetryEvent::CreateSnapshots(uint32_t aDataset, bool aClear, JSContext* cx,
uint8_t optional_argc, JS::MutableHandleValue aResult)
{
// Extract the events from storage.
nsTArray<EventRecord> events;
{
StaticMutexAutoLock locker(gTelemetryEventsMutex);
if (!gInitDone) {
return NS_ERROR_FAILURE;
}
uint32_t len = gEventRecords->Length();
for (uint32_t i = 0; i < len; ++i) {
const EventRecord& record = (*gEventRecords)[i];
const EventInfo& info = gEventInfo[record.EventId()];
if (IsInDataset(info.common_info.dataset, aDataset)) {
events.AppendElement(record);
}
}
if (aClear) {
gEventRecords->Clear();
}
}
// We serialize the events to a JS array.
JS::RootedObject eventsArray(cx, JS_NewArrayObject(cx, events.Length()));
if (!eventsArray) {
return NS_ERROR_FAILURE;
}
for (uint32_t i = 0; i < events.Length(); ++i) {
const EventRecord& record = events[i];
const EventInfo& info = gEventInfo[record.EventId()];
// Each entry is an array of one of the forms:
// [timestamp, category, method, object, value]
// [timestamp, category, method, object, null, extra]
// [timestamp, category, method, object, value, extra]
JS::AutoValueVector items(cx);
// Add timestamp.
JS::Rooted<JS::Value> val(cx);
if (!items.append(JS::NumberValue(floor(record.Timestamp())))) {
return NS_ERROR_FAILURE;
}
// Add category, method, object.
const char* strings[] = {
info.common_info.category(),
info.method(),
info.object(),
};
for (const char* s : strings) {
const NS_ConvertUTF8toUTF16 wide(s);
if (!items.append(JS::StringValue(JS_NewUCStringCopyN(cx, wide.Data(), wide.Length())))) {
return NS_ERROR_FAILURE;
}
}
// Add the optional string value only when needed.
// When extra is empty and this has no value, we can save a little space.
if (record.Value()) {
const NS_ConvertUTF8toUTF16 wide(record.Value().value());
if (!items.append(JS::StringValue(JS_NewUCStringCopyN(cx, wide.Data(), wide.Length())))) {
return NS_ERROR_FAILURE;
}
} else if (!record.Extra().IsEmpty()) {
if (!items.append(JS::NullValue())) {
return NS_ERROR_FAILURE;
}
}
// Add the optional extra dictionary.
// To save a little space, only add it when it is not empty.
if (!record.Extra().IsEmpty()) {
JS::RootedObject obj(cx, JS_NewPlainObject(cx));
if (!obj) {
return NS_ERROR_FAILURE;
}
// Add extra key & value entries.
const ExtraArray& extra = record.Extra();
for (uint32_t i = 0; i < extra.Length(); ++i) {
const NS_ConvertUTF8toUTF16 wide(extra[i].value);
JS::Rooted<JS::Value> value(cx);
value.setString(JS_NewUCStringCopyN(cx, wide.Data(), wide.Length()));
if (!JS_DefineProperty(cx, obj, extra[i].key.get(), value, JSPROP_ENUMERATE)) {
return NS_ERROR_FAILURE;
}
}
val.setObject(*obj);
if (!items.append(val)) {
return NS_ERROR_FAILURE;
}
}
// Add the record to the events array.
JS::RootedObject itemsArray(cx, JS_NewArrayObject(cx, items));
if (!JS_DefineElement(cx, eventsArray, i, itemsArray, JSPROP_ENUMERATE)) {
return NS_ERROR_FAILURE;
}
}
aResult.setObject(*eventsArray);
return NS_OK;
}
/**
* Resets all the stored events. This is intended to be only used in tests.
*/
void
TelemetryEvent::ClearEvents()
{
StaticMutexAutoLock lock(gTelemetryEventsMutex);
if (!gInitDone) {
return;
}
gEventRecords->Clear();
}
size_t
TelemetryEvent::SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf)
{
StaticMutexAutoLock locker(gTelemetryEventsMutex);
size_t n = 0;
n += gEventRecords->ShallowSizeOfIncludingThis(aMallocSizeOf);
for (uint32_t i = 0; i < gEventRecords->Length(); ++i) {
n += (*gEventRecords)[i].SizeOfExcludingThis(aMallocSizeOf);
}
n += gEventNameIDMap.ShallowSizeOfExcludingThis(aMallocSizeOf);
for (auto iter = gEventNameIDMap.ConstIter(); !iter.Done(); iter.Next()) {
n += iter.Key().SizeOfExcludingThisIfUnshared(aMallocSizeOf);
}
return n;
}

39
toolkit/components/telemetry/TelemetryEvent.h
View file

@ -1,39 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TelemetryEvent_h__
#define TelemetryEvent_h__
#include "mozilla/TelemetryEventEnums.h"
// This module is internal to Telemetry. It encapsulates Telemetry's
// event recording and storage logic. It should only be used by
// Telemetry.cpp. These functions should not be used anywhere else.
// For the public interface to Telemetry functionality, see Telemetry.h.
namespace TelemetryEvent {
void InitializeGlobalState(bool canRecordBase, bool canRecordExtended);
void DeInitializeGlobalState();
void SetCanRecordBase(bool b);
void SetCanRecordExtended(bool b);
// JS API Endpoints.
nsresult RecordEvent(const nsACString& aCategory, const nsACString& aMethod,
const nsACString& aObject, JS::HandleValue aValue,
JS::HandleValue aExtra, JSContext* aCx,
uint8_t optional_argc);
nsresult CreateSnapshots(uint32_t aDataset, bool aClear, JSContext* aCx,
uint8_t optional_argc, JS::MutableHandleValue aResult);
// Only to be used for testing.
void ClearEvents();
size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf);
} // namespace TelemetryEvent
#endif // TelemetryEvent_h__

2643
toolkit/components/telemetry/TelemetryHistogram.cpp
View file

File diff suppressed because it is too large Load diff

104
toolkit/components/telemetry/TelemetryHistogram.h
View file

@ -1,104 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TelemetryHistogram_h__
#define TelemetryHistogram_h__
#include "mozilla/TelemetryHistogramEnums.h"
#include "mozilla/TelemetryComms.h"
#include "nsXULAppAPI.h"
// This module is internal to Telemetry. It encapsulates Telemetry's
// histogram accumulation and storage logic. It should only be used by
// Telemetry.cpp. These functions should not be used anywhere else.
// For the public interface to Telemetry functionality, see Telemetry.h.
namespace TelemetryHistogram {
void CreateStatisticsRecorder();
void DestroyStatisticsRecorder();
void InitializeGlobalState(bool canRecordBase, bool canRecordExtended);
void DeInitializeGlobalState();
#ifdef DEBUG
bool GlobalStateHasBeenInitialized();
#endif
bool CanRecordBase();
void SetCanRecordBase(bool b);
bool CanRecordExtended();
void SetCanRecordExtended(bool b);
void InitHistogramRecordingEnabled();
void SetHistogramRecordingEnabled(mozilla::Telemetry::ID aID, bool aEnabled);
nsresult SetHistogramRecordingEnabled(const nsACString &id, bool aEnabled);
void Accumulate(mozilla::Telemetry::ID aHistogram, uint32_t aSample);
void Accumulate(mozilla::Telemetry::ID aID, const nsCString& aKey,
uint32_t aSample);
void Accumulate(const char* name, uint32_t sample);
void Accumulate(const char* name, const nsCString& key, uint32_t sample);
void AccumulateCategorical(mozilla::Telemetry::ID aId, const nsCString& aLabel);
void AccumulateChild(GeckoProcessType aProcessType,
const nsTArray<mozilla::Telemetry::Accumulation>& aAccumulations);
void AccumulateChildKeyed(GeckoProcessType aProcessType,
const nsTArray<mozilla::Telemetry::KeyedAccumulation>& aAccumulations);
nsresult
GetHistogramById(const nsACString &name, JSContext *cx,
JS::MutableHandle<JS::Value> ret);
nsresult
GetKeyedHistogramById(const nsACString &name, JSContext *cx,
JS::MutableHandle<JS::Value> ret);
const char*
GetHistogramName(mozilla::Telemetry::ID id);
nsresult
CreateHistogramSnapshots(JSContext *cx, JS::MutableHandle<JS::Value> ret,
bool subsession, bool clearSubsession);
nsresult
RegisteredHistograms(uint32_t aDataset, uint32_t *aCount,
char*** aHistograms);
nsresult
RegisteredKeyedHistograms(uint32_t aDataset, uint32_t *aCount,
char*** aHistograms);
nsresult
GetKeyedHistogramSnapshots(JSContext *cx, JS::MutableHandle<JS::Value> ret);
nsresult
RegisterAddonHistogram(const nsACString &id, const nsACString &name,
uint32_t histogramType, uint32_t min, uint32_t max,
uint32_t bucketCount, uint8_t optArgCount);
nsresult
GetAddonHistogram(const nsACString &id, const nsACString &name,
JSContext *cx, JS::MutableHandle<JS::Value> ret);
nsresult
UnregisterAddonHistograms(const nsACString &id);
nsresult
GetAddonHistogramSnapshots(JSContext *cx, JS::MutableHandle<JS::Value> ret);
size_t
GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
size_t
GetHistogramSizesofIncludingThis(mozilla::MallocSizeOf aMallocSizeOf);
void
IPCTimerFired(nsITimer* aTimer, void* aClosure);
} // namespace TelemetryHistogram
#endif // TelemetryHistogram_h__

35
toolkit/components/telemetry/TelemetryLog.jsm
View file

@ -1,35 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
this.EXPORTED_SYMBOLS = ["TelemetryLog"];
const Cc = Components.classes;
const Ci = Components.interfaces;
const Telemetry = Cc["@mozilla.org/base/telemetry;1"].getService(Ci.nsITelemetry);
var gLogEntries = [];
this.TelemetryLog = Object.freeze({
log: function(id, data) {
id = String(id);
var ts;
try {
ts = Math.floor(Telemetry.msSinceProcessStart());
} catch (e) {
// If timestamp is screwed up, we just give up instead of making up
// data.
return;
}
var entry = [id, ts];
if (data !== undefined) {
entry = entry.concat(Array.prototype.map.call(data, String));
}
gLogEntries.push(entry);
},
entries: function() {
return gLogEntries;
}
});

496
toolkit/components/telemetry/TelemetryReportingPolicy.jsm
View file

@ -1,496 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = [
"TelemetryReportingPolicy"
];
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm", this);
Cu.import("resource://gre/modules/Preferences.jsm", this);
Cu.import("resource://gre/modules/Services.jsm", this);
Cu.import("resource://gre/modules/Timer.jsm", this);
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://services-common/observers.js", this);
XPCOMUtils.defineLazyModuleGetter(this, "TelemetrySend",
"resource://gre/modules/TelemetrySend.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils",
"resource://gre/modules/UpdateUtils.jsm");
const LOGGER_NAME = "Toolkit.Telemetry";
const LOGGER_PREFIX = "TelemetryReportingPolicy::";
// Oldest year to allow in date preferences. The FHR infobar was implemented in
// 2012 and no dates older than that should be encountered.
const OLDEST_ALLOWED_ACCEPTANCE_YEAR = 2012;
const PREF_BRANCH = "datareporting.policy.";
// Indicates whether this is the first run or not. This is used to decide when to display
// the policy.
const PREF_FIRST_RUN = "toolkit.telemetry.reportingpolicy.firstRun";
// Allows to skip the datachoices infobar. This should only be used in tests.
const PREF_BYPASS_NOTIFICATION = PREF_BRANCH + "dataSubmissionPolicyBypassNotification";
// The submission kill switch: if this preference is disable, no submission will ever take place.
const PREF_DATA_SUBMISSION_ENABLED = PREF_BRANCH + "dataSubmissionEnabled";
// This preference holds the current policy version, which overrides
// DEFAULT_DATAREPORTING_POLICY_VERSION
const PREF_CURRENT_POLICY_VERSION = PREF_BRANCH + "currentPolicyVersion";
// This indicates the minimum required policy version. If the accepted policy version
// is lower than this, the notification bar must be showed again.
const PREF_MINIMUM_POLICY_VERSION = PREF_BRANCH + "minimumPolicyVersion";
// The version of the accepted policy.
const PREF_ACCEPTED_POLICY_VERSION = PREF_BRANCH + "dataSubmissionPolicyAcceptedVersion";
// The date user accepted the policy.
const PREF_ACCEPTED_POLICY_DATE = PREF_BRANCH + "dataSubmissionPolicyNotifiedTime";
// URL of privacy policy to be opened in a background tab on first run instead of showing the
// data choices infobar.
const PREF_FIRST_RUN_URL = PREF_BRANCH + "firstRunURL";
// The following preferences are deprecated and will be purged during the preferences
// migration process.
const DEPRECATED_FHR_PREFS = [
PREF_BRANCH + "dataSubmissionPolicyAccepted",
PREF_BRANCH + "dataSubmissionPolicyBypassAcceptance",
PREF_BRANCH + "dataSubmissionPolicyResponseType",
PREF_BRANCH + "dataSubmissionPolicyResponseTime"
];
// How much time until we display the data choices notification bar, on the first run.
const NOTIFICATION_DELAY_FIRST_RUN_MSEC = 60 * 1000; // 60s
// Same as above, for the next runs.
const NOTIFICATION_DELAY_NEXT_RUNS_MSEC = 10 * 1000; // 10s
/**
* This is a policy object used to override behavior within this module.
* Tests override properties on this object to allow for control of behavior
* that would otherwise be very hard to cover.
*/
var Policy = {
now: () => new Date(),
setShowInfobarTimeout: (callback, delayMs) => setTimeout(callback, delayMs),
clearShowInfobarTimeout: (id) => clearTimeout(id),
};
/**
* Represents a request to display data policy.
*
* Receivers of these instances are expected to call one or more of the on*
* functions when events occur.
*
* When one of these requests is received, the first thing a callee should do
* is present notification to the user of the data policy. When the notice
* is displayed to the user, the callee should call `onUserNotifyComplete`.
*
* If for whatever reason the callee could not display a notice,
* it should call `onUserNotifyFailed`.
*
* @param {Object} aLog The log object used to log the error in case of failures.
*/
function NotifyPolicyRequest(aLog) {
this._log = aLog;
}
NotifyPolicyRequest.prototype = Object.freeze({
/**
* Called when the user is notified of the policy.
*/
onUserNotifyComplete: function() {
return TelemetryReportingPolicyImpl._userNotified();
},
/**
* Called when there was an error notifying the user about the policy.
*
* @param error
* (Error) Explains what went wrong.
*/
onUserNotifyFailed: function (error) {
this._log.error("onUserNotifyFailed - " + error);
},
});
this.TelemetryReportingPolicy = {
// The current policy version number. If the version number stored in the prefs
// is smaller than this, data upload will be disabled until the user is re-notified
// about the policy changes.
DEFAULT_DATAREPORTING_POLICY_VERSION: 1,
/**
* Setup the policy.
*/
setup: function() {
return TelemetryReportingPolicyImpl.setup();
},
/**
* Shutdown and clear the policy.
*/
shutdown: function() {
return TelemetryReportingPolicyImpl.shutdown();
},
/**
* Check if we are allowed to upload data. In order to submit data both these conditions
* should be true:
* - The data submission preference should be true.
* - The datachoices infobar should have been displayed.
*
* @return {Boolean} True if we are allowed to upload data, false otherwise.
*/
canUpload: function() {
return TelemetryReportingPolicyImpl.canUpload();
},
/**
* Test only method, restarts the policy.
*/
reset: function() {
return TelemetryReportingPolicyImpl.reset();
},
/**
* Test only method, used to check if user is notified of the policy in tests.
*/
testIsUserNotified: function() {
return TelemetryReportingPolicyImpl.isUserNotifiedOfCurrentPolicy;
},
/**
* Test only method, used to simulate the infobar being shown in xpcshell tests.
*/
testInfobarShown: function() {
return TelemetryReportingPolicyImpl._userNotified();
},
};
var TelemetryReportingPolicyImpl = {
_logger: null,
// Keep track of the notification status if user wasn't notified already.
_notificationInProgress: false,
// The timer used to show the datachoices notification at startup.
_startupNotificationTimerId: null,
get _log() {
if (!this._logger) {
this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX);
}
return this._logger;
},
/**
* Get the date the policy was notified.
* @return {Object} A date object or null on errors.
*/
get dataSubmissionPolicyNotifiedDate() {
let prefString = Preferences.get(PREF_ACCEPTED_POLICY_DATE, "0");
let valueInteger = parseInt(prefString, 10);
// Bail out if we didn't store any value yet.
if (valueInteger == 0) {
this._log.info("get dataSubmissionPolicyNotifiedDate - No date stored yet.");
return null;
}
// If an invalid value is saved in the prefs, bail out too.
if (Number.isNaN(valueInteger)) {
this._log.error("get dataSubmissionPolicyNotifiedDate - Invalid date stored.");
return null;
}
// Make sure the notification date is newer then the oldest allowed date.
let date = new Date(valueInteger);
if (date.getFullYear() < OLDEST_ALLOWED_ACCEPTANCE_YEAR) {
this._log.error("get dataSubmissionPolicyNotifiedDate - The stored date is too old.");
return null;
}
return date;
},
/**
* Set the date the policy was notified.
* @param {Object} aDate A valid date object.
*/
set dataSubmissionPolicyNotifiedDate(aDate) {
this._log.trace("set dataSubmissionPolicyNotifiedDate - aDate: " + aDate);
if (!aDate || aDate.getFullYear() < OLDEST_ALLOWED_ACCEPTANCE_YEAR) {
this._log.error("set dataSubmissionPolicyNotifiedDate - Invalid notification date.");
return;
}
Preferences.set(PREF_ACCEPTED_POLICY_DATE, aDate.getTime().toString());
},
/**
* Whether submission of data is allowed.
*
* This is the master switch for remote server communication. If it is
* false, we never request upload or deletion.
*/
get dataSubmissionEnabled() {
// Default is true because we are opt-out.
return Preferences.get(PREF_DATA_SUBMISSION_ENABLED, true);
},
get currentPolicyVersion() {
return Preferences.get(PREF_CURRENT_POLICY_VERSION,
TelemetryReportingPolicy.DEFAULT_DATAREPORTING_POLICY_VERSION);
},
/**
* The minimum policy version which for dataSubmissionPolicyAccepted to
* to be valid.
*/
get minimumPolicyVersion() {
const minPolicyVersion = Preferences.get(PREF_MINIMUM_POLICY_VERSION, 1);
// First check if the current channel has a specific minimum policy version. If not,
// use the general minimum policy version.
let channel = "";
try {
channel = UpdateUtils.UpdateChannel;
} catch (e) {
this._log.error("minimumPolicyVersion - Unable to retrieve the current channel.");
return minPolicyVersion;
}
const channelPref = PREF_MINIMUM_POLICY_VERSION + ".channel-" + channel;
return Preferences.get(channelPref, minPolicyVersion);
},
get dataSubmissionPolicyAcceptedVersion() {
return Preferences.get(PREF_ACCEPTED_POLICY_VERSION, 0);
},
set dataSubmissionPolicyAcceptedVersion(value) {
Preferences.set(PREF_ACCEPTED_POLICY_VERSION, value);
},
/**
* Checks to see if the user has been notified about data submission
* @return {Bool} True if user has been notified and the notification is still valid,
* false otherwise.
*/
get isUserNotifiedOfCurrentPolicy() {
// If we don't have a sane notification date, the user was not notified yet.
if (!this.dataSubmissionPolicyNotifiedDate ||
this.dataSubmissionPolicyNotifiedDate.getTime() <= 0) {
return false;
}
// The accepted policy version should not be less than the minimum policy version.
if (this.dataSubmissionPolicyAcceptedVersion < this.minimumPolicyVersion) {
return false;
}
// Otherwise the user was already notified.
return true;
},
/**
* Test only method, restarts the policy.
*/
reset: function() {
this.shutdown();
return this.setup();
},
/**
* Setup the policy.
*/
setup: function() {
this._log.trace("setup");
// Migrate the data choices infobar, if needed.
this._migratePreferences();
// Add the event observers.
Services.obs.addObserver(this, "sessionstore-windows-restored", false);
},
/**
* Clean up the reporting policy.
*/
shutdown: function() {
this._log.trace("shutdown");
this._detachObservers();
Policy.clearShowInfobarTimeout(this._startupNotificationTimerId);
},
/**
* Detach the observers that were attached during setup.
*/
_detachObservers: function() {
Services.obs.removeObserver(this, "sessionstore-windows-restored");
},
/**
* Check if we are allowed to upload data. In order to submit data both these conditions
* should be true:
* - The data submission preference should be true.
* - The datachoices infobar should have been displayed.
*
* @return {Boolean} True if we are allowed to upload data, false otherwise.
*/
canUpload: function() {
// If data submission is disabled, there's no point in showing the infobar. Just
// forbid to upload.
if (!this.dataSubmissionEnabled) {
return false;
}
// Submission is enabled. We enable upload if user is notified or we need to bypass
// the policy.
const bypassNotification = Preferences.get(PREF_BYPASS_NOTIFICATION, false);
return this.isUserNotifiedOfCurrentPolicy || bypassNotification;
},
/**
* Migrate the data policy preferences, if needed.
*/
_migratePreferences: function() {
// Current prefs are mostly the same than the old ones, except for some deprecated ones.
for (let pref of DEPRECATED_FHR_PREFS) {
Preferences.reset(pref);
}
},
/**
* Show the data choices infobar if the user wasn't already notified and data submission
* is enabled.
*/
_showInfobar: function() {
if (!this.dataSubmissionEnabled) {
this._log.trace("_showInfobar - Data submission disabled by the policy.");
return;
}
const bypassNotification = Preferences.get(PREF_BYPASS_NOTIFICATION, false);
if (this.isUserNotifiedOfCurrentPolicy || bypassNotification) {
this._log.trace("_showInfobar - User already notified or bypassing the policy.");
return;
}
if (this._notificationInProgress) {
this._log.trace("_showInfobar - User not notified, notification already in progress.");
return;
}
this._log.trace("_showInfobar - User not notified, notifying now.");
this._notificationInProgress = true;
let request = new NotifyPolicyRequest(this._log);
Observers.notify("datareporting:notify-data-policy:request", request);
},
/**
* Called when the user is notified with the infobar or otherwise.
*/
_userNotified() {
this._log.trace("_userNotified");
this._recordNotificationData();
TelemetrySend.notifyCanUpload();
},
/**
* Record date and the version of the accepted policy.
*/
_recordNotificationData: function() {
this._log.trace("_recordNotificationData");
this.dataSubmissionPolicyNotifiedDate = Policy.now();
this.dataSubmissionPolicyAcceptedVersion = this.currentPolicyVersion;
// The user was notified and the notification data saved: the notification
// is no longer in progress.
this._notificationInProgress = false;
},
/**
* Try to open the privacy policy in a background tab instead of showing the infobar.
*/
_openFirstRunPage() {
let firstRunPolicyURL = Preferences.get(PREF_FIRST_RUN_URL, "");
if (!firstRunPolicyURL) {
return false;
}
firstRunPolicyURL = Services.urlFormatter.formatURL(firstRunPolicyURL);
let win;
try {
const { RecentWindow } = Cu.import("resource:///modules/RecentWindow.jsm", {});
win = RecentWindow.getMostRecentBrowserWindow();
} catch (e) {}
if (!win) {
this._log.info("Couldn't find browser window to open first-run page. Falling back to infobar.");
return false;
}
// We'll consider the user notified once the privacy policy has been loaded
// in a background tab even if that tab hasn't been selected.
let tab;
let progressListener = {};
progressListener.onStateChange =
(aBrowser, aWebProgress, aRequest, aStateFlags, aStatus) => {
if (aWebProgress.isTopLevel &&
tab &&
tab.linkedBrowser == aBrowser &&
aStateFlags & Ci.nsIWebProgressListener.STATE_STOP &&
aStateFlags & Ci.nsIWebProgressListener.STATE_IS_NETWORK) {
let uri = aBrowser.documentURI;
if (uri && !/^about:(blank|neterror|certerror|blocked)/.test(uri.spec)) {
this._userNotified();
} else {
this._log.info("Failed to load first-run page. Falling back to infobar.");
this._showInfobar();
}
removeListeners();
}
};
let removeListeners = () => {
win.removeEventListener("unload", removeListeners);
win.gBrowser.removeTabsProgressListener(progressListener);
};
win.addEventListener("unload", removeListeners);
win.gBrowser.addTabsProgressListener(progressListener);
tab = win.gBrowser.loadOneTab(firstRunPolicyURL, { inBackground: true });
return true;
},
observe: function(aSubject, aTopic, aData) {
if (aTopic != "sessionstore-windows-restored") {
return;
}
const isFirstRun = Preferences.get(PREF_FIRST_RUN, true);
if (isFirstRun) {
// We're performing the first run, flip firstRun preference for subsequent runs.
Preferences.set(PREF_FIRST_RUN, false);
try {
if (this._openFirstRunPage()) {
return;
}
} catch (e) {
this._log.error("Failed to open privacy policy tab: " + e);
}
}
// Show the info bar.
const delay =
isFirstRun ? NOTIFICATION_DELAY_FIRST_RUN_MSEC: NOTIFICATION_DELAY_NEXT_RUNS_MSEC;
this._startupNotificationTimerId = Policy.setShowInfobarTimeout(
// Calling |canUpload| eventually shows the infobar, if needed.
() => this._showInfobar(), delay);
},
};

1895
toolkit/components/telemetry/TelemetryScalar.cpp
View file

File diff suppressed because it is too large Load diff

64
toolkit/components/telemetry/TelemetryScalar.h
View file

@ -1,64 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef TelemetryScalar_h__
#define TelemetryScalar_h__
#include "mozilla/TelemetryScalarEnums.h"
// This module is internal to Telemetry. It encapsulates Telemetry's
// scalar accumulation and storage logic. It should only be used by
// Telemetry.cpp. These functions should not be used anywhere else.
// For the public interface to Telemetry functionality, see Telemetry.h.
namespace TelemetryScalar {
void InitializeGlobalState(bool canRecordBase, bool canRecordExtended);
void DeInitializeGlobalState();
void SetCanRecordBase(bool b);
void SetCanRecordExtended(bool b);
// JS API Endpoints.
nsresult Add(const nsACString& aName, JS::HandleValue aVal, JSContext* aCx);
nsresult Set(const nsACString& aName, JS::HandleValue aVal, JSContext* aCx);
nsresult SetMaximum(const nsACString& aName, JS::HandleValue aVal, JSContext* aCx);
nsresult CreateSnapshots(unsigned int aDataset, bool aClearScalars,
JSContext* aCx, uint8_t optional_argc,
JS::MutableHandle<JS::Value> aResult);
// Keyed JS API Endpoints.
nsresult Add(const nsACString& aName, const nsAString& aKey, JS::HandleValue aVal,
JSContext* aCx);
nsresult Set(const nsACString& aName, const nsAString& aKey, JS::HandleValue aVal,
JSContext* aCx);
nsresult SetMaximum(const nsACString& aName, const nsAString& aKey, JS::HandleValue aVal,
JSContext* aCx);
nsresult CreateKeyedSnapshots(unsigned int aDataset, bool aClearScalars,
JSContext* aCx, uint8_t optional_argc,
JS::MutableHandle<JS::Value> aResult);
// C++ API Endpoints.
void Add(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
void Set(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aValue);
void Set(mozilla::Telemetry::ScalarID aId, bool aValue);
void SetMaximum(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
// Keyed C++ API Endpoints.
void Add(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
void Set(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, bool aValue);
void SetMaximum(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
// Only to be used for testing.
void ClearScalars();
size_t GetMapShallowSizesOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf);
size_t GetScalarSizesOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf);
} // namespace TelemetryScalar
#endif // TelemetryScalar_h__

1114
toolkit/components/telemetry/TelemetrySend.jsm
View file

File diff suppressed because it is too large Load diff

2124
toolkit/components/telemetry/TelemetrySession.jsm
View file

File diff suppressed because it is too large Load diff

49
toolkit/components/telemetry/TelemetryStartup.js
View file

@ -1,49 +0,0 @@
/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryController",
"resource://gre/modules/TelemetryController.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "TelemetryEnvironment",
"resource://gre/modules/TelemetryEnvironment.jsm");
/**
* TelemetryStartup is needed to forward the "profile-after-change" notification
* to TelemetryController.jsm.
*/
function TelemetryStartup() {
}
TelemetryStartup.prototype.classID = Components.ID("{117b219f-92fe-4bd2-a21b-95a342a9d474}");
TelemetryStartup.prototype.QueryInterface = XPCOMUtils.generateQI([Components.interfaces.nsIObserver]);
TelemetryStartup.prototype.observe = function(aSubject, aTopic, aData) {
if (aTopic == "profile-after-change" || aTopic == "app-startup") {
TelemetryController.observe(null, aTopic, null);
}
if (aTopic == "profile-after-change") {
annotateEnvironment();
TelemetryEnvironment.registerChangeListener("CrashAnnotator", annotateEnvironment);
TelemetryEnvironment.onInitialized().then(() => annotateEnvironment());
}
}
function annotateEnvironment() {
try {
let cr = Cc["@mozilla.org/toolkit/crash-reporter;1"];
if (cr) {
let env = JSON.stringify(TelemetryEnvironment.currentEnvironment);
cr.getService(Ci.nsICrashReporter).annotateCrashReport("TelemetryEnvironment", env);
}
} catch (e) {
// crash reporting not built or disabled? Ignore errors
}
}
this.NSGetFactory = XPCOMUtils.generateNSGetFactory([TelemetryStartup]);

4
toolkit/components/telemetry/TelemetryStartup.manifest
View file

@ -1,4 +0,0 @@
component {117b219f-92fe-4bd2-a21b-95a342a9d474} TelemetryStartup.js
contract @mozilla.org/base/telemetry-startup;1 {117b219f-92fe-4bd2-a21b-95a342a9d474}
category profile-after-change TelemetryStartup @mozilla.org/base/telemetry-startup;1 process=main
category app-startup TelemetryStartup @mozilla.org/base/telemetry-startup;1 process=content

1882
toolkit/components/telemetry/TelemetryStorage.jsm
View file

File diff suppressed because it is too large Load diff

54
toolkit/components/telemetry/TelemetryTimestamps.jsm
View file

@ -1,54 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
this.EXPORTED_SYMBOLS = ["TelemetryTimestamps"];
const Cu = Components.utils;
/**
* This module's purpose is to collect timestamps for important
* application-specific events.
*
* The TelemetryController component attaches the timestamps stored by this module to
* the telemetry submission, substracting the process lifetime so that the times
* are relative to process startup. The overall goal is to produce a basic
* timeline of the startup process.
*/
var timeStamps = {};
this.TelemetryTimestamps = {
/**
* Adds a timestamp to the list. The addition of TimeStamps that already have
* a value stored is ignored.
*
* @param name must be a unique, generally "camelCase" descriptor of what the
* timestamp represents. e.g.: "delayedStartupStarted"
* @param value is a timeStamp in milliseconds since the epoch. If omitted,
* defaults to Date.now().
*/
add: function TT_add(name, value) {
// Default to "now" if not specified
if (value == null)
value = Date.now();
if (isNaN(value))
throw new Error("Value must be a timestamp");
// If there's an existing value, just ignore the new value.
if (timeStamps.hasOwnProperty(name))
return;
timeStamps[name] = value;
},
/**
* Returns a JS object containing all of the timeStamps as properties (can be
* easily serialized to JSON). Used by TelemetryController to retrieve the data
* to attach to the telemetry submission.
*/
get: function TT_get() {
// Return a copy of the object.
return Cu.cloneInto(timeStamps, {});
}
};

152
toolkit/components/telemetry/TelemetryUtils.jsm
View file

@ -1,152 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
this.EXPORTED_SYMBOLS = [
"TelemetryUtils"
];
const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Preferences.jsm", this);
const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000;
const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled";
const IS_CONTENT_PROCESS = (function() {
// We cannot use Services.appinfo here because in telemetry xpcshell tests,
// appinfo is initially unavailable, and becomes available only later on.
let runtime = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
return runtime.processType == Ci.nsIXULRuntime.PROCESS_TYPE_CONTENT;
})();
this.TelemetryUtils = {
/**
* True if this is a content process.
*/
get isContentProcess() {
return IS_CONTENT_PROCESS;
},
/**
* Returns the state of the Telemetry enabled preference, making sure
* it correctly evaluates to a boolean type.
*/
get isTelemetryEnabled() {
return Preferences.get(PREF_TELEMETRY_ENABLED, false) === true;
},
/**
* Turn a millisecond timestamp into a day timestamp.
*
* @param aMsec A number of milliseconds since Unix epoch.
* @return The number of whole days since Unix epoch.
*/
millisecondsToDays: function(aMsec) {
return Math.floor(aMsec / MILLISECONDS_PER_DAY);
},
/**
* Takes a date and returns it trunctated to a date with daily precision.
*/
truncateToDays: function(date) {
return new Date(date.getFullYear(),
date.getMonth(),
date.getDate(),
0, 0, 0, 0);
},
/**
* Check if the difference between the times is within the provided tolerance.
* @param {Number} t1 A time in milliseconds.
* @param {Number} t2 A time in milliseconds.
* @param {Number} tolerance The tolerance, in milliseconds.
* @return {Boolean} True if the absolute time difference is within the tolerance, false
* otherwise.
*/
areTimesClose: function(t1, t2, tolerance) {
return Math.abs(t1 - t2) <= tolerance;
},
/**
* Get the next midnight for a date.
* @param {Object} date The date object to check.
* @return {Object} The Date object representing the next midnight.
*/
getNextMidnight: function(date) {
let nextMidnight = new Date(this.truncateToDays(date));
nextMidnight.setDate(nextMidnight.getDate() + 1);
return nextMidnight;
},
/**
* Get the midnight which is closer to the provided date.
* @param {Object} date The date object to check.
* @param {Number} tolerance The tolerance within we find the closest midnight.
* @return {Object} The Date object representing the closes midnight, or null if midnight
* is not within the midnight tolerance.
*/
getNearestMidnight: function(date, tolerance) {
let lastMidnight = this.truncateToDays(date);
if (this.areTimesClose(date.getTime(), lastMidnight.getTime(), tolerance)) {
return lastMidnight;
}
const nextMidnightDate = this.getNextMidnight(date);
if (this.areTimesClose(date.getTime(), nextMidnightDate.getTime(), tolerance)) {
return nextMidnightDate;
}
return null;
},
generateUUID: function() {
let str = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator).generateUUID().toString();
// strip {}
return str.substring(1, str.length - 1);
},
/**
* Find how many months passed between two dates.
* @param {Object} aStartDate The starting date.
* @param {Object} aEndDate The ending date.
* @return {Integer} The number of months between the two dates.
*/
getElapsedTimeInMonths: function(aStartDate, aEndDate) {
return (aEndDate.getMonth() - aStartDate.getMonth())
+ 12 * (aEndDate.getFullYear() - aStartDate.getFullYear());
},
/**
* Date.toISOString() gives us UTC times, this gives us local times in
* the ISO date format. See http://www.w3.org/TR/NOTE-datetime
* @param {Object} date The input date.
* @return {String} The local time ISO string.
*/
toLocalTimeISOString: function(date) {
function padNumber(number, places) {
number = number.toString();
while (number.length < places) {
number = "0" + number;
}
return number;
}
let sign = (n) => n >= 0 ? "+" : "-";
// getTimezoneOffset counter-intuitively returns -60 for UTC+1.
let tzOffset = - date.getTimezoneOffset();
// YYYY-MM-DDThh:mm:ss.sTZD (eg 1997-07-16T19:20:30.45+01:00)
return padNumber(date.getFullYear(), 4)
+ "-" + padNumber(date.getMonth() + 1, 2)
+ "-" + padNumber(date.getDate(), 2)
+ "T" + padNumber(date.getHours(), 2)
+ ":" + padNumber(date.getMinutes(), 2)
+ ":" + padNumber(date.getSeconds(), 2)
+ "." + date.getMilliseconds()
+ sign(tzOffset) + padNumber(Math.floor(Math.abs(tzOffset / 60)), 2)
+ ":" + padNumber(Math.abs(tzOffset % 60), 2);
},
};

181
toolkit/components/telemetry/ThirdPartyCookieProbe.jsm
View file

@ -1,181 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
var Ci = Components.interfaces;
var Cu = Components.utils;
var Cr = Components.results;
Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
Cu.import("resource://gre/modules/Services.jsm", this);
this.EXPORTED_SYMBOLS = ["ThirdPartyCookieProbe"];
const MILLISECONDS_PER_DAY = 1000 * 60 * 60 * 24;
/**
* A probe implementing the measurements detailed at
* https://wiki.mozilla.org/SecurityEngineering/ThirdPartyCookies/Telemetry
*
* This implementation uses only in-memory data.
*/
this.ThirdPartyCookieProbe = function() {
/**
* A set of third-party sites that have caused cookies to be
* rejected. These sites are trimmed down to ETLD + 1
* (i.e. "x.y.com" and "z.y.com" are both trimmed down to "y.com",
* "x.y.co.uk" is trimmed down to "y.co.uk").
*
* Used to answer the following question: "For each third-party
* site, how many other first parties embed them and result in
* cookie traffic?" (see
* https://wiki.mozilla.org/SecurityEngineering/ThirdPartyCookies/Telemetry#Breadth
* )
*
* @type Map<string, RejectStats> A mapping from third-party site
* to rejection statistics.
*/
this._thirdPartyCookies = new Map();
/**
* Timestamp of the latest call to flush() in milliseconds since the Epoch.
*/
this._latestFlush = Date.now();
};
this.ThirdPartyCookieProbe.prototype = {
QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
init: function() {
Services.obs.addObserver(this, "profile-before-change", false);
Services.obs.addObserver(this, "third-party-cookie-accepted", false);
Services.obs.addObserver(this, "third-party-cookie-rejected", false);
},
dispose: function() {
Services.obs.removeObserver(this, "profile-before-change");
Services.obs.removeObserver(this, "third-party-cookie-accepted");
Services.obs.removeObserver(this, "third-party-cookie-rejected");
},
/**
* Observe either
* - "profile-before-change" (no meaningful subject or data) - time to flush statistics and unregister; or
* - "third-party-cookie-accepted"/"third-party-cookie-rejected" with
* subject: the nsIURI of the third-party that attempted to set the cookie;
* data: a string holding the uri of the page seen by the user.
*/
observe: function(docURI, topic, referrer) {
try {
if (topic == "profile-before-change") {
// A final flush, then unregister
this.flush();
this.dispose();
}
if (topic != "third-party-cookie-accepted"
&& topic != "third-party-cookie-rejected") {
// Not a third-party cookie
return;
}
// Add host to this._thirdPartyCookies
// Note: nsCookieService passes "?" if the issuer is unknown. Avoid
// normalizing in this case since its not a valid URI.
let firstParty = (referrer === "?") ? referrer : normalizeHost(referrer);
let thirdParty = normalizeHost(docURI.QueryInterface(Ci.nsIURI).host);
let data = this._thirdPartyCookies.get(thirdParty);
if (!data) {
data = new RejectStats();
this._thirdPartyCookies.set(thirdParty, data);
}
if (topic == "third-party-cookie-accepted") {
data.addAccepted(firstParty);
} else {
data.addRejected(firstParty);
}
} catch (ex) {
if (ex instanceof Ci.nsIXPCException) {
if (ex.result == Cr.NS_ERROR_HOST_IS_IP_ADDRESS ||
ex.result == Cr.NS_ERROR_INSUFFICIENT_DOMAIN_LEVELS) {
return;
}
}
// Other errors should not remain silent.
Services.console.logStringMessage("ThirdPartyCookieProbe: Uncaught error " + ex + "\n" + ex.stack);
}
},
/**
* Clear internal data, fill up corresponding histograms.
*
* @param {number} aNow (optional, used for testing purposes only)
* The current instant. Used to make tests time-independent.
*/
flush: function(aNow = Date.now()) {
let updays = (aNow - this._latestFlush) / MILLISECONDS_PER_DAY;
if (updays <= 0) {
// Unlikely, but regardless, don't risk division by zero
// or weird stuff.
return;
}
this._latestFlush = aNow;
this._thirdPartyCookies.clear();
}
};
/**
* Data gathered on cookies that a third party site has attempted to set.
*
* Privacy note: the only data actually sent to the server is the size of
* the sets.
*
* @constructor
*/
var RejectStats = function() {
/**
* The set of all sites for which we have accepted third-party cookies.
*/
this._acceptedSites = new Set();
/**
* The set of all sites for which we have rejected third-party cookies.
*/
this._rejectedSites = new Set();
/**
* Total number of attempts to set a third-party cookie that have
* been accepted. Two accepted attempts on the same site will both
* augment this count.
*/
this._acceptedRequests = 0;
/**
* Total number of attempts to set a third-party cookie that have
* been rejected. Two rejected attempts on the same site will both
* augment this count.
*/
this._rejectedRequests = 0;
};
RejectStats.prototype = {
addAccepted: function(firstParty) {
this._acceptedSites.add(firstParty);
this._acceptedRequests++;
},
addRejected: function(firstParty) {
this._rejectedSites.add(firstParty);
this._rejectedRequests++;
},
get countAcceptedSites() {
return this._acceptedSites.size;
},
get countRejectedSites() {
return this._rejectedSites.size;
},
get countAcceptedRequests() {
return this._acceptedRequests;
},
get countRejectedRequests() {
return this._rejectedRequests;
}
};
/**
* Normalize a host to its eTLD + 1.
*/
function normalizeHost(host) {
return Services.eTLD.getBaseDomainFromHost(host);
}

230
toolkit/components/telemetry/ThreadHangStats.h
View file

@ -1,230 +0,0 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef mozilla_BackgroundHangTelemetry_h
#define mozilla_BackgroundHangTelemetry_h
#include "mozilla/Array.h"
#include "mozilla/Assertions.h"
#include "mozilla/HangAnnotations.h"
#include "mozilla/Move.h"
#include "mozilla/Mutex.h"
#include "mozilla/PodOperations.h"
#include "mozilla/Vector.h"
#include "nsString.h"
#include "prinrval.h"
namespace mozilla {
namespace Telemetry {
static const size_t kTimeHistogramBuckets = 8 * sizeof(PRIntervalTime);
/* TimeHistogram is an efficient histogram that puts time durations into
exponential (base 2) buckets; times are accepted in PRIntervalTime and
stored in milliseconds. */
class TimeHistogram : public mozilla::Array<uint32_t, kTimeHistogramBuckets>
{
public:
TimeHistogram()
{
mozilla::PodArrayZero(*this);
}
// Get minimum (inclusive) range of bucket in milliseconds
uint32_t GetBucketMin(size_t aBucket) const {
MOZ_ASSERT(aBucket < ArrayLength(*this));
return (1u << aBucket) & ~1u; // Bucket 0 starts at 0, not 1
}
// Get maximum (inclusive) range of bucket in milliseconds
uint32_t GetBucketMax(size_t aBucket) const {
MOZ_ASSERT(aBucket < ArrayLength(*this));
return (1u << (aBucket + 1u)) - 1u;
}
void Add(PRIntervalTime aTime);
};
/* HangStack stores an array of const char pointers,
with optional internal storage for strings. */
class HangStack
{
public:
static const size_t sMaxInlineStorage = 8;
private:
typedef mozilla::Vector<const char*, sMaxInlineStorage> Impl;
Impl mImpl;
// Stack entries can either be a static const char*
// or a pointer to within this buffer.
mozilla::Vector<char, 0> mBuffer;
public:
HangStack() { }
HangStack(HangStack&& aOther)
: mImpl(mozilla::Move(aOther.mImpl))
, mBuffer(mozilla::Move(aOther.mBuffer))
{
}
bool operator==(const HangStack& aOther) const {
for (size_t i = 0; i < length(); i++) {
if (!IsSameAsEntry(operator[](i), aOther[i])) {
return false;
}
}
return true;
}
bool operator!=(const HangStack& aOther) const {
return !operator==(aOther);
}
const char*& operator[](size_t aIndex) {
return mImpl[aIndex];
}
const char* const& operator[](size_t aIndex) const {
return mImpl[aIndex];
}
size_t capacity() const { return mImpl.capacity(); }
size_t length() const { return mImpl.length(); }
bool empty() const { return mImpl.empty(); }
bool canAppendWithoutRealloc(size_t aNeeded) const {
return mImpl.canAppendWithoutRealloc(aNeeded);
}
void infallibleAppend(const char* aEntry) { mImpl.infallibleAppend(aEntry); }
bool reserve(size_t aRequest) { return mImpl.reserve(aRequest); }
const char** begin() { return mImpl.begin(); }
const char* const* begin() const { return mImpl.begin(); }
const char** end() { return mImpl.end(); }
const char* const* end() const { return mImpl.end(); }
const char*& back() { return mImpl.back(); }
void erase(const char** aEntry) { mImpl.erase(aEntry); }
void erase(const char** aBegin, const char** aEnd) {
mImpl.erase(aBegin, aEnd);
}
void clear() {
mImpl.clear();
mBuffer.clear();
}
bool IsInBuffer(const char* aEntry) const {
return aEntry >= mBuffer.begin() && aEntry < mBuffer.end();
}
bool IsSameAsEntry(const char* aEntry, const char* aOther) const {
// If the entry came from the buffer, we need to compare its content;
// otherwise we only need to compare its pointer.
return IsInBuffer(aEntry) ? !strcmp(aEntry, aOther) : (aEntry == aOther);
}
size_t AvailableBufferSize() const {
return mBuffer.capacity() - mBuffer.length();
}
bool EnsureBufferCapacity(size_t aCapacity) {
// aCapacity is the minimal capacity and Vector may make the actual
// capacity larger, in which case we want to use up all the space.
return mBuffer.reserve(aCapacity) &&
mBuffer.reserve(mBuffer.capacity());
}
const char* InfallibleAppendViaBuffer(const char* aText, size_t aLength);
const char* AppendViaBuffer(const char* aText, size_t aLength);
};
/* A hang histogram consists of a stack associated with the
hang, along with a time histogram of the hang times. */
class HangHistogram : public TimeHistogram
{
private:
static uint32_t GetHash(const HangStack& aStack);
HangStack mStack;
// Native stack that corresponds to the pseudostack in mStack
HangStack mNativeStack;
// Use a hash to speed comparisons
const uint32_t mHash;
// Annotations attributed to this stack
HangMonitor::HangAnnotationsVector mAnnotations;
public:
explicit HangHistogram(HangStack&& aStack)
: mStack(mozilla::Move(aStack))
, mHash(GetHash(mStack))
{
}
HangHistogram(HangHistogram&& aOther)
: TimeHistogram(mozilla::Move(aOther))
, mStack(mozilla::Move(aOther.mStack))
, mNativeStack(mozilla::Move(aOther.mNativeStack))
, mHash(mozilla::Move(aOther.mHash))
, mAnnotations(mozilla::Move(aOther.mAnnotations))
{
}
bool operator==(const HangHistogram& aOther) const;
bool operator!=(const HangHistogram& aOther) const
{
return !operator==(aOther);
}
const HangStack& GetStack() const {
return mStack;
}
HangStack& GetNativeStack() {
return mNativeStack;
}
const HangStack& GetNativeStack() const {
return mNativeStack;
}
const HangMonitor::HangAnnotationsVector& GetAnnotations() const {
return mAnnotations;
}
void Add(PRIntervalTime aTime, HangMonitor::HangAnnotationsPtr aAnnotations) {
TimeHistogram::Add(aTime);
if (aAnnotations) {
if (!mAnnotations.append(Move(aAnnotations))) {
MOZ_CRASH();
}
}
}
};
/* Thread hang stats consist of
- thread name
- time histogram of all task run times
- hang histograms of individual hangs
- annotations for each hang
*/
class ThreadHangStats
{
private:
nsCString mName;
public:
TimeHistogram mActivity;
mozilla::Vector<HangHistogram, 4> mHangs;
explicit ThreadHangStats(const char* aName)
: mName(aName)
{
}
ThreadHangStats(ThreadHangStats&& aOther)
: mName(mozilla::Move(aOther.mName))
, mActivity(mozilla::Move(aOther.mActivity))
, mHangs(mozilla::Move(aOther.mHangs))
{
}
const char* GetName() const {
return mName.get();
}
};
} // namespace Telemetry
} // namespace mozilla
#endif // mozilla_BackgroundHangTelemetry_h

231
toolkit/components/telemetry/UITelemetry.jsm
View file

@ -1,231 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
const Cu = Components.utils;
const PREF_BRANCH = "toolkit.telemetry.";
const PREF_ENABLED = PREF_BRANCH + "enabled";
this.EXPORTED_SYMBOLS = [
"UITelemetry",
];
Cu.import("resource://gre/modules/Services.jsm", this);
/**
* UITelemetry is a helper JSM used to record UI specific telemetry events.
*
* It implements nsIUITelemetryObserver, defined in nsIAndroidBridge.idl.
*/
this.UITelemetry = {
_enabled: undefined,
_activeSessions: {},
_measurements: [],
// Lazily decide whether telemetry is enabled.
get enabled() {
if (this._enabled !== undefined) {
return this._enabled;
}
// Set an observer to watch for changes at runtime.
Services.prefs.addObserver(PREF_ENABLED, this, false);
Services.obs.addObserver(this, "profile-before-change", false);
// Pick up the current value.
this._enabled = Services.prefs.getBoolPref(PREF_ENABLED, false);
return this._enabled;
},
observe: function(aSubject, aTopic, aData) {
if (aTopic == "profile-before-change") {
Services.obs.removeObserver(this, "profile-before-change");
Services.prefs.removeObserver(PREF_ENABLED, this);
this._enabled = undefined;
return;
}
if (aTopic == "nsPref:changed") {
switch (aData) {
case PREF_ENABLED:
let on = Services.prefs.getBoolPref(PREF_ENABLED);
this._enabled = on;
// Wipe ourselves if we were just disabled.
if (!on) {
this._activeSessions = {};
this._measurements = [];
}
break;
}
}
},
/**
* This exists exclusively for testing -- our events are not intended to
* be retrieved via an XPCOM interface.
*/
get wrappedJSObject() {
return this;
},
/**
* Holds the functions that provide UITelemetry's simple
* measurements. Those functions are mapped to unique names,
* and should be registered with addSimpleMeasureFunction.
*/
_simpleMeasureFunctions: {},
/**
* A hack to generate the relative timestamp from start when we don't have
* access to the Java timer.
* XXX: Bug 1007647 - Support realtime and/or uptime in JavaScript.
*/
uptimeMillis: function() {
return Date.now() - Services.startup.getStartupInfo().process;
},
/**
* Adds a single event described by a timestamp, an action, and the calling
* method.
*
* Optionally provide a string 'extras', which will be recorded as part of
* the event.
*
* All extant sessions will be recorded by name for each event.
*/
addEvent: function(aAction, aMethod, aTimestamp, aExtras) {
if (!this.enabled) {
return;
}
let sessions = Object.keys(this._activeSessions);
let aEvent = {
type: "event",
action: aAction,
method: aMethod,
sessions: sessions,
timestamp: (aTimestamp == undefined) ? this.uptimeMillis() : aTimestamp,
};
if (aExtras) {
aEvent.extras = aExtras;
}
this._recordEvent(aEvent);
},
/**
* Begins tracking a session by storing a timestamp for session start.
*/
startSession: function(aName, aTimestamp) {
if (!this.enabled) {
return;
}
if (this._activeSessions[aName]) {
// Do not overwrite a previous event start if it already exists.
return;
}
this._activeSessions[aName] = (aTimestamp == undefined) ? this.uptimeMillis() : aTimestamp;
},
/**
* Tracks the end of a session with a timestamp.
*/
stopSession: function(aName, aReason, aTimestamp) {
if (!this.enabled) {
return;
}
let sessionStart = this._activeSessions[aName];
delete this._activeSessions[aName];
if (!sessionStart) {
return;
}
let aEvent = {
type: "session",
name: aName,
reason: aReason,
start: sessionStart,
end: (aTimestamp == undefined) ? this.uptimeMillis() : aTimestamp,
};
this._recordEvent(aEvent);
},
_recordEvent: function(aEvent) {
this._measurements.push(aEvent);
},
/**
* Called by TelemetrySession to populate the simple measurement
* blob. This function will iterate over all functions added
* via addSimpleMeasureFunction and return an object with the
* results of those functions.
*/
getSimpleMeasures: function() {
if (!this.enabled) {
return {};
}
let result = {};
for (let name in this._simpleMeasureFunctions) {
result[name] = this._simpleMeasureFunctions[name]();
}
return result;
},
/**
* Allows the caller to register functions that will get called
* for simple measures during a Telemetry ping. aName is a unique
* identifier used as they key for the simple measurement in the
* object that getSimpleMeasures returns.
*
* This function throws an exception if aName already has a function
* registered for it.
*/
addSimpleMeasureFunction: function(aName, aFunction) {
if (!this.enabled) {
return;
}
if (aName in this._simpleMeasureFunctions) {
throw new Error("A simple measurement function is already registered for " + aName);
}
if (!aFunction || typeof aFunction !== 'function') {
throw new Error("addSimpleMeasureFunction called with non-function argument.");
}
this._simpleMeasureFunctions[aName] = aFunction;
},
removeSimpleMeasureFunction: function(aName) {
delete this._simpleMeasureFunctions[aName];
},
/**
* Called by TelemetrySession to populate the UI measurement
* blob.
*
* Optionally clears the set of measurements based on aClear.
*/
getUIMeasurements: function(aClear) {
if (!this.enabled) {
return [];
}
let measurements = this._measurements.slice();
if (aClear) {
this._measurements = [];
}
return measurements;
}
};

111
toolkit/components/telemetry/WebrtcTelemetry.cpp
View file

@ -1,111 +0,0 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "Telemetry.h"
#include "TelemetryCommon.h"
#include "WebrtcTelemetry.h"
#include "jsapi.h"
#include "nsPrintfCString.h"
#include "nsTHashtable.h"
using mozilla::Telemetry::Common::AutoHashtable;
void
WebrtcTelemetry::RecordIceCandidateMask(const uint32_t iceCandidateBitmask,
const bool success)
{
WebrtcIceCandidateType *entry = mWebrtcIceCandidates.GetEntry(iceCandidateBitmask);
if (!entry) {
entry = mWebrtcIceCandidates.PutEntry(iceCandidateBitmask);
if (MOZ_UNLIKELY(!entry))
return;
}
if (success) {
entry->mData.webrtc.successCount++;
} else {
entry->mData.webrtc.failureCount++;
}
}
bool
ReflectIceEntry(const WebrtcTelemetry::WebrtcIceCandidateType *entry,
const WebrtcTelemetry::WebrtcIceCandidateStats *stat, JSContext *cx,
JS::Handle<JSObject*> obj)
{
if ((stat->successCount == 0) && (stat->failureCount == 0))
return true;
const uint32_t &bitmask = entry->GetKey();
JS::Rooted<JSObject*> statsObj(cx, JS_NewPlainObject(cx));
if (!statsObj)
return false;
if (!JS_DefineProperty(cx, obj,
nsPrintfCString("%lu", bitmask).BeginReading(),
statsObj, JSPROP_ENUMERATE)) {
return false;
}
if (stat->successCount && !JS_DefineProperty(cx, statsObj, "successCount",
stat->successCount,
JSPROP_ENUMERATE)) {
return false;
}
if (stat->failureCount && !JS_DefineProperty(cx, statsObj, "failureCount",
stat->failureCount,
JSPROP_ENUMERATE)) {
return false;
}
return true;
}
bool
ReflectIceWebrtc(WebrtcTelemetry::WebrtcIceCandidateType *entry, JSContext *cx,
JS::Handle<JSObject*> obj)
{
return ReflectIceEntry(entry, &entry->mData.webrtc, cx, obj);
}
bool
WebrtcTelemetry::AddIceInfo(JSContext *cx, JS::Handle<JSObject*> iceObj)
{
JS::Rooted<JSObject*> statsObj(cx, JS_NewPlainObject(cx));
if (!statsObj)
return false;
if (!mWebrtcIceCandidates.ReflectIntoJS(ReflectIceWebrtc, cx, statsObj)) {
return false;
}
return JS_DefineProperty(cx, iceObj, "webrtc",
statsObj, JSPROP_ENUMERATE);
}
bool
WebrtcTelemetry::GetWebrtcStats(JSContext *cx, JS::MutableHandle<JS::Value> ret)
{
JS::Rooted<JSObject*> root_obj(cx, JS_NewPlainObject(cx));
if (!root_obj)
return false;
ret.setObject(*root_obj);
JS::Rooted<JSObject*> ice_obj(cx, JS_NewPlainObject(cx));
if (!ice_obj)
return false;
JS_DefineProperty(cx, root_obj, "IceCandidatesStats", ice_obj,
JSPROP_ENUMERATE);
if (!AddIceInfo(cx, ice_obj))
return false;
return true;
}
size_t
WebrtcTelemetry::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const
{
return mWebrtcIceCandidates.ShallowSizeOfExcludingThis(aMallocSizeOf);
}

42
toolkit/components/telemetry/WebrtcTelemetry.h
View file

@ -1,42 +0,0 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef WebrtcTelemetry_h__
#define WebrtcTelemetry_h__
#include "nsBaseHashtable.h"
#include "nsHashKeys.h"
#include "TelemetryCommon.h"
class WebrtcTelemetry {
public:
struct WebrtcIceCandidateStats {
uint32_t successCount;
uint32_t failureCount;
WebrtcIceCandidateStats() :
successCount(0),
failureCount(0)
{
}
};
struct WebrtcIceStatsCategory {
struct WebrtcIceCandidateStats webrtc;
};
typedef nsBaseHashtableET<nsUint32HashKey, WebrtcIceStatsCategory> WebrtcIceCandidateType;
void RecordIceCandidateMask(const uint32_t iceCandidateBitmask, bool success);
bool GetWebrtcStats(JSContext *cx, JS::MutableHandle<JS::Value> ret);
size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) const;
private:
bool AddIceInfo(JSContext *cx, JS::Handle<JSObject*> rootObj);
mozilla::Telemetry::Common::AutoHashtable<WebrtcIceCandidateType> mWebrtcIceCandidates;
};
#endif // WebrtcTelemetry_h__

12
toolkit/components/telemetry/datareporting-prefs.js
View file

@ -1,12 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
pref("datareporting.policy.dataSubmissionEnabled", true);
pref("datareporting.policy.dataSubmissionPolicyNotifiedTime", "0");
pref("datareporting.policy.dataSubmissionPolicyAcceptedVersion", 0);
pref("datareporting.policy.dataSubmissionPolicyBypassNotification", false);
pref("datareporting.policy.currentPolicyVersion", 2);
pref("datareporting.policy.minimumPolicyVersion", 1);
pref("datareporting.policy.minimumPolicyVersion.channel-beta", 2);
pref("datareporting.policy.firstRunURL", "");

74
toolkit/components/telemetry/docs/collection/custom-pings.rst
View file

@ -1,74 +0,0 @@
=======================
Submitting custom pings
=======================
Custom pings can be submitted from JavaScript using:
.. code-block:: js
TelemetryController.submitExternalPing(type, payload, options)
- ``type`` - a ``string`` that is the type of the ping, limited to ``/^[a-z0-9][a-z0-9-]+[a-z0-9]$/i``.
- ``payload`` - the actual payload data for the ping, has to be a JSON style object.
- ``options`` - optional, an object containing additional options:
- ``addClientId``- whether to add the client id to the ping, defaults to ``false``
- ``addEnvironment`` - whether to add the environment data to the ping, defaults to ``false``
- ``overrideEnvironment`` - a JSON style object that overrides the environment data
``TelemetryController`` will assemble a ping with the passed payload and the specified options.
That ping will be archived locally for use with Shield and inspection in ``about:telemetry``.
If the preferences allow upload of Telemetry pings, the ping will be uploaded at the next opportunity (this is subject to throttling, retry-on-failure, etc.).
Submission constraints
----------------------
When submitting pings on shutdown, they should not be submitted after Telemetry shutdown.
Pings should be submitted at the latest within:
- the `observer notification <https://developer.mozilla.org/de/docs/Observer_Notifications#Application_shutdown>`_ ``"profile-before-change"``
- the :ref:`AsyncShutdown phase <AsyncShutdown_phases>` ``sendTelemetry``
There are other constraints that can lead to a ping submission getting dropped:
- invalid ping type strings
- invalid payload types: E.g. strings instead of objects.
- oversized payloads: We currently only drop pings >1MB, but targetting sizes of <=10KB is recommended.
Tools
=====
Helpful tools for designing new pings include:
- `gzipServer <https://github.com/mozilla/gzipServer>`_ - a Python script that can run locally and receives and saves Telemetry pings. Making Firefox send to it allows inspecting outgoing pings easily.
- ``about:telemetry`` - allows inspecting submitted pings from the local archive, including all custom ones.
Designing custom pings
======================
In general, creating a new custom ping means you don't benefit automatically from the existing tooling. Further work is needed to make data show up in re:dash or other analysis tools.
In addition to the `data collection review <https://wiki.mozilla.org/Firefox/Data_Collection>`_, questions to guide a new pings design are:
- Submission interval & triggers:
- What events trigger ping submission?
- What interval is the ping submitted in?
- Is there a throttling mechanism?
- What is the desired latency? (submitting "at least daily" still leads to certain latency tails)
- Are pings submitted on a clock schedule? Or based on "time since session start", "time since last ping" etc.? (I.e. will we get sharp spikes in submission volume?)
- Size and volume:
- What’s the size of the submitted payload?
- What's the full ping size including metadata in the pipeline?
- What’s the target population?
- What's the overall estimated volume?
- Dataset:
- Is it opt-out?
- Does it need to be opt-out?
- Does it need to be in a separate ping? (why can’t the data live in probes?)
- Privacy:
- Is there risk to leak PII?
- How is that risk mitigated?
- Data contents:
- Does the submitted data answer the posed product questions?
- Does the shape of the data allow to answer the questions efficiently?
- Is the data limited to whats needed to answer the questions?
- Does the data use common formats? (i.e. can we re-use tooling or analysis know-how)

5
toolkit/components/telemetry/docs/collection/histograms.rst
View file

@ -1,5 +0,0 @@
==========
Histograms
==========
Recording into histograms is currently documented in `a MDN article <https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Adding_a_new_Telemetry_probe>`_.

35
toolkit/components/telemetry/docs/collection/index.rst
View file

@ -1,35 +0,0 @@
===============
Data collection
===============
There are different APIs and formats to collect data in Firefox, all suiting different use cases.
In general, we aim to submit data in a common format where possible. This has several advantages; from common code and tooling to sharing analysis know-how.
In cases where this isn't possible and more flexibility is needed, we can submit custom pings or consider adding different data formats to existing pings.
*Note:* Every new data collection must go through a `data collection review <https://wiki.mozilla.org/Firefox/Data_Collection>`_.
The current data collection possibilities include:
* :doc:`scalars` allow recording of a single value (string, boolean, a number)
* :doc:`histograms` can efficiently record multiple data points
* ``environment`` data records information about the system and settings a session occurs in
* ``TelemetryLog`` allows collecting ordered event entries (note: this does not have supporting analysis tools)
* :doc:`measuring elapsed time <measuring-time>`
* :doc:`custom pings <custom-pings>`
.. toctree::
:maxdepth: 2
:titlesonly:
:hidden:
:glob:
scalars
histograms
measuring-time
custom-pings
Browser Usage Telemetry
~~~~~~~~~~~~~~~~~~~~~~~
For more information, see :ref:`browserusagetelemetry`.

74
toolkit/components/telemetry/docs/collection/measuring-time.rst
View file

@ -1,74 +0,0 @@
======================
Measuring elapsed time
======================
To make it easier to measure how long operations take, we have helpers for both JavaScript and C++.
These helpers record the elapsed time into histograms, so you have to create suitable histograms for them first.
From JavaScript
===============
JavaScript can measure elapsed time using `TelemetryStopwatch.jsm <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/TelemetryStopwatch.jsm>`_.
``TelemetryStopwatch`` is a helper that simplifies recording elapsed time (in milliseconds) into histograms (plain or keyed).
API:
.. code-block:: js
TelemetryStopwatch = {
// Start, cancel & finish recording elapsed time into a histogram.
// |aObject| is optional. If specificied, the timer is associated with this
// object, so multiple time measurements can be done concurrently.
start(histogramId, aObject);
cancel(histogramId, aObject);
finish(histogramId, aObject);
// Start, cancel & finished recording elapsed time into a keyed histogram.
// |key| specificies the key to record into.
// |aObject| is optional and used as above.
startKeyed(histogramId, key, aObject);
cancelKeyed(histogramId, key, aObject);
finishKeyed(histogramId, key, aObject);
};
Example:
.. code-block:: js
TelemetryStopwatch.start("SAMPLE_FILE_LOAD_TIME_MS");
// ... start loading file.
if (failedToOpenFile) {
// Cancel this if the operation failed early etc.
TelemetryStopwatch.cancel("SAMPLE_FILE_LOAD_TIME_MS");
return;
}
// ... do more work.
TelemetryStopwatch.finish("SAMPLE_FILE_LOAD_TIME_MS");
From C++
========
API:
.. code-block:: cpp
// This helper class is the preferred way to record elapsed time.
template<ID id, TimerResolution res = MilliSecond>
class AutoTimer {
// Record into a plain histogram.
explicit AutoTimer(TimeStamp aStart = TimeStamp::Now());
// Record into a keyed histogram, with key |aKey|.
explicit AutoTimer(const nsCString& aKey,
TimeStamp aStart = TimeStamp::Now());
};
void AccumulateTimeDelta(ID id, TimeStamp start, TimeStamp end = TimeStamp::Now());
Example:
.. code-block:: cpp
{
Telemetry::AutoTimer<Telemetry::FIND_PLUGINS> telemetry;
// ... scan disk for plugins.
}
// When leaving the scope, AutoTimers destructor will record the time that passed.

140
toolkit/components/telemetry/docs/collection/scalars.rst
View file

@ -1,140 +0,0 @@
=======
Scalars
=======
Historically we started to overload our histogram mechanism to also collect scalar data,
such as flag values, counts, labels and others.
The scalar measurement types are the suggested way to collect that kind of scalar data.
We currently only support recording of scalars from the parent process.
The serialized scalar data is submitted with the :doc:`main pings <../data/main-ping>`.
The API
=======
Scalar probes can be managed either through the `nsITelemetry interface <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/nsITelemetry.idl>`_
or the `C++ API <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/Telemetry.h>`_.
JS API
------
Probes in privileged JavaScript code can use the following functions to manipulate scalars:
.. code-block:: js
Services.telemetry.scalarAdd(aName, aValue);
Services.telemetry.scalarSet(aName, aValue);
Services.telemetry.scalarSetMaximum(aName, aValue);
Services.telemetry.keyedScalarAdd(aName, aKey, aValue);
Services.telemetry.keyedScalarSet(aName, aKey, aValue);
Services.telemetry.keyedScalarSetMaximum(aName, aKey, aValue);
These functions can throw if, for example, an operation is performed on a scalar type that doesn't support it
(e.g. calling scalarSetMaximum on a scalar of the string kind). Please look at the `code documentation <https://dxr.mozilla.org/mozilla-central/search?q=regexp%3ATelemetryScalar%3A%3A(Set%7CAdd)+file%3ATelemetryScalar.cpp&redirect=false>`_ for
additional information.
C++ API
-------
Probes in native code can use the more convenient helper functions declared in `Telemetry.h <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/Telemetry.h>`_:
.. code-block:: cpp
void ScalarAdd(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
void ScalarSet(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aValue);
void ScalarSet(mozilla::Telemetry::ScalarID aId, bool aValue);
void ScalarSetMaximum(mozilla::Telemetry::ScalarID aId, uint32_t aValue);
void ScalarAdd(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
void ScalarSet(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, bool aValue);
void ScalarSetMaximum(mozilla::Telemetry::ScalarID aId, const nsAString& aKey, uint32_t aValue);
The YAML definition file
========================
Scalar probes are required to be registered, both for validation and transparency reasons,
in the `Scalars.yaml <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/Scalars.yaml>`_
definition file.
The probes in the definition file are represented in a fixed-depth, two-level structure:
.. code-block:: yaml
# The following is a group.
a.group.hierarchy:
a_probe_name:
kind: uint
...
another_probe:
kind: string
...
...
group2:
probe:
kind: int
...
Group and probe names need to follow a few rules:
- they cannot exceed 40 characters each;
- group names must be alpha-numeric + ``.``, with no leading/trailing digit or ``.``;
- probe names must be alpha-numeric + ``_``, with no leading/trailing digit or ``_``.
A probe can be defined as follows:
.. code-block:: yaml
a.group.hierarchy:
a_scalar:
bug_numbers:
- 1276190
description: A nice one-line description.
expires: never
kind: uint
notification_emails:
- telemetry-client-dev@mozilla.com
Required Fields
---------------
- ``bug_numbers``: A list of unsigned integers representing the number of the bugs the probe was introduced in.
- ``description``: A single or multi-line string describing what data the probe collects and when it gets collected.
- ``expires``: The version number in which the scalar expires, e.g. "30"; a version number of type "N" and "N.0" is automatically converted to "N.0a1" in order to expire the scalar also in the development channels. A telemetry probe acting on an expired scalar will print a warning into the browser console. For scalars that never expire the value ``never`` can be used.
- ``kind``: A string representing the scalar type. Allowed values are ``uint``, ``string`` and ``boolean``.
- ``notification_emails``: A list of email addresses to notify with alerts of expiring probes. More importantly, these are used by the data steward to verify that the probe is still useful.
Optional Fields
---------------
- ``cpp_guard``: A string that gets inserted as an ``#ifdef`` directive around the automatically generated C++ declaration. This is typically used for platform-specific scalars, e.g. ``ANDROID``.
- ``release_channel_collection``: This can be either ``opt-in`` (default) or ``opt-out``. With the former the scalar is submitted by default on pre-release channels; on the release channel only if the user opted into additional data collection. With the latter the scalar is submitted by default on release and pre-release channels, unless the user opted out.
- ``keyed``: A boolean that determines whether this is a keyed scalar. It defaults to ``False``.
String type restrictions
------------------------
To prevent abuses, the content of a string scalar is limited to 50 characters in length. Trying
to set a longer string will result in an error and no string being set.
Keyed Scalars
-------------
Keyed scalars are collections of one of the available scalar types, indexed by a string key that can contain UTF8 characters and cannot be longer than 70 characters. Keyed scalars can contain up to 100 keys. This scalar type is for example useful when you want to break down certain counts by a name, like how often searches happen with which search engine.
Keyed scalars should only be used if the set of keys are not known beforehand. If the keys are from a known set of strings, other options are preferred if suitable, like categorical histograms or splitting measurements up into separate scalars.
The processor scripts
=====================
The scalar definition file is processed and checked for correctness at compile time. If it
conforms to the specification, the processor scripts generate two C++ headers files, included
by the Telemetry C++ core.
gen-scalar-data.py
------------------
This script is called by the build system to generate the ``TelemetryScalarData.h`` C++ header
file out of the scalar definitions.
This header file contains an array holding the scalar names and version strings, in addition
to an array of ``ScalarInfo`` structures representing all the scalars.
gen-scalar-enum.py
------------------
This script is called by the build system to generate the ``TelemetryScalarEnums.h`` C++ header
file out of the scalar definitions.
This header file contains an enum class with all the scalar identifiers used to access them
from code through the C++ API.

12
toolkit/components/telemetry/docs/concepts/archiving.rst
View file

@ -1,12 +0,0 @@
=========
Archiving
=========
When archiving is enabled through the relevant pref (``toolkit.telemetry.archive.enabled``), pings submitted to ``TelemetryController`` are also stored locally in the user profile directory, in ``<profile-dir>/datareporting/archived``.
To allow for cheaper lookup of archived pings, storage follows a specific naming scheme for both the directory and the ping file name: `<YYYY-MM>/<timestamp>.<UUID>.<type>.jsonlz4`.
* ``<YYYY-MM>`` - The subdirectory name, generated from the ping creation date.
* ``<timestamp>`` - Timestamp of the ping creation date.
* ``<UUID>`` - The ping identifier.
* ``<type>`` - The ping type.

23
toolkit/components/telemetry/docs/concepts/crashes.rst
View file

@ -1,23 +0,0 @@
=======
Crashes
=======
There are many different kinds of crashes for Firefox, there is not a single system used to record all of them.
Main process crashes
====================
If the Firefox main process dies, that should be recorded as an aborted session. We would submit a :doc:`main ping <../data/main-ping>` with the reason ``aborted-session``.
If we have a crash dump for that crash, we should also submit a :doc:`crash ping <../data/crash-ping>`.
The ``aborted-session`` information is first written to disk 60 seconds after startup, any earlier crashes will not trigger an ``aborted-session`` ping.
Also, the ``aborted-session`` is updated at least every 5 minutes, so it may lag behind the last session state.
Crashes during startup should be recorded in the next sessions main ping in the ``STARTUP_CRASH_DETECTED`` histogram.
Child process crashes
=====================
If a Firefox plugin, content or gmplugin process dies unexpectedly, this is recorded in the main pings ``SUBPROCESS_ABNORMAL_ABORT`` keyed histogram.
If we catch a crash report for this, then additionally the ``SUBPROCESS_CRASHES_WITH_DUMP`` keyed histogram is incremented.

23
toolkit/components/telemetry/docs/concepts/index.rst
View file

@ -1,23 +0,0 @@
========
Concepts
========
There are common concepts used throughout Telemetry:
* :doc:`pings <pings>` - the packets we use to submit data
* :doc:`sessions & subsessions <sessions>` - how we slice a users' time in the browser
* *measurements* - how we :doc:`collect data <../collection/index>`
* *opt-in* & *opt-out* - the different sets of data we collect
* :doc:`submission <submission>` - how we send data to the servers
* :doc:`archiving <archiving>` - retaining ping data locally
* :doc:`crashes <crashes>` - the different data crashes generate
.. toctree::
:maxdepth: 2
:titlesonly:
:glob:
:hidden:
pings
crashes
*

32
toolkit/components/telemetry/docs/concepts/pings.rst
View file

@ -1,32 +0,0 @@
.. _telemetry_pings:
=====================
Telemetry pings
=====================
A *Telemetry ping* is the data that we send to Mozillas Telemetry servers.
That data is stored as a JSON object client-side and contains common information to all pings and a payload specific to a certain *ping types*.
The top-level structure is defined by the :doc:`common ping format <../data/common-ping>` format.
It contains:
* some basic information shared between different ping types
* the :doc:`environment data <../data/environment>` (optional)
* the data specific to the *ping type*, the *payload*.
Ping types
==========
We send Telemetry with different ping types. The :doc:`main <../data/main-ping>` ping is the ping that contains the bulk of the Telemetry measurements for Firefox. For more specific use-cases, we send other ping types.
Pings sent from code that ships with Firefox are listed in the :doc:`data documentation <../data/index>`.
Important examples are:
* :doc:`main <../data/main-ping>` - contains the information collected by Telemetry (Histograms, hang stacks, ...)
* :doc:`saved-session <../data/main-ping>` - has the same format as a main ping, but it contains the *"classic"* Telemetry payload with measurements covering the whole browser session. This is only a separate type to make storage of saved-session easier server-side. This is temporary and will be removed soon.
* :doc:`crash <../data/crash-ping>` - a ping that is captured and sent after Firefox crashes.
* ``activation`` - *planned* - sent right after installation or profile creation
* ``upgrade`` - *planned* - sent right after an upgrade
* :doc:`deletion <../data/deletion-ping>` - sent when FHR upload is disabled, requesting deletion of the data associated with this user

40
toolkit/components/telemetry/docs/concepts/sessions.rst
View file

@ -1,40 +0,0 @@
========
Sessions
========
A *session* is the time from when Firefox starts until it shut down.
A session can be very long-running. E.g. for Mac users that are used to always put their laptops into sleep-mode, Firefox may run for weeks.
We slice the sessions into smaller logical units called *subsessions*.
Subsessions
===========
The first subsession starts when the browser starts. After that, we split the subsession for different reasons:
* ``daily``, when crossing local midnight. This keeps latency acceptable by triggering a ping at least daily for most active users.
* ``environment-change``, when a change to the *environment* happens. This happens for important changes to the Firefox settings and when addons activate or deactivate.
On a subsession split, a :doc:`main ping <../data/main-ping>` with that reason will be submitted. We store the reason in the pings payload, to see what triggered it.
A session always ends with a subsession with one of two reason:
* ``shutdown``, when the browser was cleanly shut down. To avoid delaying shutdown, we only save this ping to disk and send it at the next opportunity (typically the next browsing session).
* ``aborted-session``, when the browser crashed. While Firefox is active, we write the current ``main`` ping data to disk every 5 minutes. If the browser crashes, we find this data on disk on the next start and send it with this reason.
.. image:: subsession_triggers.png
Subsession data
===============
A subsessions data consists of:
* general information: the date the subsession started, how long it lasted, etc.
* specific measurements: histogram & scalar data, etc.
This has some advantages:
* Latency - Sending a ping with all the data of a subsession immediately after it ends means we get the data from installs faster. For ``main`` pings, we aim to send a ping at least daily by starting a new subsession at local midnight.
* Correlation - By starting new subsessions when fundamental settings change (i.e. changes to the *environment*), we can correlate a subsessions data better to those settings.

34
toolkit/components/telemetry/docs/concepts/submission.rst
View file

@ -1,34 +0,0 @@
==========
Submission
==========
*Note:* The server-side behaviour is documented in the `HTTP Edge Server specification <https://wiki.mozilla.org/CloudServices/DataPipeline/HTTPEdgeServerSpecification>`_.
Pings are submitted via a common API on ``TelemetryController``.
If a ping fails to successfully submit to the server immediately (e.g. because
of missing internet connection), Telemetry will store it on disk and retry to
send it until the maximum ping age is exceeded (14 days).
*Note:* the :doc:`main pings <../data/main-ping>` are kept locally even after successful submission to enable the HealthReport and SelfSupport features. They will be deleted after their retention period of 180 days.
Submission logic
================
Sending of pending pings starts as soon as the delayed startup is finished. They are sent in batches, newest-first, with up
to 10 persisted pings per batch plus all unpersisted pings.
The send logic then waits for each batch to complete.
If it succeeds we trigger the next send of a ping batch. This is delayed as needed to only trigger one batch send per minute.
If ping sending encounters an error that means retrying later, a backoff timeout behavior is
triggered, exponentially increasing the timeout for the next try from 1 minute up to a limit of 120 minutes.
Any new ping submissions and "idle-daily" events reset this behavior as a safety mechanism and trigger immediate ping sending.
Status codes
============
The telemetry server team is working towards `the common services status codes <https://wiki.mozilla.org/CloudServices/DataPipeline/HTTPEdgeServerSpecification#Server_Responses>`_, but for now the following logic is sufficient for Telemetry:
* `2XX` - success, don't resubmit
* `4XX` - there was some problem with the request - the client should not try to resubmit as it would just receive the same response
* `5XX` - there was a server-side error, the client should try to resubmit later

BIN
toolkit/components/telemetry/docs/concepts/subsession_triggers.png
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 1.2 MiB

42
toolkit/components/telemetry/docs/data/addons-malware-ping.rst
View file

@ -1,42 +0,0 @@
Add-ons malware ping
====================
This ping is generated by an add-on created by Mozilla and shipped to users on older versions of Firefox (44-46). The ping contains information about the profile that might have been altered by a third party malicious add-on.
Structure:
.. code-block:: js
{
type: "malware-addon-states",
...
clientId: <UUID>,
environment: { ... },
// Common ping data.
payload: {
// True if the blocklist was disabled at startup time.
blocklistDisabled: <bool>,
// True if the malicious add-on exists and is enabled. False if it
// exists and is disabled or null if the add-on was not found.
mainAddonActive: <bool | null>,
// A value of the malicious add-on block list state, or null if the
// add-on was not found.
mainAddonBlocked: <int | null>,
// True if a malicious user.js file was found in the profile.
foundUserJS: <bool>,
// If a malicious secmodd.db file was found the extension ID that the // file contained..
secmoddAddon: <string | null>, .
// A list of IDs for extensions which were hidden by malicious CSS.
hiddenAddons: [
<string>,
...
],
// A mapping of installed add-on IDs with known malicious
// update URL patterns to their exact update URLs.
updateURLs: {
<extensionID>: <updateURL>,
...
}
}
}

42
toolkit/components/telemetry/docs/data/common-ping.rst
View file

@ -1,42 +0,0 @@
Common ping format
==================
This defines the top-level structure of a Telemetry ping.
It contains basic information shared between different ping types, which enables proper storage and processing of the raw pings server-side.
It also contains optional further information:
* the :doc:`environment data <../data/environment>`, which contains important info to correlate the measurements against
* the ``clientId``, a UUID identifying a profile and allowing user-oriented correlation of data
*Note:* Both are not submitted with all ping types due to privacy concerns. This and the data it that can be correlated against is inspected under the `data collection policy <https://wiki.mozilla.org/Firefox/Data_Collection>`_.
Finally, the structure also contains the `payload`, which is the specific data submitted for the respective *ping type*.
Structure:
.. code-block:: js
{
type: <string>, // "main", "activation", "deletion", "saved-session", ...
id: <UUID>, // a UUID that identifies this ping
creationDate: <ISO date>, // the date the ping was generated
version: <number>, // the version of the ping format, currently 4
application: {
architecture: <string>, // build architecture, e.g. x86
buildId: <string>, // "20141126041045"
name: <string>, // "Firefox"
version: <string>, // "35.0"
displayVersion: <string>, // "35.0b3"
vendor: <string>, // "Mozilla"
platformVersion: <string>, // "35.0"
xpcomAbi: <string>, // e.g. "x86-msvc"
channel: <string>, // "beta"
},
clientId: <UUID>, // optional
environment: { ... }, // optional, not all pings contain the environment
payload: { ... }, // the actual payload data for this ping type
}

191
toolkit/components/telemetry/docs/data/core-ping.rst
View file

@ -1,191 +0,0 @@
"core" ping
============
This mobile-specific ping is intended to provide the most critical
data in a concise format, allowing for frequent uploads.
Since this ping is used to measure retention, it should be sent
each time the browser is opened.
Submission will be per the Edge server specification::
/submit/telemetry/docId/docType/appName/appVersion/appUpdateChannel/appBuildID
* ``docId`` is a UUID for deduping
* ``docType`` is “core”
* ``appName`` is “Fennec”
* ``appVersion`` is the version of the application (e.g. "46.0a1")
* ``appUpdateChannel`` is “release”, “beta”, etc.
* ``appBuildID`` is the build number
Note: Counts below (e.g. search & usage times) are “since the last
ping”, not total for the whole application lifetime.
Structure:
.. code-block:: js
{
"v": 7, // ping format version
"clientId": <string>, // client id, e.g.
// "c641eacf-c30c-4171-b403-f077724e848a"
"seq": <positive integer>, // running ping counter, e.g. 3
"locale": <string>, // application locale, e.g. "en-US"
"os": <string>, // OS name.
"osversion": <string>, // OS version.
"device": <string>, // Build.MANUFACTURER + " - " + Build.MODEL
// where manufacturer is truncated to 12 characters
// & model is truncated to 19 characters
"arch": <string>, // e.g. "arm", "x86"
"profileDate": <pos integer>, // Profile creation date in days since
// UNIX epoch.
"defaultSearch": <string>, // Identifier of the default search engine,
// e.g. "yahoo".
"distributionId": <string>, // Distribution identifier (optional)
"created": <string>, // date the ping was created
// in local time, "yyyy-mm-dd"
"tz": <integer>, // timezone offset (in minutes) of the
// device when the ping was created
"sessions": <integer>, // number of sessions since last upload
"durations": <integer>, // combined duration, in seconds, of all
// sessions since last upload
"searches": <object>, // Optional, object of search use counts in the
// format: { "engine.source": <pos integer> }
// e.g.: { "yahoo.suggestion": 3, "other.listitem": 1 }
"experiments": [<string>, /* … */], // Optional, array of identifiers
// for the active experiments
}
Field details
-------------
device
~~~~~~
The ``device`` field is filled in with information specified by the hardware
manufacturer. As such, it could be excessively long and use excessive amounts
of limited user data. To avoid this, we limit the length of the field. We're
more likely have collisions for models within a manufacturer (e.g. "Galaxy S5"
vs. "Galaxy Note") than we are for shortened manufacturer names so we provide
more characters for the model than the manufacturer.
distributionId
~~~~~~~~~~~~~~
The ``distributionId`` contains the distribution ID as specified by
preferences.json for a given distribution. More information on distributions
can be found `here <https://wiki.mozilla.org/Mobile/Distribution_Files>`_.
It is optional.
defaultSearch
~~~~~~~~~~~~~
On Android, this field may be ``null``. To get the engine, we rely on
``SearchEngineManager#getDefaultEngine``, which searches in several places in
order to find the search engine identifier:
* Shared Preferences
* The distribution (if it exists)
* The localized default engine
If the identifier could not be retrieved, this field is ``null``. If the
identifier is retrieved, we attempt to create an instance of the search
engine from the search plugins (in order):
* In the distribution
* From the localized plugins shipped with the browser
* The third-party plugins that are installed in the profile directory
If the plugins fail to create a search engine instance, this field is also
``null``.
This field can also be ``null`` when a custom search engine is set as the
default.
sessions & durations
~~~~~~~~~~~~~~~~~~~~
On Android, a session is the time when Firefox is focused in the foreground.
`sessions` tracks the number of sessions since the last upload and
`durations` is the accumulated duration in seconds of all of these
sessions. Note that showing a dialog (including a Firefox dialog) will
take Firefox out of focus & end the current session.
An implementation that records a session when Firefox is completely hidden is
preferrable (e.g. to avoid the dialog issue above), however, it's more complex
to implement and so we chose not to, at least for the initial implementation.
profileDate
~~~~~~~~~~~
On Android, this value is created at profile creation time and retrieved or,
for legacy profiles, taken from the package install time (note: this is not the
same exact metric as profile creation time but we compromised in favor of ease
of implementation).
Additionally on Android, this field may be ``null`` in the unlikely event that
all of the following events occur:
#. The times.json file does not exist
#. The package install date could not be persisted to disk
The reason we don't just return the package install time even if the date could
not be persisted to disk is to ensure the value doesn't change once we start
sending it: we only want to send consistent values.
searches
~~~~~~~~
In the case a search engine is added by a user, the engine identifier "other" is used, e.g. "other.<source>".
Sources in Android are based on the existing UI telemetry values and are as
follows:
* actionbar: the user types in the url bar and hits enter to use the default
search engine
* listitem: the user selects a search engine from the list of secondary search
engines at the bottom of the screen
* suggestion: the user clicks on a search suggestion or, in the case that
suggestions are disabled, the row corresponding with the main engine
Other parameters
----------------
HTTP "Date" header
~~~~~~~~~~~~~~~~~~
This header is used to track the submission date of the core ping in the format
specified by
`rfc 2616 sec 14.18 <https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18>`_,
et al (e.g. "Tue, 01 Feb 2011 14:00:00 GMT").
Version history
---------------
* v7: added ``sessionCount`` & ``sessionDuration``
* v6: added ``searches``
* v5: added ``created`` & ``tz``
* v4: ``profileDate`` will return package install time when times.json is not available
* v3: added ``defaultSearch``
* v2: added ``distributionId``
* v1: initial version
Notes
~~~~~
* ``distributionId`` (v2) actually landed after ``profileDate`` (v4) but was
uplifted to 46, whereas ``profileDate`` landed on 47. The version numbers in
code were updated to be increasing (bug 1264492) and the version history docs
rearranged accordingly.
Android implementation notes
----------------------------
On Android, the uploader has a high probability of delivering the complete data
for a given client but not a 100% probability. This was a conscious decision to
keep the code simple. The cases where we can lose data:
* Resetting the field measurements (including incrementing the sequence number)
and storing a ping for upload are not atomic. Android can kill our process
for memory pressure in between these distinct operations so we can just lose
a ping's worth of data. That sequence number will be missing on the server.
* If we exceed some number of pings on disk that have not yet been uploaded,
we remove old pings to save storage space. For those pings, we will lose
their data and their sequence numbers will be missing on the server.
Note: we never expect to drop data without also dropping a sequence number so
we are able to determine when data loss occurs.

144
toolkit/components/telemetry/docs/data/crash-ping.rst
View file

@ -1,144 +0,0 @@
"crash" ping
============
This ping is captured after the main Firefox process crashes, whether or not the crash report is submitted to crash-stats.mozilla.org. It includes non-identifying metadata about the crash.
The environment block that is sent with this ping varies: if Firefox was running long enough to record the environment block before the crash, then the environment at the time of the crash will be recorded and ``hasCrashEnvironment`` will be true. If Firefox crashed before the environment was recorded, ``hasCrashEnvironment`` will be false and the recorded environment will be the environment at time of submission.
The client ID is submitted with this ping.
Structure:
.. code-block:: js
{
version: 1,
type: "crash",
... common ping data
clientId: <UUID>,
environment: { ... },
payload: {
crashDate: "YYYY-MM-DD",
sessionId: <UUID>, // may be missing for crashes that happen early
// in startup. Added in Firefox 48 with the
// intention of uplifting to Firefox 46
crashId: <UUID>, // Optional, ID of the associated crash
stackTraces: { ... }, // Optional, see below
metadata: { // Annotations saved while Firefox was running. See nsExceptionHandler.cpp for more information
ProductName: "Firefox",
ReleaseChannel: <channel>,
Version: <version number>,
BuildID: "YYYYMMDDHHMMSS",
AvailablePageFile: <size>, // Windows-only, available paging file
AvailablePhysicalMemory: <size>, // Windows-only, available physical memory
AvailableVirtualMemory: <size>, // Windows-only, available virtual memory
BlockedDllList: <list>, // Windows-only, see WindowsDllBlocklist.cpp for details
BlocklistInitFailed: 1, // Windows-only, present only if the DLL blocklist initialization failed
CrashTime: <time>, // Seconds since the Epoch
ContainsMemoryReport: 1, // Optional
EventLoopNestingLevel: <levels>, // Optional, present only if >0
IsGarbageCollecting: 1, // Optional, present only if set to 1
MozCrashReason: <reason>, // Optional, contains the string passed to MOZ_CRASH()
OOMAllocationSize: <size>, // Size of the allocation that caused an OOM
SecondsSinceLastCrash: <duration>, // Seconds elapsed since the last crash occurred
SystemMemoryUsePercentage: <percentage>, // Windows-only, percent of memory in use
TelemetrySessionId: <id>, // Active telemetry session ID when the crash was recorded
TextureUsage: <usage>, // Optional, usage of texture memory in bytes
TotalPageFile: <size>, // Windows-only, paging file in use
TotalPhysicalMemory: <size>, // Windows-only, physical memory in use
TotalVirtualMemory: <size>, // Windows-only, virtual memory in use
UptimeTS: <duration>, // Seconds since Firefox was started
User32BeforeBlocklist: 1, // Windows-only, present only if user32.dll was loaded before the DLL blocklist has been initialized
},
hasCrashEnvironment: bool
}
}
Stack Traces
------------
The crash ping may contain a ``stackTraces`` field which has been populated
with stack traces for all threads in the crashed process. The format of this
field is similar to the one used by Socorro for representing a crash. The main
differences are that redundant fields are not stored and that the module a
frame belongs to is referenced by index in the module array rather than by its
file name.
Note that this field does not contain data from the application; only bare
stack traces and module lists are stored.
.. code-block:: js
{
status: <string>, // Status of the analysis, "OK" or an error message
crash_info: { // Basic crash information
type: <string>, // Type of crash, SIGSEGV, assertion, etc...
address: <addr>, // Crash address crash, hex format, see the notes below
crashing_thread: <index> // Index in the thread array below
},
main_module: <index>, // Index of Firefox' executable in the module list
modules: [{
base_addr: <addr>, // Base address of the module, hex format
end_addr: <addr>, // End address of the module, hex format
code_id: <string>, // Unique ID of this module, see the notes below
debug_file: <string>, // Name of the file holding the debug information
debug_id: <string>, // ID or hash of the debug information file
filename: <string>, // File name
version: <string>, // Library/executable version
},
... // List of modules ordered by base memory address
],
threads: [{ // Stack traces for every thread
frames: [{
module_index: <index>, // Index of the module this frame belongs to
ip: <ip>, // Program counter, hex format
trust: <string> // Trust of this frame, see the notes below
},
... // List of frames, the first frame is the topmost
]
}]
}
Notes
~~~~~
Memory addresses and instruction pointers are always stored as strings in
hexadecimal format (e.g. "0x4000"). They can be made of up to 16 characters for
64-bit addresses.
The crash type is both OS and CPU dependent and can be either a descriptive
string (e.g. SIGSEGV, EXCEPTION_ACCESS_VIOLATION) or a raw numeric value. The
crash address meaning depends on the type of crash. In a segmentation fault the
crash address will be the memory address whose access caused the fault; in a
crash triggered by an illegal instruction exception the address will be the
instruction pointer where the invalid instruction resides.
See `breakpad <https://chromium.googlesource.com/breakpad/breakpad/+/c99d374dde62654a024840accfb357b2851daea0/src/processor/minidump_processor.cc#675>`_'s
relevant code for further information.
Since it's not always possible to establish with certainty the address of the
previous frame while walking the stack, every frame has a trust value that
represents how it was found and thus how certain we are that it's a real frame.
The trust levels are (from least trusted to most trusted):
+---------------+---------------------------------------------------+
| Trust | Description |
+===============+===================================================+
| context | Given as instruction pointer in a context |
+---------------+---------------------------------------------------+
| prewalked | Explicitly provided by some external stack walker |
+---------------+---------------------------------------------------+
| cfi | Derived from call frame info |
+---------------+---------------------------------------------------+
| frame_pointer | Derived from frame pointer |
+---------------+---------------------------------------------------+
| cfi_scan | Found while scanning stack using call frame info |
+---------------+---------------------------------------------------+
| scan | Scanned the stack, found this |
+---------------+---------------------------------------------------+
| none | Unknown, this is most likely not a valid frame |
+---------------+---------------------------------------------------+
The ``code_id`` field holds a unique ID used to distinguish between different
versions and builds of the same module. See `breakpad <https://chromium.googlesource.com/breakpad/breakpad/+/24f5931c5e0120982c0cbf1896641e3ef2bdd52f/src/google_breakpad/processor/code_module.h#60>`_'s
description for further information. This field is populated only on Windows.

19
toolkit/components/telemetry/docs/data/deletion-ping.rst
View file

@ -1,19 +0,0 @@
"deletion" ping
===============
This ping is generated when a user turns off FHR upload from the Preferences panel, changing the related ``datareporting.healthreport.uploadEnabled`` preference. This requests that all associated data from that user be deleted.
This ping contains the client id and no environment data.
Structure:
.. code-block:: js
{
version: 4,
type: "deletion",
... common ping data
clientId: <UUID>,
payload: { }
}

372
toolkit/components/telemetry/docs/data/environment.rst
View file

@ -1,372 +0,0 @@
Environment
===========
The environment consists of data that is expected to be characteristic for performance and other behavior and not expected to change too often.
Changes to most of these data points are detected (where possible and sensible) and will lead to a session split in the :doc:`main-ping`.
The environment data may also be submitted by other ping types.
*Note:* This is not submitted with all ping types due to privacy concerns. This and other data is inspected under the `data collection policy <https://wiki.mozilla.org/Firefox/Data_Collection>`_.
Some parts of the environment must be fetched asynchronously at startup. We don't want other Telemetry components to block on waiting for the environment, so some items may be missing from it until the async fetching finished.
This currently affects the following sections:
- profile
- addons
Structure:
.. code-block:: js
{
build: {
applicationId: <string>, // nsIXULAppInfo.ID
applicationName: <string>, // "Firefox"
architecture: <string>, // e.g. "x86", build architecture for the active build
architecturesInBinary: <string>, // e.g. "i386-x86_64", from nsIMacUtils.architecturesInBinary, only present for mac universal builds
buildId: <string>, // e.g. "20141126041045"
version: <string>, // e.g. "35.0"
vendor: <string>, // e.g. "Mozilla"
platformVersion: <string>, // e.g. "35.0"
xpcomAbi: <string>, // e.g. "x86-msvc"
},
settings: {
addonCompatibilityCheckEnabled: <bool>, // Whether application compatibility is respected for add-ons
blocklistEnabled: <bool>, // true on failure
isDefaultBrowser: <bool>, // null on failure, not available on Android
defaultSearchEngine: <string>, // e.g. "yahoo"
defaultSearchEngineData: {, // data about the current default engine
name: <string>, // engine name, e.g. "Yahoo"; or "NONE" if no default
loadPath: <string>, // where the engine line is located; missing if no default
origin: <string>, // 'default', 'verified', 'unverified', or 'invalid'; based on the presence and validity of the engine's loadPath verification hash.
submissionURL: <string> // missing if no default or for user-installed engines
},
searchCohort: <string>, // optional, contains an identifier for any active search A/B experiments
e10sEnabled: <bool>, // whether e10s is on, i.e. browser tabs open by default in a different process
e10sCohort: <string>, // which e10s cohort was assigned for this user
telemetryEnabled: <bool>, // false on failure
locale: <string>, // e.g. "it", null on failure
update: {
channel: <string>, // e.g. "release", null on failure
enabled: <bool>, // true on failure
autoDownload: <bool>, // true on failure
},
userPrefs: {
// Only prefs which are changed from the default value are listed
// in this block
"pref.name.value": value // some prefs send the value
"pref.name.url": "<user-set>" // For some privacy-sensitive prefs
// only the fact that the value has been changed is recorded
},
attribution: { // optional, only present if the installation has attribution data
// all of these values are optional.
source: <string>, // referring partner domain, when install happens via a known partner
medium: <string>, // category of the source, such as "organic" for a search engine
campaign: <string>, // identifier of the particular campaign that led to the download of the product
content: <string>, // identifier to indicate the particular link within a campaign
},
},
profile: {
creationDate: <integer>, // integer days since UNIX epoch, e.g. 16446
resetDate: <integer>, // integer days since UNIX epoch, e.g. 16446 - optional
},
partner: { // This section may not be immediately available on startup
distributionId: <string>, // pref "distribution.id", null on failure
distributionVersion: <string>, // pref "distribution.version", null on failure
partnerId: <string>, // pref mozilla.partner.id, null on failure
distributor: <string>, // pref app.distributor, null on failure
distributorChannel: <string>, // pref app.distributor.channel, null on failure
partnerNames: [
// list from prefs app.partner.<name>=<name>
],
},
system: {
memoryMB: <number>,
virtualMaxMB: <number>, // windows-only
isWow64: <bool>, // windows-only
cpu: {
count: <number>, // desktop only, e.g. 8, or null on failure - logical cpus
cores: <number>, // desktop only, e.g., 4, or null on failure - physical cores
vendor: <string>, // desktop only, e.g. "GenuineIntel", or null on failure
family: <number>, // desktop only, null on failure
model: <number, // desktop only, null on failure
stepping: <number>, // desktop only, null on failure
l2cacheKB: <number>, // L2 cache size in KB, only on windows & mac
l3cacheKB: <number>, // desktop only, L3 cache size in KB
speedMHz: <number>, // desktop only, cpu clock speed in MHz
extensions: [
<string>,
...
// as applicable:
// "MMX", "SSE", "SSE2", "SSE3", "SSSE3", "SSE4A", "SSE4_1",
// "SSE4_2", "AVX", "AVX2", "EDSP", "ARMv6", "ARMv7", "NEON"
],
},
device: { // This section is only available on mobile devices.
model: <string>, // the "device" from FHR, null on failure
manufacturer: <string>, // null on failure
hardware: <string>, // null on failure
isTablet: <bool>, // null on failure
},
os: {
name: <string>, // "Windows_NT" or null on failure
version: <string>, // e.g. "6.1", null on failure
kernelVersion: <string>, // android/b2g only or null on failure
servicePackMajor: <number>, // windows only or null on failure
servicePackMinor: <number>, // windows only or null on failure
windowsBuildNumber: <number>, // windows 10 only or null on failure
windowsUBR: <number>, // windows 10 only or null on failure
installYear: <number>, // windows only or null on failure
locale: <string>, // "en" or null on failure
},
hdd: {
profile: { // hdd where the profile folder is located
model: <string>, // windows only or null on failure
revision: <string>, // windows only or null on failure
},
binary: { // hdd where the application binary is located
model: <string>, // windows only or null on failure
revision: <string>, // windows only or null on failure
},
system: { // hdd where the system files are located
model: <string>, // windows only or null on failure
revision: <string>, // windows only or null on failure
},
},
gfx: {
D2DEnabled: <bool>, // null on failure
DWriteEnabled: <bool>, // null on failure
//DWriteVersion: <string>, // temporarily removed, pending bug 1154500
adapters: [
{
description: <string>, // e.g. "Intel(R) HD Graphics 4600", null on failure
vendorID: <string>, // null on failure
deviceID: <string>, // null on failure
subsysID: <string>, // null on failure
RAM: <number>, // in MB, null on failure
driver: <string>, // null on failure
driverVersion: <string>, // null on failure
driverDate: <string>, // null on failure
GPUActive: <bool>, // currently always true for the first adapter
},
...
],
// Note: currently only added on Desktop. On Linux, only a single
// monitor is returned representing the entire virtual screen.
monitors: [
{
screenWidth: <number>, // screen width in pixels
screenHeight: <number>, // screen height in pixels
refreshRate: <number>, // refresh rate in hertz (present on Windows only).
// (values <= 1 indicate an unknown value)
pseudoDisplay: <bool>, // networked screen (present on Windows only)
scale: <number>, // backing scale factor (present on Mac only)
},
...
],
features: {
compositor: <string>, // Layers backend for compositing (eg "d3d11", "none", "opengl")
// Each the following features can have one of the following statuses:
// "unused" - This feature has not been requested.
// "unavailable" - Safe Mode or OS restriction prevents use.
// "blocked" - Blocked due to an internal condition such as safe mode.
// "blacklisted" - Blocked due to a blacklist restriction.
// "disabled" - User explicitly disabled this default feature.
// "failed" - This feature was attempted but failed to initialize.
// "available" - User has this feature available.
"d3d11" { // This feature is Windows-only.
status: <string>,
warp: <bool>, // Software rendering (WARP) mode was chosen.
textureSharing: <bool> // Whether or not texture sharing works.
version: <number>, // The D3D11 device feature level.
blacklisted: <bool>, // Whether D3D11 is blacklisted; use to see whether WARP
// was blacklist induced or driver-failure induced.
},
"d2d" { // This feature is Windows-only.
status: <string>,
version: <string>, // Either "1.0" or "1.1".
},
},
},
},
addons: {
activeAddons: { // the currently enabled addons
<addon id>: {
blocklisted: <bool>,
description: <string>, // null if not available
name: <string>,
userDisabled: <bool>,
appDisabled: <bool>,
version: <string>,
scope: <integer>,
type: <string>, // "extension", "service", ...
foreignInstall: <bool>,
hasBinaryComponents: <bool>
installDay: <number>, // days since UNIX epoch, 0 on failure
updateDay: <number>, // days since UNIX epoch, 0 on failure
signedState: <integer>, // whether the add-on is signed by AMO, only present for extensions
isSystem: <bool>, // true if this is a System Add-on
},
...
},
theme: { // the active theme
id: <string>,
blocklisted: <bool>,
description: <string>,
name: <string>,
userDisabled: <bool>,
appDisabled: <bool>,
version: <string>,
scope: <integer>,
foreignInstall: <bool>,
hasBinaryComponents: <bool>
installDay: <number>, // days since UNIX epoch, 0 on failure
updateDay: <number>, // days since UNIX epoch, 0 on failure
},
activePlugins: [
{
name: <string>,
version: <string>,
description: <string>,
blocklisted: <bool>,
disabled: <bool>,
clicktoplay: <bool>,
mimeTypes: [<string>, ...],
updateDay: <number>, // days since UNIX epoch, 0 on failure
},
...
],
activeGMPlugins: {
<gmp id>: {
version: <string>,
userDisabled: <bool>,
applyBackgroundUpdates: <integer>,
},
...
},
activeExperiment: { // section is empty if there's no active experiment
id: <string>, // id
branch: <string>, // branch name
},
persona: <string>, // id of the current persona, null on GONK
},
}
build
-----
buildId
~~~~~~~
Firefox builds downloaded from mozilla.org use a 14-digit buildId. Builds included in other distributions may have a different format (e.g. only 10 digits).
Settings
--------
defaultSearchEngine
~~~~~~~~~~~~~~~~~~~
Note: Deprecated, use defaultSearchEngineData instead.
Contains the string identifier or name of the default search engine provider. This will not be present in environment data collected before the Search Service initialization.
The special value ``NONE`` could occur if there is no default search engine.
The special value ``UNDEFINED`` could occur if a default search engine exists but its identifier could not be determined.
This field's contents are ``Services.search.defaultEngine.identifier`` (if defined) or ``"other-"`` + ``Services.search.defaultEngine.name`` if not. In other words, search engines without an ``.identifier`` are prefixed with ``other-``.
defaultSearchEngineData
~~~~~~~~~~~~~~~~~~~~~~~
Contains data identifying the engine currently set as the default.
The object contains:
- a ``name`` property with the name of the engine, or ``NONE`` if no
engine is currently set as the default.
- a ``loadPath`` property: an anonymized path of the engine xml file, e.g.
jar:[app]/omni.ja!browser/engine.xml
(where 'browser' is the name of the chrome package, not a folder)
[profile]/searchplugins/engine.xml
[distribution]/searchplugins/common/engine.xml
[other]/engine.xml
- an ``origin`` property: the value will be ``default`` for engines that are built-in or from distribution partners, ``verified`` for user-installed engines with valid verification hashes, ``unverified`` for non-default engines without verification hash, and ``invalid`` for engines with broken verification hashes.
- a ``submissionURL`` property with the HTTP url we would use to search.
For privacy, we don't record this for user-installed engines.
``loadPath`` and ``submissionURL`` are not present if ``name`` is ``NONE``.
searchCohort
~~~~~~~~~~~~
If the user has been enrolled into a search default change experiment, this contains the string identifying the experiment the user is taking part in. Most user profiles will never be part of any search default change experiment, and will not send this value.
userPrefs
~~~~~~~~~
This object contains user preferences.
Each key in the object is the name of a preference. A key's value depends on the policy with which the preference was collected. There are two such policies, "value" and "state". For preferences collected under the "value" policy, the value will be the preference's value. For preferences collected under the "state" policy, the value will be an opaque marker signifying only that the preference has a user value. The "state" policy is therefore used when user privacy is a concern.
The following is a partial list of collected preferences.
- ``browser.search.suggest.enabled``: The "master switch" for search suggestions everywhere in Firefox (search bar, urlbar, etc.). Defaults to true.
- ``browser.urlbar.suggest.searches``: True if search suggestions are enabled in the urlbar. Defaults to false.
- ``browser.urlbar.userMadeSearchSuggestionsChoice``: True if the user has clicked Yes or No in the urlbar's opt-in notification. Defaults to false.
- ``browser.zoom.full``: True if zoom is enabled for both text and images, that is if "Zoom Text Only" is not enabled. Defaults to true. Collection of this preference has been enabled in Firefox 50 and will be disabled again in Firefox 53 (`Bug 979323 <https://bugzilla.mozilla.org/show_bug.cgi?id=979323>`_).
- ``security.sandbox.content.level``: The meanings of the values are OS dependent, but 0 means not sandboxed for all OS. Details of the meanings can be found in the `Firefox prefs file <http://hg.mozilla.org/mozilla-central/file/tip/browser/app/profile/firefox.js>`_.
attribution
~~~~~~~~~~~
This object contains the attribution data for the product installation.
Attribution data is used to link installations of Firefox with the source that the user arrived at the Firefox download page from. It would indicate, for instance, when a user executed a web search for Firefox and arrived at the download page from there, directly navigated to the site, clicked on a link from a particular social media campaign, etc.
The attribution data is included in some versions of the default Firefox installer for Windows (the "stub" installer) and stored as part of the installation. All platforms other than Windows and also Windows installations that did not use the stub installer do not have this data and will not include the ``attribution`` object.
partner
-------
If the user is using a partner repack, this contains information identifying the repack being used, otherwise "partnerNames" will be an empty array and other entries will be null. The information may be missing when the profile just becomes available. In Firefox for desktop, the information along with other customizations defined in distribution.ini are processed later in the startup phase, and will be fully applied when "distribution-customization-complete" notification is sent.
Distributions are most reliably identified by the ``distributionId`` field. Partner information can be found in the `partner repacks <https://github.com/mozilla-partners>`_ (`the old one <http://hg.mozilla.org/build/partner-repacks/>`_ is deprecated): it contains one private repository per partner.
Important values for ``distributionId`` include:
- "MozillaOnline" for the Mozilla China repack.
- "canonical", for the `Ubuntu Firefox repack <http://bazaar.launchpad.net/~mozillateam/firefox/firefox.trusty/view/head:/debian/distribution.ini>`_.
- "yandex", for the Firefox Build by Yandex.
system
------
os
~~
This object contains operating system information.
- ``name``: the name of the OS.
- ``version``: a string representing the OS version.
- ``kernelVersion``: an Android/B2G only string representing the kernel version.
- ``servicePackMajor``: the Windows only major version number for the installed service pack.
- ``servicePackMinor``: the Windows only minor version number for the installed service pack.
- ``windowsBuildNumber``: the Windows build number, only available for Windows >= 10.
- ``windowsUBR``: the Windows UBR number, only available for Windows >= 10. This value is incremented by Windows cumulative updates patches.
- ``installYear``: the Windows only integer representing the year the OS was installed.
- ``locale``: the string representing the OS locale.
addons
------
activeAddons
~~~~~~~~~~~~
Starting from Firefox 44, the length of the following string fields: ``name``, ``description`` and ``version`` is limited to 100 characters. The same limitation applies to the same fields in ``theme`` and ``activePlugins``.

63
toolkit/components/telemetry/docs/data/heartbeat-ping.rst
View file

@ -1,63 +0,0 @@
"heartbeat" ping
=================
This ping is submitted after a Firefox Heartbeat survey. Even if the user exits
the browser, closes the survey window, or ignores the survey, Heartbeat will
provide a ping to Telemetry for sending during the same session.
The payload contains the user's survey response (if any) as well as timestamps
of various Heartbeat events (survey shown, survey closed, link clicked, etc).
The ping will also report the "surveyId", "surveyVersion" and "testing"
Heartbeat survey parameters (if they are present in the survey config).
These "meta fields" will be repeated verbatim in the payload section.
The environment block and client ID are submitted with this ping.
Structure:
.. code-block:: js
{
type: "heartbeat",
version: 4,
clientId: <UUID>,
environment: { /* ... */ }
// ... common ping data
payload: {
version: 1,
flowId: <string>,
... timestamps below ...
offeredTS: <integer epoch timestamp>,
learnMoreTS: <integer epoch timestamp>,
votedTS: <integer epoch timestamp>,
engagedTS: <integer epoch timestamp>,
closedTS: <integer epoch timestamp>,
expiredTS: <integer epoch timestamp>,
windowClosedTS: <integer epoch timestamp>,
// ... user's rating below
score: <integer>,
// ... survey meta fields below
surveyId: <string>,
surveyVersion: <integer>,
testing: <boolean>
}
}
Notes:
* Pings will **NOT** have all possible timestamps, timestamps are only reported for events that actually occurred.
* Timestamp meanings:
* offeredTS: when the survey was shown to the user
* learnMoreTS: when the user clicked on the "Learn More" link
* votedTS: when the user voted
* engagedTS: when the user clicked on the survey-provided button (alternative to voting feature)
* closedTS: when the Heartbeat notification bar was closed
* expiredTS: indicates that the survey expired after 2 hours of no interaction (threshold regulated by "browser.uitour.surveyDuration" pref)
* windowClosedTS: the user closed the entire Firefox window containing the survey, thus ending the survey. This timestamp will also be reported when the survey is ended by the browser being shut down.
* The surveyId/surveyVersion fields identify a specific survey (like a "1040EZ" tax paper form). The flowID is a UUID that uniquely identifies a single user's interaction with the survey. Think of it as a session token.
* The self-support page cannot include additional data in this payload. Only the the 4 flowId/surveyId/surveyVersion/testing fields are under the self-support page's control.
See also: :doc:`common ping fields <common-ping>`

18
toolkit/components/telemetry/docs/data/index.rst
View file

@ -1,18 +0,0 @@
==================
Data documentation
==================
.. toctree::
:maxdepth: 2
:titlesonly:
:glob:
common-ping
environment
main-ping
deletion-ping
crash-ping
*-ping
addons-malware-ping
The `mozilla-pipeline-schemas repository <https://github.com/mozilla-services/mozilla-pipeline-schemas/>`_ contains schemas for some of the pings.

609
toolkit/components/telemetry/docs/data/main-ping.rst
View file

@ -1,609 +0,0 @@
"main" ping
===========
.. toctree::
:maxdepth: 2
This is the "main" Telemetry ping type, whose payload contains most of the measurements that are used to track the performance and health of Firefox in the wild.
It includes the histograms and other performance and diagnostic data.
This ping is triggered by different scenarios, which is documented by the ``reason`` field:
* ``aborted-session`` - this ping is regularly saved to disk (every 5 minutes), overwriting itself, and deleted at shutdown. If a previous aborted session ping is found at startup, it gets sent to the server. The first aborted-session ping is generated as soon as Telemetry starts
* ``environment-change`` - the :doc:`environment` changed, so the session measurements got reset and a new subsession starts
* ``shutdown`` - triggered when the browser session ends
* ``daily`` - a session split triggered in 24h hour intervals at local midnight. If an ``environment-change`` ping is generated by the time it should be sent, the daily ping is rescheduled for the next midnight
* ``saved-session`` - the *"classic"* Telemetry payload with measurements covering the whole browser session (only submitted for a transition period)
Most reasons lead to a session split, initiating a new *subsession*. We reset important measurements for those subsessions.
After a new subsession split, the ``internal-telemetry-after-subsession-split`` topic is notified to all the observers. *This is an internal topic and is only meant for internal Telemetry usage.*
*Note:* ``saved-session`` is sent with a different ping type (``saved-session``, not ``main``), but otherwise has the same format as discussed here.
Structure:
.. code-block:: js
{
version: 4,
info: {
reason: <string>, // what triggered this ping: "saved-session", "environment-change", "shutdown", ...
revision: <string>, // the Histograms.json revision
timezoneOffset: <integer>, // time-zone offset from UTC, in minutes, for the current locale
previousBuildId: <string>, // null if this is the first run, or the previous build ID is unknown
sessionId: <uuid>, // random session id, shared by subsessions
subsessionId: <uuid>, // random subsession id
previousSessionId: <uuid>, // session id of the previous session, null on first run.
previousSubsessionId: <uuid>, // subsession id of the previous subsession (even if it was in a different session),
// null on first run.
subsessionCounter: <unsigned integer>, // the running no. of this subsession since the start of the browser session
profileSubsessionCounter: <unsigned integer>, // the running no. of all subsessions for the whole profile life time
sessionStartDate: <ISO date>, // daily precision
subsessionStartDate: <ISO date>, // daily precision, ISO date in local time
sessionLength: <integer>, // the session length until now in seconds, monotonic
subsessionLength: <integer>, // the subsession length in seconds, monotonic
flashVersion: <string>, // obsolete, use ``environment.addons.activePlugins``
addons: <string>, // obsolete, use ``environment.addons``
},
processes: {...},
childPayloads: [...], // only present with e10s; reduced payloads from content processes, null on failure
simpleMeasurements: {...},
// The following properties may all be null if we fail to collect them.
histograms: {...},
keyedHistograms: {...},
chromeHangs: {...},
threadHangStats: [...],
log: [...],
webrtc: {...},
gc: {...},
fileIOReports: {...},
lateWrites: {...},
addonDetails: {...},
addonHistograms: {...},
UIMeasurements: [...],
slowSQL: {...},
slowSQLstartup: {...},
}
info
----
sessionLength
~~~~~~~~~~~~~
The length of the current session so far in seconds.
This uses a monotonic clock, so this may mismatch with other measurements that
are not monotonic like calculations based on ``Date.now()``.
If the monotonic clock failed, this will be ``-1``.
subsessionLength
~~~~~~~~~~~~~~~~
The length of this subsession in seconds.
This uses a monotonic clock, so this may mismatch with other measurements that are not monotonic (e.g. based on Date.now()).
If ``sessionLength`` is ``-1``, the monotonic clock is not working.
processes
---------
This section contains per-process data.
Structure:
.. code-block:: js
"processes" : {
... other processes ...
"parent": {
scalars: {...},
},
"content": {
histograms: {...},
keyedHistograms: {...},
},
}
histograms and keyedHistograms
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This section contains histograms and keyed histograms accumulated on content processes. Histograms recorded on a content child process have different character than parent histograms. For instance, ``GC_MS`` will be much different in ``processes.content`` as it has to contend with web content, whereas the instance in ``payload.histograms`` has only to contend with browser JS. Also, some histograms may be absent if never recorded on a content child process (``EVENTLOOP_UI_ACTIVITY`` is parent-process-only).
This format was adopted in Firefox 51 via bug 1218576.
scalars
~~~~~~~
This section contains the :doc:`../collection/scalars` that are valid for the current platform. Scalars are not created nor submitted if no data was added to them, and are only reported with subsession pings. Scalar data is only currently reported for the main process. Their type and format is described by the ``Scalars.yaml`` file. Its most recent version is available `here <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/Scalars.yaml>`_. The ``info.revision`` field indicates the revision of the file that describes the reported scalars.
childPayloads
-------------
The Telemetry payloads sent by child processes, recorded on child process shutdown (event ``content-child-shutdown`` observed). They are reduced session payloads, only available with e10s. Among some other things, they don't contain histograms, keyed histograms, addon details, addon histograms, or UI Telemetry.
Note: Child payloads are not collected and cleared with subsession splits, they are currently only meaningful when analysed from ``saved-session`` or ``main`` pings with ``reason`` set to ``shutdown``.
Note: Before Firefox 51 and bug 1218576, content process histograms and keyedHistograms were in the individual child payloads instead of being aggregated into ``processes.content``.
simpleMeasurements
------------------
This section contains a list of simple measurements, or counters. In addition to the ones highlighted below, Telemetry timestamps (see `here <https://dxr.mozilla.org/mozilla-central/search?q=%22TelemetryTimestamps.add%22&redirect=false&case=true>`_ and `here <https://dxr.mozilla.org/mozilla-central/search?q=%22recordTimestamp%22&redirect=false&case=true>`_) can be reported.
totalTime
~~~~~~~~~
A non-monotonic integer representing the number of seconds the session has been alive.
uptime
~~~~~~
A non-monotonic integer representing the number of minutes the session has been alive.
addonManager
~~~~~~~~~~~~
Only available in the extended set of measures, it contains a set of counters related to Addons. See `here <https://dxr.mozilla.org/mozilla-central/search?q=%22AddonManagerPrivate.recordSimpleMeasure%22&redirect=false&case=true>`_ for a list of recorded measures.
UITelemetry
~~~~~~~~~~~
Only available in the extended set of measures. For more see :ref:`uitelemetry`.
startupInterrupted
~~~~~~~~~~~~~~~~~~
A boolean set to true if startup was interrupted by an interactive prompt.
js
~~
This section contains a series of counters from the JavaScript engine.
Structure:
.. code-block:: js
"js" : {
"setProto": <unsigned integer>, // Number of times __proto__ is set
"customIter": <unsigned integer> // Number of times __iterator__ is used (i.e., is found for a for-in loop)
}
maximalNumberOfConcurrentThreads
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
An integer representing the highest number of threads encountered so far during the session.
startupSessionRestoreReadBytes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Windows-only integer representing the number of bytes read by the main process up until the session store has finished restoring the windows.
startupSessionRestoreWriteBytes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Windows-only integer representing the number of bytes written by the main process up until the session store has finished restoring the windows.
startupWindowVisibleReadBytes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Windows-only integer representing the number of bytes read by the main process up until after a XUL window is made visible.
startupWindowVisibleWriteBytes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Windows-only integer representing the number of bytes written by the main process up until after a XUL window is made visible.
debuggerAttached
~~~~~~~~~~~~~~~~
A boolean set to true if a debugger is attached to the main process.
shutdownDuration
~~~~~~~~~~~~~~~~
The time, in milliseconds, it took to complete the last shutdown.
failedProfileLockCount
~~~~~~~~~~~~~~~~~~~~~~
The number of times the system failed to lock the user profile.
savedPings
~~~~~~~~~~
Integer count of the number of pings that need to be sent.
activeTicks
~~~~~~~~~~~
Integer count of the number of five-second intervals ('ticks') the user was considered 'active' (sending UI events to the window). An extra event is fired immediately when the user becomes active after being inactive. This is for some mouse and gamepad events, and all touch, keyboard, wheel, and pointer events (see `EventStateManager.cpp <https://dxr.mozilla.org/mozilla-central/rev/e6463ae7eda2775bc84593bb4a0742940bb87379/dom/events/EventStateManager.cpp#549>`_).
This measure might be useful to give a trend of how much a user actually interacts with the browser when compared to overall session duration. It does not take into account whether or not the window has focus or is in the foreground. Just if it is receiving these interaction events.
Note that in ``main`` pings, this measure is reset on subsession splits, while in ``saved-session`` pings it covers the whole browser session.
pingsOverdue
~~~~~~~~~~~~
Integer count of pending pings that are overdue.
histograms
----------
This section contains the histograms that are valid for the current platform. ``Flag`` and ``count`` histograms are always created and submitted, with their default value being respectively ``false`` and ``0``. Other histogram types (`see here <https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Adding_a_new_Telemetry_probe#Choosing_a_Histogram_Type>`_) are not created nor submitted if no data was added to them. The type and format of the reported histograms is described by the ``Histograms.json`` file. Its most recent version is available `here <https://dxr.mozilla.org/mozilla-central/source/toolkit/components/telemetry/Histograms.json>`_. The ``info.revision`` field indicates the revision of the file that describes the reported histograms.
keyedHistograms
---------------
This section contains the keyed histograms available for the current platform.
As of Firefox 48, this section does not contain empty keyed histograms anymore.
threadHangStats
---------------
Contains the statistics about the hangs in main and background threads. Note that hangs in this section capture the [C++ pseudostack](https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Profiling_with_the_Built-in_Profiler#Native_stack_vs._Pseudo_stack) and an incomplete JS stack, which is not 100% precise.
To avoid submitting overly large payloads, some limits are applied:
* Identical, adjacent "(chrome script)" or "(content script)" stack entries are collapsed together. If a stack is reduced, the "(reduced stack)" frame marker is added as the oldest frame.
* The depth of the reported stacks is limited to 11 entries. This value represents the 99.9th percentile of the thread hangs stack depths reported by Telemetry.
Structure:
.. code-block:: js
"threadHangStats" : [
{
"name" : "Gecko",
"activity" : {...}, // a time histogram of all task run times
"hangs" : [
{
"stack" : [
"Startup::XRE_Main",
"Timer::Fire",
"(content script)",
"IPDL::PPluginScriptableObject::SendGetChildProperty",
... up to 11 frames ...
],
"nativeStack": [...], // optionally available
"histogram" : {...}, // the time histogram of the hang times
"annotations" : [
{
"pluginName" : "Shockwave Flash",
"pluginVersion" : "18.0.0.209"
},
... other annotations ...
]
},
],
},
... other threads ...
]
chromeHangs
-----------
Contains the statistics about the hangs happening exclusively on the main thread of the parent process. Precise C++ stacks are reported. This is only available on Nightly Release on Windows, when building using "--enable-profiling" switch.
Some limits are applied:
* Reported chrome hang stacks are limited in depth to 50 entries.
* The maximum number of reported stacks is 50.
Structure:
.. code-block:: js
"chromeHangs" : {
"memoryMap" : [
["wgdi32.pdb", "08A541B5942242BDB4AEABD8C87E4CFF2"],
["igd10iumd32.pdb", "D36DEBF2E78149B5BE1856B772F1C3991"],
... other entries in the format ["module name", "breakpad identifier"] ...
],
"stacks" : [
[
[
0, // the module index or -1 for invalid module indices
190649 // the offset of this program counter in its module or an absolute pc
],
[1, 2540075],
... other frames, up to 50 ...
],
... other stacks, up to 50 ...
],
"durations" : [8, ...], // the hang durations (in seconds)
"systemUptime" : [692, ...], // the system uptime (in minutes) at the time of the hang
"firefoxUptime" : [672, ...], // the Firefox uptime (in minutes) at the time of the hang
"annotations" : [
[
[0, ...], // the indices of the related hangs
{
"pluginName" : "Shockwave Flash",
"pluginVersion" : "18.0.0.209",
... other annotations as key:value pairs ...
}
],
...
]
},
log
---
This section contains a log of important or unusual events reported through Telemetry.
Structure:
.. code-block:: js
"log": [
[
"Event_ID",
3785, // the timestamp (in milliseconds) for the log entry
... other data ...
],
...
]
webrtc
------
Contains special statistics gathered by WebRTC related components.
So far only a bitmask for the ICE candidate type present in a successful or
failed WebRTC connection is getting reported through C++ code as
IceCandidatesStats, because the required bitmask is too big to be represented
in a regular enum histogram. Further this data differentiates between Loop
(aka Firefox Hello) connections and everything else, which is categorized as
WebRTC.
Note: in most cases the webrtc and loop dictionaries inside of
IceCandidatesStats will simply be empty as the user has not used any WebRTC
PeerConnection at all during the ping report time.
Structure:
.. code-block:: js
"webrtc": {
"IceCandidatesStats": {
"webrtc": {
"34526345": {
"successCount": 5
},
"2354353": {
"failureCount": 1
}
},
"loop": {
"2349346359": {
"successCount": 3
},
"73424": {
"successCount": 1,
"failureCount": 5
}
}
}
},
gc
--
Contains statistics about selected garbage collections. To avoid
bloating the ping, only a few GCs are included. There are two
selection strategies. We always save the two GCs with the worst
max_pause time. Additionally, in content processes, two collections
are selected at random. If a GC runs for C milliseconds and the total
time for all GCs since the session began is T milliseconds, then the
GC has a C/T probablility of being selected for one of these "slots".
Structure:
.. code-block:: js
"gc": {
"random": [
{
// Timestamps are in milliseconds since startup. All the times here
// are wall-clock times, which may not be monotonically increasing.
"timestamp": 294872.2,
// All durations are in milliseconds.
"max_pause": 73.629,
"total_time": 364.951, // Sum of all slice times.
"zones_collected": 9,
"total_zones": 9,
"total_compartments": 309,
"minor_gcs": 44,
"store_buffer_overflows": 19,
"mmu_20ms": 0,
"mmu_50ms": 0,
// Reasons include "None", "NonIncrementalRequested",
// "AbortRequested", "KeepAtomsSet", "IncrementalDisabled",
// "ModeChange", "MallocBytesTrigger", "GCBytesTrigger",
// "ZoneChange".
"nonincremental_reason": "None",
"allocated": 37, // In megabytes.
"added_chunks": 54,
"removed_chunks": 12,
// Total number of slices (some of which may not appear
// in the "slices" array).
"num_slices": 15,
// We record at most 4 slices.
"slices": [
{
"slice": 0, // The index of this slice.
"pause": 23.221, // How long the slice took.
"when": 0, // Milliseconds since the start of the GC.
"reason": "SET_NEW_DOCUMENT",
// GC state when the slice started
"initial_state": "NotActive",
// GC state when the slice ended
"final_state": "Mark",
// Budget is either "Xms", "work(Y)", or
// "unlimited".
"budget": "10ms",
// Number of page faults during the slice.
"page_faults": 0,
"start_timestamp": 294875,
"end_timestamp": 294879,
// Time taken by each phase. There are at most 65 possible
// phases, but usually only a few phases run in a given slice.
"times": {
"wait_background_thread": 0.012,
"mark_discard_code": 2.845,
"purge": 0.723,
"mark": 9.831,
"mark_roots": 0.102,
"buffer_gray_roots": 3.095,
"mark_cross_compartment_wrappers": 0.039,
"mark_c_and_js_stacks": 0.005,
"mark_runtime_wide_data": 2.313,
"mark_embedding": 0.117,
"mark_compartments": 2.27,
"unmark": 1.063,
"minor_gcs_to_evict_nursery": 8.701,
...
}
},
{ ... },
],
// Sum of the phase times across all slices, including
// omitted slices. As before, there are <= 65 possible phases.
"totals": {
"wait_background_thread": 0.012,
"mark_discard_code": 2.845,
"purge": 0.723,
"mark": 9.831,
"mark_roots": 0.102,
"buffer_gray_roots": 3.095,
"mark_cross_compartment_wrappers": 0.039,
"mark_c_and_js_stacks": 0.005,
"mark_runtime_wide_data": 2.313,
"mark_embedding": 0.117,
"mark_compartments": 2.27,
"unmark": 1.063,
"minor_gcs_to_evict_nursery": 8.701,
...
}
},
... // Up to four more selected GCs follow.
],
"worst": [
... // Same as above, but the 2 worst GCs by max_pause.
]
},
fileIOReports
-------------
Contains the statistics of main-thread I/O recorded during the execution. Only the I/O stats for the XRE and the profile directories are currently reported, neither of them disclosing the full local path.
Structure:
.. code-block:: js
"fileIOReports": {
"{xre}": [
totalTime, // Accumulated duration of all operations
creates, // Number of create/open operations
reads, // Number of read operations
writes, // Number of write operations
fsyncs, // Number of fsync operations
stats, // Number of stat operations
],
"{profile}": [ ... ],
...
}
lateWrites
----------
This sections reports writes to the file system that happen during shutdown. The reported data contains the stack and the loaded libraries at the time the writes happened.
Structure:
.. code-block:: js
"lateWrites" : {
"memoryMap" : [
["wgdi32.pdb", "08A541B5942242BDB4AEABD8C87E4CFF2"],
... other entries in the format ["module name", "breakpad identifier"] ...
],
"stacks" : [
[
[
0, // the module index or -1 for invalid module indices
190649 // the offset of this program counter in its module or an absolute pc
],
[1, 2540075],
... other frames ...
],
... other stacks ...
],
},
addonDetails
------------
This section contains per-addon telemetry details, as reported by each addon provider. The XPI provider is the only one reporting at the time of writing (`see DXR <https://dxr.mozilla.org/mozilla-central/search?q=setTelemetryDetails&case=true>`_). Telemetry does not manipulate or enforce a specific format for the supplied provider's data.
Structure:
.. code-block:: js
"addonDetails": {
"XPI": {
"adbhelper@mozilla.org": {
"scan_items": 24,
"scan_MS": 3,
"location": "app-profile",
"name": "ADB Helper",
"creator": "Mozilla & Android Open Source Project",
"startup_MS": 30
},
...
},
...
}
addonHistograms
---------------
This section contains the histogram registered by the addons (`see here <https://dxr.mozilla.org/mozilla-central/rev/584870f1cbc5d060a57e147ce249f736956e2b62/toolkit/components/telemetry/nsITelemetry.idl#303>`_). This section is not present if no addon histogram is available.
UITelemetry
-----------
See the ``UITelemetry data format`` documentation.
slowSQL
-------
This section contains the informations about the slow SQL queries for both the main and other threads. The execution of an SQL statement is considered slow if it takes 50ms or more on the main thread or 100ms or more on other threads. Slow SQL statements will be automatically trimmed to 1000 characters. This limit doesn't include the ellipsis and database name, that are appended at the end of the stored statement.
Structure:
.. code-block:: js
"slowSQL": {
"mainThread": {
"Sanitized SQL Statement": [
1, // the number of times this statement was hit
200 // the total time (in milliseconds) that was spent on this statement
],
...
},
"otherThreads": {
"VACUUM /* places.sqlite */": [
1,
330
],
...
}
},
slowSQLStartup
--------------
This section contains the slow SQL statements gathered at startup (until the "sessionstore-windows-restored" event is fired). The structure of this section resembles the one for `slowSQL`_.
UIMeasurements
--------------
This section contains UI specific telemetry measurements and events. This section is mainly populated with Android-specific data and events (`see here <https://dxr.mozilla.org/mozilla-central/search?q=regexp%3AUITelemetry.%28addEvent|startSession|stopSession%29&redirect=false&case=false>`_).
Structure:
.. code-block:: js
"UIMeasurements": [
{
"type": "event", // either "session" or "event"
"action": "action.1",
"method": "menu",
"sessions": [],
"timestamp": 12345,
"extras": "settings"
},
{
"type": "session",
"name": "awesomescreen.1",
"reason": "commit",
"start": 123,
"end": 456
}
...
],

182
toolkit/components/telemetry/docs/data/sync-ping.rst
View file

@ -1,182 +0,0 @@
"sync" ping
===========
This is an aggregated format that contains information about each sync that occurred during a timeframe. It is submitted every 12 hours, and on browser shutdown, but only if the syncs property would not be empty. The ping does not contain the enviroment block, nor the clientId.
Each item in the syncs property is generated after a sync is completed, for both successful and failed syncs, and contains measurements pertaining to sync performance and error information.
A JSON-schema document describing the exact format of the ping's payload property can be found at `services/sync/tests/unit/sync\_ping\_schema.json <https://dxr.mozilla.org/mozilla-central/source/services/sync/tests/unit/sync_ping_schema.json>`_.
Structure:
.. code-block:: js
{
version: 4,
type: "sync",
... common ping data
payload: {
version: 1,
discarded: <integer count> // Number of syncs discarded -- left out if zero.
why: <string>, // Why did we submit the ping? Either "shutdown" or "schedule".
// Array of recorded syncs. The ping is not submitted if this would be empty
syncs: [{
when: <integer milliseconds since epoch>,
took: <integer duration in milliseconds>,
uid: <string>, // Hashed FxA unique ID, or string of 32 zeros.
deviceID: <string>, // Hashed FxA Device ID, hex string of 64 characters, not included if the user is not logged in.
didLogin: <bool>, // Optional, is this the first sync after login? Excluded if we don't know.
why: <string>, // Optional, why the sync occured, excluded if we don't know.
// Optional, excluded if there was no error.
failureReason: {
name: <string>, // "httperror", "networkerror", "shutdownerror", etc.
code: <integer>, // Only present for "httperror" and "networkerror".
error: <string>, // Only present for "othererror" and "unexpectederror".
from: <string>, // Optional, and only present for "autherror".
},
// Optional, excluded if we couldn't get a valid uid or local device id
devices: [{
os: <string>, // OS string as reported by Services.appinfo.OS,
version: <string>, // Firefox version, as reported by Services.appinfo.version
id: <string>, // Hashed FxA device id for device
}],
// Internal sync status information. Omitted if it would be empty.
status: {
sync: <string>, // The value of the Status.sync property, unless it indicates success.
service: <string>, // The value of the Status.service property, unless it indicates success.
},
// Information about each engine's sync.
engines: [
{
name: <string>, // "bookmarks", "tabs", etc.
took: <integer duration in milliseconds>, // Optional, values of 0 are omitted.
status: <string>, // The value of Status.engines, if it holds a non-success value.
// Optional, excluded if all items would be 0. A missing item indicates a value of 0.
incoming: {
applied: <integer>, // Number of records applied
succeeded: <integer>, // Number of records that applied without error
failed: <integer>, // Number of records that failed to apply
newFailed: <integer>, // Number of records that failed for the first time this sync
reconciled: <integer>, // Number of records that were reconciled
},
// Optional, excluded if it would be empty. Records that would be
// empty (e.g. 0 sent and 0 failed) are omitted.
outgoing: [
{
sent: <integer>, // Number of outgoing records sent. Zero values are omitted.
failed: <integer>, // Number that failed to send. Zero values are omitted.
}
],
// Optional, excluded if there were no errors
failureReason: { ... }, // Same as above.
// Optional, excluded if it would be empty or if the engine cannot
// or did not run validation on itself.
validation: {
// Optional validator version, default of 0.
version: <integer>,
checked: <integer>,
took: <non-monotonic integer duration in milliseconds>,
// Entries with a count of 0 are excluded, the array is excluded if no problems are found.
problems: [
{
name: <string>, // The problem identified.
count: <integer>, // Number of times it occurred.
}
],
// Format is same as above, this is only included if we tried and failed
// to run validation, and if it's present, all other fields in this object are optional.
failureReason: { ... },
}
}
]
}]
}
}
info
----
discarded
~~~~~~~~~
The ping may only contain a certain number of entries in the ``"syncs"`` array, currently 500 (it is determined by the ``"services.sync.telemetry.maxPayloadCount"`` preference). Entries beyond this are discarded, and recorded in the discarded count.
syncs.took
~~~~~~~~~~
These values should be monotonic. If we can't get a monotonic timestamp, -1 will be reported on the payload, and the values will be omitted from the engines. Additionally, the value will be omitted from an engine if it would be 0 (either due to timer inaccuracy or finishing instantaneously).
syncs.uid
~~~~~~~~~
This property containing a hash of the FxA account identifier, which is a 32 character hexidecimal string. In the case that we are unable to authenticate with FxA and have never authenticated in the past, it will be a placeholder string consisting of 32 repeated ``0`` characters.
syncs.why
~~~~~~~~~
One of the following values:
- ``startup``: This is the first sync triggered after browser startup.
- ``schedule``: This is a sync triggered because it has been too long since the last sync.
- ``score``: This sync is triggered by a high score value one of sync's trackers, indicating that many changes have occurred since the last sync.
- ``user``: The user manually triggered the sync.
- ``tabs``: The user opened the synced tabs sidebar, which triggers a sync.
syncs.status
~~~~~~~~~~~~
The ``engine.status``, ``payload.status.sync``, and ``payload.status.service`` properties are sync error codes, which are listed in `services/sync/modules/constants.js <https://dxr.mozilla.org/mozilla-central/source/services/sync/modules/constants.js>`_, and success values are not reported.
syncs.failureReason
~~~~~~~~~~~~~~~~~~~
Stores error information, if any is present. Always contains the "name" property, which identifies the type of error it is. The types can be.
- ``httperror``: Indicates that we recieved an HTTP error response code, but are unable to be more specific about the error. Contains the following properties:
- ``code``: Integer HTTP status code.
- ``nserror``: Indicates that an exception with the provided error code caused sync to fail.
- ``code``: The nsresult error code (integer).
- ``shutdownerror``: Indicates that the sync failed because we shut down before completion.
- ``autherror``: Indicates an unrecoverable authentication error.
- ``from``: Where the authentication error occurred, one of the following values: ``tokenserver``, ``fxaccounts``, or ``hawkclient``.
- ``othererror``: Indicates that it is a sync error code that we are unable to give more specific information on. As with the ``syncStatus`` property, it is a sync error code, which are listed in `services/sync/modules/constants.js <https://dxr.mozilla.org/mozilla-central/source/services/sync/modules/constants.js>`_.
- ``error``: String identifying which error was present.
- ``unexpectederror``: Indicates that some other error caused sync to fail, typically an uncaught exception.
- ``error``: The message provided by the error.
- ``sqlerror``: Indicates that we recieved a ``mozIStorageError`` from a database query.
- ``code``: Value of the ``error.result`` property, one of the constants listed `here <https://developer.mozilla.org/en-US/docs/Mozilla/Tech/XPCOM/Reference/Interface/MozIStorageError#Constants>`_.
syncs.engine.name
~~~~~~~~~~~~~~~~~
Third-party engines are not reported, so only the following values are allowed: ``addons``, ``bookmarks``, ``clients``, ``forms``, ``history``, ``passwords``, ``prefs``, and ``tabs``.
syncs.engine.validation.problems
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
For engines that can run validation on themselves, an array of objects describing validation errors that have occurred. Items that would have a count of 0 are excluded. Each engine will have its own set of items that it might put in the ``name`` field, but there are a finite number. See ``BookmarkProblemData.getSummary`` in `services/sync/modules/bookmark\_validator.js <https://dxr.mozilla.org/mozilla-central/source/services/sync/modules/bookmark_validator.js>`_ for an example.
syncs.devices
~~~~~~~~~~~~~
The list of remote devices associated with this account, as reported by the clients collection. The ID of each device is hashed using the same algorithm as the local id.

26
toolkit/components/telemetry/docs/data/uitour-ping.rst
View file

@ -1,26 +0,0 @@
"uitour-tag" ping
=================
This ping is submitted via the UITour setTreatmentTag API. It may be used by
the tour to record what settings were made by a user or to track the result of
A/B experiments.
The client ID is submitted with this ping.
Structure:
.. code-block:: js
{
version: 1,
type: "uitour-tag",
clientId: <string>,
payload: {
tagName: <string>,
tagValue: <string>
}
}
See also: :doc:`common ping fields <common-ping>`

226
toolkit/components/telemetry/docs/fhr/architecture.rst
View file

@ -1,226 +0,0 @@
.. _healthreport_architecture:
============
Architecture
============
``healthreporter.jsm`` contains the main interface for FHR, the
``HealthReporter`` type. An instance of this is created by the
``data_reporting_service`.
``providers.jsm`` contains numerous ``Metrics.Provider`` and
``Metrics.Measurement`` used for collecting application metrics. If you
are looking for the FHR probes, this is where they are.
Storage
=======
Firefox Health Report stores data in 3 locations:
* Metrics measurements and provider state is stored in a SQLite database
(via ``Metrics.Storage``).
* Service state (such as the IDs of documents uploaded) is stored in a
JSON file on disk (via OS.File).
* Lesser state and run-time options are stored in preferences.
Preferences
===========
Preferences controlling behavior of Firefox Health Report live in the
``datareporting.healthreport.*`` branch.
Service and Data Control
------------------------
The follow preferences control behavior of the service and data upload.
service.enabled
Controls whether the entire health report service runs. The overall
service performs data collection, storing, and submission.
This is the primary kill switch for Firefox Health Report
outside of the build system variable. i.e. if you are using an
official Firefox build and wish to disable FHR, this is what you
should set to false to prevent FHR from not only submitting but
also collecting data.
uploadEnabled
Whether uploading of data is enabled. This is the preference the
checkbox in the preferences UI reflects. If this is
disabled, FHR still collects data - it just doesn't upload it.
service.loadDelayMsec
How long (in milliseconds) after initial application start should FHR
wait before initializing.
FHR may initialize sooner than this if the FHR service is requested.
This will happen if e.g. the user goes to ``about:healthreport``.
service.loadDelayFirstRunMsec
How long (in milliseconds) FHR should wait to initialize on first
application run.
FHR waits longer than normal to initialize on first application run
because first-time initialization can use a lot of I/O to initialize
the SQLite database and this I/O should not interfere with the
first-run user experience.
documentServerURI
The URI of a Bagheera server that FHR should interface with for
submitting documents.
You typically do not need to change this.
documentServerNamespace
The namespace on the document server FHR should upload documents to.
You typically do not need to change this.
infoURL
The URL of a page containing more info about FHR, it's privacy
policy, etc.
about.reportUrl
The URL to load in ``about:healthreport``.
about.reportUrlUnified
The URL to load in ``about:healthreport``. This is used instead of ``reportUrl`` for UnifiedTelemetry when it is not opt-in.
service.providerCategories
A comma-delimited list of category manager categories that contain
registered ``Metrics.Provider`` records. Read below for how provider
registration works.
If the entire service is disabled, you lose data collection. This means
that **local** data analysis won't be available because there is no data
to analyze! Keep in mind that Firefox Health Report can be useful even
if it's not submitting data to remote servers!
Logging
-------
The following preferences allow you to control the logging behavior of
Firefox Health Report.
logging.consoleEnabled
Whether to write log messages to the web console. This is true by
default.
logging.consoleLevel
The minimum log level FHR messages must have to be written to the
web console. By default, only FHR warnings or errors will be written
to the web console. During normal/expected operation, no messages of
this type should be produced.
logging.dumpEnabled
Whether to write log messages via ``dump()``. If true, FHR will write
messages to stdout/stderr.
This is typically only enabled when developing FHR.
logging.dumpLevel
The minimum log level messages must have to be written via
``dump()``.
State
-----
currentDaySubmissionFailureCount
How many submission failures the client has encountered while
attempting to upload the most recent document.
lastDataSubmissionFailureTime
The time of the last failed document upload.
lastDataSubmissionRequestedTime
The time of the last document upload attempt.
lastDataSubmissionSuccessfulTime
The time of the last successful document upload.
nextDataSubmissionTime
The time the next data submission is scheduled for. FHR will not
attempt to upload a new document before this time.
pendingDeleteRemoteData
Whether the client currently has a pending request to delete remote
data. If true, the client will attempt to delete all remote data
before an upload is performed.
FHR stores various state in preferences.
Registering Providers
=====================
Firefox Health Report providers are registered via the category manager.
See ``HealthReportComponents.manifest`` for providers defined in this
directory.
Essentially, the category manager receives the name of a JS type and the
URI of a JSM to import that exports this symbol. At run-time, the
providers registered in the category manager are instantiated.
Providers are registered via the category manager to make registration
simple and less prone to errors. Any XPCOM component can create a
category manager entry. Therefore, new data providers can be added
without having to touch core Firefox Health Report code. Additionally,
category manager registration means providers are more likely to be
registered on FHR's terms, when it wants. If providers were registered
in code at application run-time, there would be the risk of other
components prematurely instantiating FHR (causing a performance hit if
performed at an inopportune time) or semi-complicated code around
observers or listeners. Category manager entries are only 1 line per
provider and leave FHR in control: they are simple and safe.
Document Generation and Lifecycle
=================================
FHR will attempt to submit a JSON document containing data every 24 wall
clock hours.
At upload time, FHR will query the database for **all** information from
the last 180 days and assemble this data into a JSON document. We
attempt to upload this JSON document with a client-generated UUID to the
configured server.
Before we attempt upload, the generated UUID is stored in the JSON state
file on local disk. At this point, the client assumes the document with
that UUID has been successfully stored on the server.
If the client is aware of other document UUIDs that presumably exist on
the server, those UUIDs are sent with the upload request so the client
can request those UUIDs be deleted. This helps ensure that each client
only has 1 document/UUID on the server at any one time.
Importance of Persisting UUIDs
------------------------------
The choices of how, where, and when document UUIDs are stored and updated
are very important. One should not attempt to change things unless she
has a very detailed understanding of why things are the way they are.
The client is purposefully very conservative about forgetting about
generated UUIDs. In other words, once a UUID is generated, the client
deliberately holds on to that UUID until it's very confident that UUID
is no longer stored on the server. The reason we do this is because
*orphaned* documents/UUIDs on the server can lead to faulty analysis,
such as over-reporting the number of Firefox installs that stop being
used.
When uploading a new UUID, we update the state and save the state file
to disk *before* an upload attempt because if the upload succeeds but
the response never makes it back to the client, we want the client to
know about the uploaded UUID so it can delete it later to prevent an
orphan.
We maintain a list of UUIDs locally (not simply the last UUID) because
multiple upload attempts could fail the same way as the previous
paragraph describes and we have no way of knowing which (if any)
actually succeeded. The safest approach is to assume every document
produced managed to get uploaded some how.
We store the UUIDs on a file on disk and not anywhere else because we
want storage to be robust. We originally stored UUIDs in preferences,
which only flush to disk periodically. Writes to preferences were
apparently getting lost. We switched to writing directly to files to
eliminate this window.

1997
toolkit/components/telemetry/docs/fhr/dataformat.rst
View file

File diff suppressed because it is too large Load diff

83
toolkit/components/telemetry/docs/fhr/identifiers.rst
View file

@ -1,83 +0,0 @@
.. _healthreport_identifiers:
===========
Identifiers
===========
Firefox Health Report records some identifiers to keep track of clients
and uploaded documents.
Identifier Types
================
Document/Upload IDs
-------------------
A random UUID called the *Document ID* or *Upload ID* is generated when the FHR
client creates or uploads a new document.
When clients generate a new *Document ID*, they persist this ID to disk
**before** the upload attempt.
As part of the upload, the client sends all old *Document IDs* to the server
and asks the server to delete them. In well-behaving clients, the server
has a single record for each client with a randomly-changing *Document ID*.
Client IDs
----------
A *Client ID* is an identifier that **attempts** to uniquely identify an
individual FHR client. Please note the emphasis on *attempts* in that last
sentence: *Client IDs* do not guarantee uniqueness.
The *Client ID* is generated when the client first runs or as needed.
The *Client ID* is transferred to the server as part of every upload. The
server is thus able to affiliate multiple document uploads with a single
*Client ID*.
Client ID Versions
^^^^^^^^^^^^^^^^^^
The semantics for how a *Client ID* is generated are versioned.
Version 1
The *Client ID* is a randomly-generated UUID.
History of Identifiers
======================
In the beginning, there were just *Document IDs*. The thinking was clients
would clean up after themselves and leave at most 1 active document on the
server.
Unfortunately, this did not work out. Using brute force analysis to
deduplicate records on the server, a number of interesting patterns emerged.
Orphaning
Clients would upload a new payload while not deleting the old payload.
Divergent records
Records would share data up to a certain date and then the data would
almost completely diverge. This appears to be indicative of profile
copying.
Rollback
Records would share data up to a certain date. Each record in this set
would contain data for a day or two but no extra data. This could be
explained by filesystem rollback on the client.
A significant percentage of the records on the server belonged to
misbehaving clients. Identifying these records was extremely resource
intensive and error-prone. These records were undermining the ability
to use Firefox Health Report data.
Thus, the *Client ID* was born. The intent of the *Client ID* was to
uniquely identify clients so the extreme effort required and the
questionable reliability of deduplicating server data would become
problems of the past.
The *Client ID* was originally a randomly-generated UUID (version 1). This
allowed detection of orphaning and rollback. However, these version 1
*Client IDs* were still susceptible to use on multiple profiles and
machines if the profile was copied.

34
toolkit/components/telemetry/docs/fhr/index.rst
View file

@ -1,34 +0,0 @@
================================
Firefox Health Report (Obsolete)
================================
**Firefox Health Report (FHR) is obsolete and no longer ships with Firefox.
This documentation will live here for a few more cycles.**
Firefox Health Report is a background service that collects application
metrics and periodically submits them to a central server. The core
parts of the service are implemented in this directory. However, the
actual XPCOM service is implemented in the
``data_reporting_service`.
The core types can actually be instantiated multiple times and used to
power multiple data submission services within a single Gecko
application. In other words, everything in this directory is effectively
a reusable library. However, the terminology and some of the features
are very specific to what the Firefox Health Report feature requires.
.. toctree::
:maxdepth: 1
architecture
dataformat
identifiers
Legal and Privacy Concerns
==========================
Because Firefox Health Report collects and submits data to remote
servers and is an opt-out feature, there are legal and privacy
concerns over what data may be collected and submitted. **Additions or
changes to submitted data should be signed off by responsible
parties.**

25
toolkit/components/telemetry/docs/index.rst
View file

@ -1,25 +0,0 @@
.. _telemetry:
=========
Telemetry
=========
Telemetry is a feature that allows data collection. This is being used to collect performance metrics and other information about how Firefox performs in the wild.
Client-side, this consists of:
* data collection in `Histograms <https://developer.mozilla.org/en-US/docs/Mozilla/Performance/Adding_a_new_Telemetry_probe>`_, :doc:`collection/scalars` and other data structures
* assembling :doc:`concepts/pings` with the general information and the data payload
* sending them to the server and local ping retention
*Note:* the `data collection policy <https://wiki.mozilla.org/Firefox/Data_Collection>`_ documents the process and requirements that are applied here.
.. toctree::
:maxdepth: 5
:titlesonly:
concepts/index
collection/index
data/index
internals/index
fhr/index

9
toolkit/components/telemetry/docs/internals/index.rst
View file

@ -1,9 +0,0 @@
=========
Internals
=========
.. toctree::
:maxdepth: 2
:titlesonly:
preferences

119
toolkit/components/telemetry/docs/internals/preferences.rst
View file

@ -1,119 +0,0 @@
Preferences
===========
Telemetry behaviour is controlled through the preferences listed here.
Default behaviors
-----------------
Sending only happens on official builds (i.e. with ``MOZILLA_OFFICIAL`` set) with ``MOZ_TELEMETRY_REPORTING`` defined.
All other builds drop all outgoing pings, so they will also not retry sending them later.
Preferences
-----------
``toolkit.telemetry.unified``
This controls whether unified behavior is enabled. If true:
* Telemetry is always enabled and recording *base* data.
* Telemetry will send additional ``main`` pings.
``toolkit.telemetry.enabled``
If ``unified`` is off, this controls whether the Telemetry module is enabled.
If ``unified`` is on, this controls whether to record *extended* data.
This preference is controlled through the `Preferences` dialog.
Note that the default value here of this pref depends on the define ``RELEASE_OR_BETA`` and the channel.
If ``RELEASE_OR_BETA`` is set, ``MOZ_TELEMETRY_ON_BY_DEFAULT`` gets set, which means this pref will default to ``true``.
This is overridden by the preferences code on the "beta" channel, the pref also defaults to ``true`` there.
``datareporting.healthreport.uploadEnabled``
Send the data we record if user has consented to FHR. This preference is controlled through the `Preferences` dialog.
``toolkit.telemetry.archive.enabled``
Allow pings to be archived locally. This can only be enabled if ``unified`` is on.
``toolkit.telemetry.server``
The server Telemetry pings are sent to.
``toolkit.telemetry.log.level``
This sets the Telemetry logging verbosity per ``Log.jsm``, with ``Trace`` or ``0`` being the most verbose and the default being ``Warn``.
By default logging goes only the console service.
``toolkit.telemetry.log.dump``
Sets whether to dump Telemetry log messages to ``stdout`` too.
Data-choices notification
-------------------------
``toolkit.telemetry.reportingpolicy.firstRun``
This preference is not present until the first run. After, its value is set to false. This is used to show the infobar with a more aggressive timeout if it wasn't shown yet.
``datareporting.policy.firstRunURL``
If set, a browser tab will be opened on first run instead of the infobar.
``datareporting.policy.dataSubmissionEnabled``
This is the data submission master kill switch. If disabled, no policy is shown or upload takes place, ever.
``datareporting.policy.dataSubmissionPolicyNotifiedTime``
Records the date user was shown the policy. This preference is also used on Android.
``datareporting.policy.dataSubmissionPolicyAcceptedVersion``
Records the version of the policy notified to the user. This preference is also used on Android.
``datareporting.policy.dataSubmissionPolicyBypassNotification``
Used in tests, it allows to skip the notification check.
``datareporting.policy.currentPolicyVersion``
Stores the current policy version, overrides the default value defined in TelemetryReportingPolicy.jsm.
``datareporting.policy.minimumPolicyVersion``
The minimum policy version that is accepted for the current policy. This can be set per channel.
``datareporting.policy.minimumPolicyVersion.channel-NAME``
This is the only channel-specific version that we currently use for the minimum policy version.
Testing
-------
The following prefs are for testing purpose only.
``toolkit.telemetry.initDelay``
Delay before initializing telemetry (seconds).
``toolkit.telemetry.minSubsessionLength``
Minimum length of a telemetry subsession (seconds).
``toolkit.telemetry.collectInterval``
Minimum interval between data collection (seconds).
``toolkit.telemetry.scheduler.tickInterval``
Interval between scheduler ticks (seconds).
``toolkit.telemetry.scheduler.idleTickInterval``
Interval between scheduler ticks when the user is idle (seconds).
``toolkit.telemetry.idleTimeout``
Timeout until we decide whether a user is idle or not (seconds).

142
toolkit/components/telemetry/gen-event-data.py
View file

@ -1,142 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out event information for C++. The events are defined
# in a file provided as a command-line argument.
from __future__ import print_function
from shared_telemetry_utils import StringTable, static_assert
import parse_events
import sys
import itertools
# The banner/text at the top of the generated file.
banner = """/* This file is auto-generated, only for internal use in TelemetryEvent.h,
see gen-event-data.py. */
"""
file_header = """\
#ifndef mozilla_TelemetryEventData_h
#define mozilla_TelemetryEventData_h
#include "EventInfo.h"
namespace {
"""
file_footer = """\
} // namespace
#endif // mozilla_TelemetryEventData_h
"""
def write_extra_table(events, output, string_table):
table_name = "gExtraKeysTable"
extra_table = []
extra_count = 0
print("const uint32_t %s[] = {" % table_name, file=output)
for e in events:
extra_index = 0
extra_keys = e.extra_keys
if len(extra_keys) > 0:
extra_index = extra_count
extra_count += len(extra_keys)
indexes = string_table.stringIndexes(extra_keys)
print(" // %s, [%s], [%s]" % (
e.category,
", ".join(e.methods),
", ".join(e.objects)),
file=output)
print(" // extra_keys: %s" % ", ".join(extra_keys), file=output)
print(" %s," % ", ".join(map(str, indexes)),
file=output)
extra_table.append((extra_index, len(extra_keys)))
print("};", file=output)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % table_name,
"index overflow")
return extra_table
def write_common_event_table(events, output, string_table, extra_table):
table_name = "gCommonEventInfo"
extra_count = 0
print("const CommonEventInfo %s[] = {" % table_name, file=output)
for e,extras in zip(events, extra_table):
# Write a comment to make the file human-readable.
print(" // category: %s" % e.category, file=output)
print(" // methods: [%s]" % ", ".join(e.methods), file=output)
print(" // objects: [%s]" % ", ".join(e.objects), file=output)
# Write the common info structure
print(" {%d, %d, %d, %d, %d, %s}," %
(string_table.stringIndex(e.category),
string_table.stringIndex(e.expiry_version),
extras[0], # extra keys index
extras[1], # extra keys count
e.expiry_day,
e.dataset),
file=output)
print("};", file=output)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % table_name,
"index overflow")
def write_event_table(events, output, string_table):
table_name = "gEventInfo"
print("const EventInfo %s[] = {" % table_name, file=output)
for common_info_index,e in enumerate(events):
for method_name, object_name in itertools.product(e.methods, e.objects):
print(" // category: %s, method: %s, object: %s" %
(e.category, method_name, object_name),
file=output)
print(" {gCommonEventInfo[%d], %d, %d}," %
(common_info_index,
string_table.stringIndex(method_name),
string_table.stringIndex(object_name)),
file=output)
print("};", file=output)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % table_name,
"index overflow")
def main(output, *filenames):
# Load the event data.
if len(filenames) > 1:
raise Exception('We don\'t support loading from more than one file.')
events = parse_events.load_events(filenames[0])
# Write the scalar data file.
print(banner, file=output)
print(file_header, file=output)
# Write the extra keys table.
string_table = StringTable()
extra_table = write_extra_table(events, output, string_table)
print("", file=output)
# Write a table with the common event data.
write_common_event_table(events, output, string_table, extra_table)
print("", file=output)
# Write the data for individual events.
write_event_table(events, output, string_table)
print("", file=output)
# Write the string table.
string_table_name = "gEventsStringTable"
string_table.writeDefinition(output, string_table_name)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % string_table_name,
"index overflow")
print("", file=output)
print(file_footer, file=output)
if __name__ == '__main__':
main(sys.stdout, *sys.argv[1:])

73
toolkit/components/telemetry/gen-event-enum.py
View file

@ -1,73 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out C++ enum definitions that represent the different event types.
#
# The events are defined in files provided as command-line arguments.
from __future__ import print_function
import sys
import parse_events
banner = """/* This file is auto-generated, see gen-event-enum.py. */
"""
file_header = """\
#ifndef mozilla_TelemetryEventEnums_h
#define mozilla_TelemetryEventEnums_h
namespace mozilla {
namespace Telemetry {
namespace EventID {
"""
file_footer = """\
} // namespace EventID
} // namespace mozilla
} // namespace Telemetry
#endif // mozilla_TelemetryEventEnums_h
"""
def main(output, *filenames):
# Load the events first.
if len(filenames) > 1:
raise Exception('We don\'t support loading from more than one file.')
events = parse_events.load_events(filenames[0])
grouped = dict()
index = 0
for e in events:
category = e.category
if not category in grouped:
grouped[category] = []
grouped[category].append((index, e))
index += len(e.enum_labels)
# Write the enum file.
print(banner, file=output)
print(file_header, file=output);
for category,indexed in grouped.iteritems():
category_cpp = indexed[0][1].category_cpp
print("// category: %s" % category, file=output)
print("enum class %s : uint32_t {" % category_cpp, file=output)
for event_index,e in indexed:
cpp_guard = e.cpp_guard
if cpp_guard:
print("#if defined(%s)" % cpp_guard, file=output)
for offset,label in enumerate(e.enum_labels):
print(" %s = %d," % (label, event_index + offset), file=output)
if cpp_guard:
print("#endif", file=output)
print("};\n", file=output)
print("const uint32_t EventCount = %d;\n" % index, file=output)
print(file_footer, file=output)
if __name__ == '__main__':
main(sys.stdout, *sys.argv[1:])

52
toolkit/components/telemetry/gen-histogram-bucket-ranges.py
View file

@ -1,52 +0,0 @@
#!/usr/bin/env python
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out detailed histogram information, including the ranges of the
# buckets specified by each histogram.
import sys
import re
import histogram_tools
import json
from collections import OrderedDict
def main(argv):
filenames = argv
all_histograms = OrderedDict()
for histogram in histogram_tools.from_files(filenames):
name = histogram.name()
parameters = OrderedDict()
table = {
'boolean': '2',
'flag': '3',
'enumerated': '1',
'linear': '1',
'exponential': '0',
'count': '4',
}
# Use __setitem__ because Python lambdas are so limited.
histogram_tools.table_dispatch(histogram.kind(), table,
lambda k: parameters.__setitem__('kind', k))
if histogram.low() == 0:
parameters['min'] = 1
else:
parameters['min'] = histogram.low()
try:
buckets = histogram.ranges()
parameters['buckets'] = buckets
parameters['max'] = buckets[-1]
parameters['bucket_count'] = len(buckets)
except histogram_tools.DefinitionException:
continue
all_histograms.update({ name: parameters });
print json.dumps({ 'histograms': all_histograms})
main(sys.argv[1:])

178
toolkit/components/telemetry/gen-histogram-data.py
View file

@ -1,178 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out histogram information for C++. The histograms are defined
# in a file provided as a command-line argument.
from __future__ import print_function
from shared_telemetry_utils import StringTable, static_assert
import sys
import histogram_tools
import itertools
banner = """/* This file is auto-generated, see gen-histogram-data.py. */
"""
def print_array_entry(output, histogram, name_index, exp_index, label_index, label_count):
cpp_guard = histogram.cpp_guard()
if cpp_guard:
print("#if defined(%s)" % cpp_guard, file=output)
print(" { %s, %s, %s, %s, %d, %d, %s, %d, %d, %s }," \
% (histogram.low(),
histogram.high(),
histogram.n_buckets(),
histogram.nsITelemetry_kind(),
name_index,
exp_index,
histogram.dataset(),
label_index,
label_count,
"true" if histogram.keyed() else "false"), file=output)
if cpp_guard:
print("#endif", file=output)
def write_histogram_table(output, histograms):
string_table = StringTable()
label_table = []
label_count = 0
print("const HistogramInfo gHistograms[] = {", file=output)
for histogram in histograms:
name_index = string_table.stringIndex(histogram.name())
exp_index = string_table.stringIndex(histogram.expiration())
labels = histogram.labels()
label_index = 0
if len(labels) > 0:
label_index = label_count
label_table.append((histogram.name(), string_table.stringIndexes(labels)))
label_count += len(labels)
print_array_entry(output, histogram,
name_index, exp_index,
label_index, len(labels))
print("};\n", file=output)
strtab_name = "gHistogramStringTable"
string_table.writeDefinition(output, strtab_name)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % strtab_name,
"index overflow")
print("\nconst uint32_t gHistogramLabelTable[] = {", file=output)
for name,indexes in label_table:
print("/* %s */ %s," % (name, ", ".join(map(str, indexes))), file=output)
print("};", file=output)
# Write out static asserts for histogram data. We'd prefer to perform
# these checks in this script itself, but since several histograms
# (generally enumerated histograms) use compile-time constants for
# their upper bounds, we have to let the compiler do the checking.
def static_asserts_for_boolean(output, histogram):
pass
def static_asserts_for_flag(output, histogram):
pass
def static_asserts_for_count(output, histogram):
pass
def static_asserts_for_enumerated(output, histogram):
n_values = histogram.high()
static_assert(output, "%s > 2" % n_values,
"Not enough values for %s" % histogram.name())
def shared_static_asserts(output, histogram):
name = histogram.name()
low = histogram.low()
high = histogram.high()
n_buckets = histogram.n_buckets()
static_assert(output, "%s < %s" % (low, high), "low >= high for %s" % name)
static_assert(output, "%s > 2" % n_buckets, "Not enough values for %s" % name)
static_assert(output, "%s >= 1" % low, "Incorrect low value for %s" % name)
static_assert(output, "%s > %s" % (high, n_buckets),
"high must be > number of buckets for %s; you may want an enumerated histogram" % name)
def static_asserts_for_linear(output, histogram):
shared_static_asserts(output, histogram)
def static_asserts_for_exponential(output, histogram):
shared_static_asserts(output, histogram)
def write_histogram_static_asserts(output, histograms):
print("""
// Perform the checks at the beginning of HistogramGet at
// compile time, so that incorrect histogram definitions
// give compile-time errors, not runtime errors.""", file=output)
table = {
'boolean' : static_asserts_for_boolean,
'flag' : static_asserts_for_flag,
'count': static_asserts_for_count,
'enumerated' : static_asserts_for_enumerated,
'categorical' : static_asserts_for_enumerated,
'linear' : static_asserts_for_linear,
'exponential' : static_asserts_for_exponential,
}
for histogram in histograms:
histogram_tools.table_dispatch(histogram.kind(), table,
lambda f: f(output, histogram))
def write_debug_histogram_ranges(output, histograms):
ranges_lengths = []
# Collect all the range information from individual histograms.
# Write that information out as well.
print("#ifdef DEBUG", file=output)
print("const int gBucketLowerBounds[] = {", file=output)
for histogram in histograms:
ranges = []
try:
ranges = histogram.ranges()
except histogram_tools.DefinitionException:
pass
ranges_lengths.append(len(ranges))
# Note that we do not test cpp_guard here. We do this so we
# will have complete information about all the histograms in
# this array. Just having information about the ranges of
# histograms is not platform-specific; if there are histograms
# that have platform-specific constants in their definitions,
# those histograms will fail in the .ranges() call above and
# we'll have a zero-length array to deal with here.
if len(ranges) > 0:
print(','.join(map(str, ranges)), ',', file=output)
else:
print('/* Skipping %s */' % histogram.name(), file=output)
print("};", file=output)
# Write the offsets into gBucketLowerBounds.
print("struct bounds { int offset; int length; };", file=output)
print("const struct bounds gBucketLowerBoundIndex[] = {", file=output)
offset = 0
for (histogram, range_length) in itertools.izip(histograms, ranges_lengths):
cpp_guard = histogram.cpp_guard()
# We do test cpp_guard here, so that histogram IDs are valid
# indexes into this array.
if cpp_guard:
print("#if defined(%s)" % cpp_guard, file=output)
print("{ %d, %d }," % (offset, range_length), file=output)
if cpp_guard:
print("#endif", file=output)
offset += range_length
print("};", file=output)
print("#endif", file=output)
def main(output, *filenames):
histograms = list(histogram_tools.from_files(filenames))
print(banner, file=output)
write_histogram_table(output, histograms)
write_histogram_static_asserts(output, histograms)
write_debug_histogram_ranges(output, histograms)
if __name__ == '__main__':
main(sys.stdout, *sys.argv[1:])

107
toolkit/components/telemetry/gen-histogram-enum.py
View file

@ -1,107 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out a C++ enum definition whose members are the names of
# histograms as well as the following other members:
#
# - HistogramCount
# - HistogramFirstUseCounter
# - HistogramLastUseCounter
# - HistogramUseCounterCount
#
# The histograms are defined in files provided as command-line arguments.
from __future__ import print_function
import histogram_tools
import itertools
import sys
banner = """/* This file is auto-generated, see gen-histogram-enum.py. */
"""
header = """
#ifndef mozilla_TelemetryHistogramEnums_h
#define mozilla_TelemetryHistogramEnums_h
#include "mozilla/TemplateLib.h"
namespace mozilla {
namespace Telemetry {
"""
footer = """
} // namespace mozilla
} // namespace Telemetry
#endif // mozilla_TelemetryHistogramEnums_h"""
def main(output, *filenames):
# Print header.
print(banner, file=output)
print(header, file=output)
# Load the histograms.
all_histograms = list(histogram_tools.from_files(filenames))
groups = itertools.groupby(all_histograms,
lambda h: h.name().startswith("USE_COUNTER2_"))
# Print the histogram enums.
# Note that histogram_tools.py guarantees that all of the USE_COUNTER2_*
# histograms are defined in a contiguous block. We therefore assume
# that there's at most one group for which use_counter_group is true.
print("enum ID : uint32_t {", file=output)
seen_use_counters = False
for (use_counter_group, histograms) in groups:
if use_counter_group:
seen_use_counters = True
# The HistogramDUMMY* enum variables are used to make the computation
# of Histogram{First,Last}UseCounter easier. Otherwise, we'd have to
# special case the first and last histogram in the group.
if use_counter_group:
print(" HistogramFirstUseCounter,", file=output)
print(" HistogramDUMMY1 = HistogramFirstUseCounter - 1,", file=output)
for histogram in histograms:
cpp_guard = histogram.cpp_guard()
if cpp_guard:
print("#if defined(%s)" % cpp_guard, file=output)
print(" %s," % histogram.name(), file=output)
if cpp_guard:
print("#endif", file=output)
if use_counter_group:
print(" HistogramDUMMY2,", file=output)
print(" HistogramLastUseCounter = HistogramDUMMY2 - 1,", file=output)
print(" HistogramCount,", file=output)
if seen_use_counters:
print(" HistogramUseCounterCount = HistogramLastUseCounter - HistogramFirstUseCounter + 1", file=output)
else:
print(" HistogramFirstUseCounter = 0,", file=output)
print(" HistogramLastUseCounter = 0,", file=output)
print(" HistogramUseCounterCount = 0", file=output)
print("};", file=output)
# Write categorical label enums.
categorical = filter(lambda h: h.kind() == "categorical", all_histograms)
enums = [("LABELS_" + h.name(), h.labels(), h.name()) for h in categorical]
for name,labels,_ in enums:
print("\nenum class %s : uint32_t {" % name, file=output)
print(" %s" % ",\n ".join(labels), file=output)
print("};", file=output)
print("\ntemplate<class T> struct IsCategoricalLabelEnum : FalseType {};", file=output)
for name,_,_ in enums:
print("template<> struct IsCategoricalLabelEnum<%s> : TrueType {};" % name, file=output)
print("\ntemplate<class T> struct CategoricalLabelId {};", file=output)
for name,_,id in enums:
print("template<> struct CategoricalLabelId<%s> : IntegralConstant<uint32_t, %s> {};" % (name, id), file=output)
# Footer.
print(footer, file=output)
if __name__ == '__main__':
main(sys.stdout, *sys.argv[1:])

90
toolkit/components/telemetry/gen-scalar-data.py
View file

@ -1,90 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out scalar information for C++. The scalars are defined
# in a file provided as a command-line argument.
from __future__ import print_function
from shared_telemetry_utils import StringTable, static_assert
import parse_scalars
import sys
# The banner/text at the top of the generated file.
banner = """/* This file is auto-generated, only for internal use in TelemetryScalar.h,
see gen-scalar-data.py. */
"""
file_header = """\
#ifndef mozilla_TelemetryScalarData_h
#define mozilla_TelemetryScalarData_h
#include "ScalarInfo.h"
namespace {
"""
file_footer = """\
} // namespace
#endif // mozilla_TelemetryScalarData_h
"""
def write_scalar_info(scalar, output, name_index, expiration_index):
"""Writes a scalar entry to the output file.
:param scalar: a ScalarType instance describing the scalar.
:param output: the output stream.
:param name_index: the index of the scalar name in the strings table.
:param expiration_index: the index of the expiration version in the strings table.
"""
cpp_guard = scalar.cpp_guard
if cpp_guard:
print("#if defined(%s)" % cpp_guard, file=output)
print(" {{ {}, {}, {}, {}, {} }},"\
.format(scalar.nsITelemetry_kind,
name_index,
expiration_index,
scalar.dataset,
"true" if scalar.keyed else "false"),
file=output)
if cpp_guard:
print("#endif", file=output)
def write_scalar_tables(scalars, output):
"""Writes the scalar and strings tables to an header file.
:param scalars: a list of ScalarType instances describing the scalars.
:param output: the output stream.
"""
string_table = StringTable()
print("const ScalarInfo gScalars[] = {", file=output)
for s in scalars:
# We add both the scalar label and the expiration string to the strings
# table.
name_index = string_table.stringIndex(s.label)
exp_index = string_table.stringIndex(s.expires)
# Write the scalar info entry.
write_scalar_info(s, output, name_index, exp_index)
print("};", file=output)
string_table_name = "gScalarsStringTable"
string_table.writeDefinition(output, string_table_name)
static_assert(output, "sizeof(%s) <= UINT32_MAX" % string_table_name,
"index overflow")
def main(output, *filenames):
# Load the scalars first.
if len(filenames) > 1:
raise Exception('We don\'t support loading from more than one file.')
scalars = parse_scalars.load_scalars(filenames[0])
# Write the scalar data file.
print(banner, file=output)
print(file_header, file=output)
write_scalar_tables(scalars, output)
print(file_footer, file=output)
if __name__ == '__main__':
main(sys.stdout, *sys.argv[1:])

56
toolkit/components/telemetry/gen-scalar-enum.py
View file

@ -1,56 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# Write out a C++ enum definition whose members are the names of
# scalar types.
#
# The scalars are defined in files provided as command-line arguments.
from __future__ import print_function
import sys
import parse_scalars
banner = """/* This file is auto-generated, see gen-scalar-enum.py. */
"""
file_header = """\
#ifndef mozilla_TelemetryScalarEnums_h
#define mozilla_TelemetryScalarEnums_h
namespace mozilla {
namespace Telemetry {
enum class ScalarID : uint32_t {\
"""
file_footer = """\
};
} // namespace mozilla
} // namespace Telemetry
#endif // mozilla_TelemetryScalarEnums_h
"""
def main(output, *filenames):
# Load the scalars first.
if len(filenames) > 1:
raise Exception('We don\'t support loading from more than one file.')
scalars = parse_scalars.load_scalars(filenames[0])
# Write the enum file.
print(banner, file=output)
print(file_header, file=output);
for s in scalars:
cpp_guard = s.cpp_guard
if cpp_guard:
print("#if defined(%s)" % cpp_guard, file=output)
print(" %s," % s.enum_label, file=output)
if cpp_guard:
print("#endif", file=output)
print(" ScalarCount,", file=output)
print(file_footer, file=output)
if __name__ == '__main__':
main(sys.stdout, *sys.argv[1:])

10
toolkit/components/telemetry/healthreport-prefs.js
View file

@ -1,10 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
pref("datareporting.healthreport.infoURL", "https://www.mozilla.org/legal/privacy/firefox.html#health-report");
// Health Report is enabled by default on all channels.
pref("datareporting.healthreport.uploadEnabled", true);
pref("datareporting.healthreport.about.reportUrl", "https://fhr.cdn.mozilla.net/%LOCALE%/v4/");

1960
toolkit/components/telemetry/histogram-whitelists.json
View file

File diff suppressed because it is too large Load diff

513
toolkit/components/telemetry/histogram_tools.py
View file

@ -1,513 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import collections
import itertools
import json
import math
import os
import re
import sys
# Constants.
MAX_LABEL_LENGTH = 20
MAX_LABEL_COUNT = 100
# histogram_tools.py is used by scripts from a mozilla-central build tree
# and also by outside consumers, such as the telemetry server. We need
# to ensure that importing things works in both contexts. Therefore,
# unconditionally importing things that are local to the build tree, such
# as buildconfig, is a no-no.
try:
import buildconfig
# Need to update sys.path to be able to find usecounters.
sys.path.append(os.path.join(buildconfig.topsrcdir, 'dom/base/'))
except ImportError:
# Must be in an out-of-tree usage scenario. Trust that whoever is
# running this script knows we need the usecounters module and has
# ensured it's in our sys.path.
pass
from collections import OrderedDict
def table_dispatch(kind, table, body):
"""Call body with table[kind] if it exists. Raise an error otherwise."""
if kind in table:
return body(table[kind])
else:
raise BaseException, "don't know how to handle a histogram of kind %s" % kind
class DefinitionException(BaseException):
pass
def linear_buckets(dmin, dmax, n_buckets):
ret_array = [0] * n_buckets
dmin = float(dmin)
dmax = float(dmax)
for i in range(1, n_buckets):
linear_range = (dmin * (n_buckets - 1 - i) + dmax * (i - 1)) / (n_buckets - 2)
ret_array[i] = int(linear_range + 0.5)
return ret_array
def exponential_buckets(dmin, dmax, n_buckets):
log_max = math.log(dmax);
bucket_index = 2;
ret_array = [0] * n_buckets
current = dmin
ret_array[1] = current
for bucket_index in range(2, n_buckets):
log_current = math.log(current)
log_ratio = (log_max - log_current) / (n_buckets - bucket_index)
log_next = log_current + log_ratio
next_value = int(math.floor(math.exp(log_next) + 0.5))
if next_value > current:
current = next_value
else:
current = current + 1
ret_array[bucket_index] = current
return ret_array
always_allowed_keys = ['kind', 'description', 'cpp_guard', 'expires_in_version',
'alert_emails', 'keyed', 'releaseChannelCollection',
'bug_numbers']
whitelists = None;
try:
whitelist_path = os.path.join(os.path.abspath(os.path.realpath(os.path.dirname(__file__))), 'histogram-whitelists.json')
with open(whitelist_path, 'r') as f:
try:
whitelists = json.load(f)
for name, whitelist in whitelists.iteritems():
whitelists[name] = set(whitelist)
except ValueError, e:
raise BaseException, 'error parsing whitelist (%s)' % whitelist_path
except IOError:
whitelists = None
print 'Unable to parse whitelist (%s). Assuming all histograms are acceptable.' % whitelist_path
class Histogram:
"""A class for representing a histogram definition."""
def __init__(self, name, definition, strict_type_checks=False):
"""Initialize a histogram named name with the given definition.
definition is a dict-like object that must contain at least the keys:
- 'kind': The kind of histogram. Must be one of 'boolean', 'flag',
'count', 'enumerated', 'linear', or 'exponential'.
- 'description': A textual description of the histogram.
- 'strict_type_checks': A boolean indicating whether to use the new, stricter type checks.
The server-side still has to deal with old, oddly typed submissions,
so we have to skip them there by default.
The key 'cpp_guard' is optional; if present, it denotes a preprocessor
symbol that should guard C/C++ definitions associated with the histogram."""
self._strict_type_checks = strict_type_checks
self._is_use_counter = name.startswith("USE_COUNTER2_")
self.verify_attributes(name, definition)
self._name = name
self._description = definition['description']
self._kind = definition['kind']
self._cpp_guard = definition.get('cpp_guard')
self._keyed = definition.get('keyed', False)
self._expiration = definition.get('expires_in_version')
self._labels = definition.get('labels', [])
self.compute_bucket_parameters(definition)
table = {
'boolean': 'BOOLEAN',
'flag': 'FLAG',
'count': 'COUNT',
'enumerated': 'LINEAR',
'categorical': 'CATEGORICAL',
'linear': 'LINEAR',
'exponential': 'EXPONENTIAL',
}
table_dispatch(self.kind(), table,
lambda k: self._set_nsITelemetry_kind(k))
datasets = { 'opt-in': 'DATASET_RELEASE_CHANNEL_OPTIN',
'opt-out': 'DATASET_RELEASE_CHANNEL_OPTOUT' }
value = definition.get('releaseChannelCollection', 'opt-in')
if not value in datasets:
raise DefinitionException, "unknown release channel collection policy for " + name
self._dataset = "nsITelemetry::" + datasets[value]
def name(self):
"""Return the name of the histogram."""
return self._name
def description(self):
"""Return the description of the histogram."""
return self._description
def kind(self):
"""Return the kind of the histogram.
Will be one of 'boolean', 'flag', 'count', 'enumerated', 'categorical', 'linear',
or 'exponential'."""
return self._kind
def expiration(self):
"""Return the expiration version of the histogram."""
return self._expiration
def nsITelemetry_kind(self):
"""Return the nsITelemetry constant corresponding to the kind of
the histogram."""
return self._nsITelemetry_kind
def _set_nsITelemetry_kind(self, kind):
self._nsITelemetry_kind = "nsITelemetry::HISTOGRAM_%s" % kind
def low(self):
"""Return the lower bound of the histogram."""
return self._low
def high(self):
"""Return the high bound of the histogram."""
return self._high
def n_buckets(self):
"""Return the number of buckets in the histogram."""
return self._n_buckets
def cpp_guard(self):
"""Return the preprocessor symbol that should guard C/C++ definitions
associated with the histogram. Returns None if no guarding is necessary."""
return self._cpp_guard
def keyed(self):
"""Returns True if this a keyed histogram, false otherwise."""
return self._keyed
def dataset(self):
"""Returns the dataset this histogram belongs into."""
return self._dataset
def labels(self):
"""Returns a list of labels for a categorical histogram, [] for others."""
return self._labels
def ranges(self):
"""Return an array of lower bounds for each bucket in the histogram."""
table = {
'boolean': linear_buckets,
'flag': linear_buckets,
'count': linear_buckets,
'enumerated': linear_buckets,
'categorical': linear_buckets,
'linear': linear_buckets,
'exponential': exponential_buckets,
}
return table_dispatch(self.kind(), table,
lambda p: p(self.low(), self.high(), self.n_buckets()))
def compute_bucket_parameters(self, definition):
table = {
'boolean': Histogram.boolean_flag_bucket_parameters,
'flag': Histogram.boolean_flag_bucket_parameters,
'count': Histogram.boolean_flag_bucket_parameters,
'enumerated': Histogram.enumerated_bucket_parameters,
'categorical': Histogram.categorical_bucket_parameters,
'linear': Histogram.linear_bucket_parameters,
'exponential': Histogram.exponential_bucket_parameters,
}
table_dispatch(self.kind(), table,
lambda p: self.set_bucket_parameters(*p(definition)))
def verify_attributes(self, name, definition):
global always_allowed_keys
general_keys = always_allowed_keys + ['low', 'high', 'n_buckets']
table = {
'boolean': always_allowed_keys,
'flag': always_allowed_keys,
'count': always_allowed_keys,
'enumerated': always_allowed_keys + ['n_values'],
'categorical': always_allowed_keys + ['labels'],
'linear': general_keys,
'exponential': general_keys,
}
# We removed extended_statistics_ok on the client, but the server-side,
# where _strict_type_checks==False, has to deal with historical data.
if not self._strict_type_checks:
table['exponential'].append('extended_statistics_ok')
table_dispatch(definition['kind'], table,
lambda allowed_keys: Histogram.check_keys(name, definition, allowed_keys))
self.check_name(name)
self.check_field_types(name, definition)
self.check_whitelistable_fields(name, definition)
self.check_expiration(name, definition)
self.check_label_values(name, definition)
def check_name(self, name):
if '#' in name:
raise ValueError, '"#" not permitted for %s' % (name)
# Avoid C++ identifier conflicts between histogram enums and label enum names.
if name.startswith("LABELS_"):
raise ValueError, "Histogram name '%s' can not start with LABELS_" % (name)
# To make it easier to generate C++ identifiers from this etc., we restrict
# the histogram names to a strict pattern.
# We skip this on the server to avoid failures with old Histogram.json revisions.
if self._strict_type_checks:
pattern = '^[a-z][a-z0-9_]+[a-z0-9]$'
if not re.match(pattern, name, re.IGNORECASE):
raise ValueError, "Histogram name '%s' doesn't confirm to '%s'" % (name, pattern)
def check_expiration(self, name, definition):
field = 'expires_in_version'
expiration = definition.get(field)
if not expiration:
return
# We forbid new probes from using "expires_in_version" : "default" field/value pair.
# Old ones that use this are added to the whitelist.
if expiration == "default" and name not in whitelists['expiry_default']:
raise ValueError, 'New histogram "%s" cannot have "default" %s value.' % (name, field)
if re.match(r'^[1-9][0-9]*$', expiration):
expiration = expiration + ".0a1"
elif re.match(r'^[1-9][0-9]*\.0$', expiration):
expiration = expiration + "a1"
definition[field] = expiration
def check_label_values(self, name, definition):
labels = definition.get('labels')
if not labels:
return
invalid = filter(lambda l: len(l) > MAX_LABEL_LENGTH, labels)
if len(invalid) > 0:
raise ValueError, 'Label values for %s exceed length limit of %d: %s' % \
(name, MAX_LABEL_LENGTH, ', '.join(invalid))
if len(labels) > MAX_LABEL_COUNT:
raise ValueError, 'Label count for %s exceeds limit of %d' % \
(name, MAX_LABEL_COUNT)
# To make it easier to generate C++ identifiers from this etc., we restrict
# the label values to a strict pattern.
pattern = '^[a-z][a-z0-9_]+[a-z0-9]$'
invalid = filter(lambda l: not re.match(pattern, l, re.IGNORECASE), labels)
if len(invalid) > 0:
raise ValueError, 'Label values for %s are not matching pattern "%s": %s' % \
(name, pattern, ', '.join(invalid))
# Check for the presence of fields that old histograms are whitelisted for.
def check_whitelistable_fields(self, name, definition):
# Use counters don't have any mechanism to add the fields checked here,
# so skip the check for them.
# We also don't need to run any of these checks on the server.
if self._is_use_counter or not self._strict_type_checks:
return
# In the pipeline we don't have whitelists available.
if whitelists is None:
return
for field in ['alert_emails', 'bug_numbers']:
if field not in definition and name not in whitelists[field]:
raise KeyError, 'New histogram "%s" must have a %s field.' % (name, field)
if field in definition and name in whitelists[field]:
msg = 'Should remove histogram "%s" from the whitelist for "%s" in histogram-whitelists.json'
raise KeyError, msg % (name, field)
def check_field_types(self, name, definition):
# Define expected types for the histogram properties.
type_checked_fields = {
"n_buckets": int,
"n_values": int,
"low": int,
"high": int,
"keyed": bool,
"expires_in_version": basestring,
"kind": basestring,
"description": basestring,
"cpp_guard": basestring,
"releaseChannelCollection": basestring,
}
# For list fields we check the items types.
type_checked_list_fields = {
"bug_numbers": int,
"alert_emails": basestring,
"labels": basestring,
}
# For the server-side, where _strict_type_checks==False, we want to
# skip the stricter type checks for these fields for dealing with
# historical data.
coerce_fields = ["low", "high", "n_values", "n_buckets"]
if not self._strict_type_checks:
def try_to_coerce_to_number(v):
try:
return eval(v, {})
except:
return v
for key in [k for k in coerce_fields if k in definition]:
definition[key] = try_to_coerce_to_number(definition[key])
# This handles old "keyed":"true" definitions (bug 1271986).
if definition.get("keyed", None) == "true":
definition["keyed"] = True
def nice_type_name(t):
if t is basestring:
return "string"
return t.__name__
for key, key_type in type_checked_fields.iteritems():
if not key in definition:
continue
if not isinstance(definition[key], key_type):
raise ValueError, ('value for key "{0}" in Histogram "{1}" '
'should be {2}').format(key, name, nice_type_name(key_type))
for key, key_type in type_checked_list_fields.iteritems():
if not key in definition:
continue
if not all(isinstance(x, key_type) for x in definition[key]):
raise ValueError, ('all values for list "{0}" in Histogram "{1}" '
'should be {2}').format(key, name, nice_type_name(key_type))
@staticmethod
def check_keys(name, definition, allowed_keys):
for key in definition.iterkeys():
if key not in allowed_keys:
raise KeyError, '%s not permitted for %s' % (key, name)
def set_bucket_parameters(self, low, high, n_buckets):
self._low = low
self._high = high
self._n_buckets = n_buckets
if whitelists is not None and self._n_buckets > 100 and type(self._n_buckets) is int:
if self._name not in whitelists['n_buckets']:
raise KeyError, ('New histogram "%s" is not permitted to have more than 100 buckets. '
'Histograms with large numbers of buckets use disproportionately high amounts of resources. '
'Contact the Telemetry team (e.g. in #telemetry) if you think an exception ought to be made.' % self._name)
@staticmethod
def boolean_flag_bucket_parameters(definition):
return (1, 2, 3)
@staticmethod
def linear_bucket_parameters(definition):
return (definition.get('low', 1),
definition['high'],
definition['n_buckets'])
@staticmethod
def enumerated_bucket_parameters(definition):
n_values = definition['n_values']
return (1, n_values, n_values + 1)
@staticmethod
def categorical_bucket_parameters(definition):
n_values = len(definition['labels'])
return (1, n_values, n_values + 1)
@staticmethod
def exponential_bucket_parameters(definition):
return (definition.get('low', 1),
definition['high'],
definition['n_buckets'])
# We support generating histograms from multiple different input files, not
# just Histograms.json. For each file's basename, we have a specific
# routine to parse that file, and return a dictionary mapping histogram
# names to histogram parameters.
def from_Histograms_json(filename):
with open(filename, 'r') as f:
try:
histograms = json.load(f, object_pairs_hook=OrderedDict)
except ValueError, e:
raise BaseException, "error parsing histograms in %s: %s" % (filename, e.message)
return histograms
def from_UseCounters_conf(filename):
return usecounters.generate_histograms(filename)
def from_nsDeprecatedOperationList(filename):
operation_regex = re.compile('^DEPRECATED_OPERATION\\(([^)]+)\\)')
histograms = collections.OrderedDict()
with open(filename, 'r') as f:
for line in f:
match = operation_regex.search(line)
if not match:
continue
op = match.group(1)
def add_counter(context):
name = 'USE_COUNTER2_DEPRECATED_%s_%s' % (op, context.upper())
histograms[name] = {
'expires_in_version': 'never',
'kind': 'boolean',
'description': 'Whether a %s used %s' % (context, op)
}
add_counter('document')
add_counter('page')
return histograms
FILENAME_PARSERS = {
'Histograms.json': from_Histograms_json,
'nsDeprecatedOperationList.h': from_nsDeprecatedOperationList,
}
# Similarly to the dance above with buildconfig, usecounters may not be
# available, so handle that gracefully.
try:
import usecounters
FILENAME_PARSERS['UseCounters.conf'] = from_UseCounters_conf
except ImportError:
pass
def from_files(filenames):
"""Return an iterator that provides a sequence of Histograms for
the histograms defined in filenames.
"""
all_histograms = OrderedDict()
for filename in filenames:
parser = FILENAME_PARSERS[os.path.basename(filename)]
histograms = parser(filename)
# OrderedDicts are important, because then the iteration order over
# the parsed histograms is stable, which makes the insertion into
# all_histograms stable, which makes ordering in generated files
# stable, which makes builds more deterministic.
if not isinstance(histograms, OrderedDict):
raise BaseException, "histogram parser didn't provide an OrderedDict"
for (name, definition) in histograms.iteritems():
if all_histograms.has_key(name):
raise DefinitionException, "duplicate histogram name %s" % name
all_histograms[name] = definition
# We require that all USE_COUNTER2_* histograms be defined in a contiguous
# block.
use_counter_indices = filter(lambda x: x[1].startswith("USE_COUNTER2_"),
enumerate(all_histograms.iterkeys()));
if use_counter_indices:
lower_bound = use_counter_indices[0][0]
upper_bound = use_counter_indices[-1][0]
n_counters = upper_bound - lower_bound + 1
if n_counters != len(use_counter_indices):
raise DefinitionException, "use counter histograms must be defined in a contiguous block"
# Check that histograms that were removed from Histograms.json etc. are also removed from the whitelists.
if whitelists is not None:
all_whitelist_entries = itertools.chain.from_iterable(whitelists.itervalues())
orphaned = set(all_whitelist_entries) - set(all_histograms.keys())
if len(orphaned) > 0:
msg = 'The following entries are orphaned and should be removed from histogram-whitelists.json: %s'
raise BaseException, msg % (', '.join(sorted(orphaned)))
for (name, definition) in all_histograms.iteritems():
yield Histogram(name, definition, strict_type_checks=True)

119
toolkit/components/telemetry/moz.build
View file

@ -1,119 +0,0 @@
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
HAS_MISC_RULE = True
include('/ipc/chromium/chromium-config.mozbuild')
FINAL_LIBRARY = 'xul'
DEFINES['MOZ_APP_VERSION'] = '"%s"' % CONFIG['MOZ_APP_VERSION']
LOCAL_INCLUDES += [
'/xpcom/build',
'/xpcom/threads',
]
SPHINX_TREES['telemetry'] = 'docs'
if CONFIG['GNU_CXX']:
CXXFLAGS += ['-Wno-error=shadow']
XPIDL_SOURCES += [
'nsITelemetry.idl',
]
XPIDL_MODULE = 'telemetry'
EXPORTS.mozilla += [
'!TelemetryEventEnums.h',
'!TelemetryHistogramEnums.h',
'!TelemetryScalarEnums.h',
'ProcessedStack.h',
'Telemetry.h',
'TelemetryComms.h',
'ThreadHangStats.h',
]
SOURCES += [
'Telemetry.cpp',
'TelemetryCommon.cpp',
'TelemetryEvent.cpp',
'TelemetryHistogram.cpp',
'TelemetryScalar.cpp',
'WebrtcTelemetry.cpp',
]
EXTRA_COMPONENTS += [
'TelemetryStartup.js',
'TelemetryStartup.manifest'
]
EXTRA_JS_MODULES += [
'GCTelemetry.jsm',
'TelemetryArchive.jsm',
'TelemetryController.jsm',
'TelemetryEnvironment.jsm',
'TelemetryLog.jsm',
'TelemetryReportingPolicy.jsm',
'TelemetrySend.jsm',
'TelemetrySession.jsm',
'TelemetryStorage.jsm',
'TelemetryTimestamps.jsm',
'TelemetryUtils.jsm',
'ThirdPartyCookieProbe.jsm',
'UITelemetry.jsm',
]
GENERATED_FILES = [
'TelemetryEventData.h',
'TelemetryEventEnums.h',
'TelemetryHistogramData.inc',
'TelemetryHistogramEnums.h',
'TelemetryScalarData.h',
'TelemetryScalarEnums.h',
]
# Generate histogram files.
histogram_files = [
'Histograms.json',
'/dom/base/UseCounters.conf',
'/dom/base/nsDeprecatedOperationList.h',
]
data = GENERATED_FILES['TelemetryHistogramData.inc']
data.script = 'gen-histogram-data.py'
data.inputs = histogram_files
enums = GENERATED_FILES['TelemetryHistogramEnums.h']
enums.script = 'gen-histogram-enum.py'
enums.inputs = histogram_files
# Generate scalar files.
scalar_files = [
'Scalars.yaml',
]
scalar_data = GENERATED_FILES['TelemetryScalarData.h']
scalar_data.script = 'gen-scalar-data.py'
scalar_data.inputs = scalar_files
scalar_enums = GENERATED_FILES['TelemetryScalarEnums.h']
scalar_enums.script = 'gen-scalar-enum.py'
scalar_enums.inputs = scalar_files
# Generate event files.
event_files = [
'Events.yaml',
]
event_data = GENERATED_FILES['TelemetryEventData.h']
event_data.script = 'gen-event-data.py'
event_data.inputs = event_files
event_enums = GENERATED_FILES['TelemetryEventEnums.h']
event_enums.script = 'gen-event-enum.py'
event_enums.inputs = event_files

469
toolkit/components/telemetry/nsITelemetry.idl
View file

@ -1,469 +0,0 @@
/* -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil; tab-width: 8 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsISupports.idl"
#include "nsIFile.idl"
[scriptable,function, uuid(3d3b9075-5549-4244-9c08-b64fefa1dd60)]
interface nsIFetchTelemetryDataCallback : nsISupports
{
void complete();
};
[scriptable, uuid(273d2dd0-6c63-475a-b864-cb65160a1909)]
interface nsITelemetry : nsISupports
{
/**
* Histogram types:
* HISTOGRAM_EXPONENTIAL - buckets increase exponentially
* HISTOGRAM_LINEAR - buckets increase linearly
* HISTOGRAM_BOOLEAN - For storing 0/1 values
* HISTOGRAM_FLAG - For storing a single value; its count is always == 1.
* HISTOGRAM_COUNT - For storing counter values without bucketing.
* HISTOGRAM_CATEGORICAL - For storing enumerated values by label.
*/
const unsigned long HISTOGRAM_EXPONENTIAL = 0;
const unsigned long HISTOGRAM_LINEAR = 1;
const unsigned long HISTOGRAM_BOOLEAN = 2;
const unsigned long HISTOGRAM_FLAG = 3;
const unsigned long HISTOGRAM_COUNT = 4;
const unsigned long HISTOGRAM_CATEGORICAL = 5;
/**
* Scalar types:
* SCALAR_COUNT - for storing a numeric value
* SCALAR_STRING - for storing a string value
* SCALAR_BOOLEAN - for storing a boolean value
*/
const unsigned long SCALAR_COUNT = 0;
const unsigned long SCALAR_STRING = 1;
const unsigned long SCALAR_BOOLEAN = 2;
/**
* Dataset types:
* DATASET_RELEASE_CHANNEL_OPTOUT - the basic dataset that is on-by-default on all channels
* DATASET_RELEASE_CHANNEL_OPTIN - the extended dataset that is opt-in on release,
* opt-out on pre-release channels.
*/
const unsigned long DATASET_RELEASE_CHANNEL_OPTOUT = 0;
const unsigned long DATASET_RELEASE_CHANNEL_OPTIN = 1;
/**
* An object containing a snapshot from all of the currently registered histograms.
* { name1: {data1}, name2:{data2}...}
* where data is consists of the following properties:
* min - Minimal bucket size
* max - Maximum bucket size
* histogram_type - HISTOGRAM_EXPONENTIAL, HISTOGRAM_LINEAR, HISTOGRAM_BOOLEAN
* or HISTOGRAM_COUNT
* counts - array representing contents of the buckets in the histogram
* ranges - an array with calculated bucket sizes
* sum - sum of the bucket contents
*/
[implicit_jscontext]
readonly attribute jsval histogramSnapshots;
/**
* Get a snapshot of the internally duplicated subsession histograms.
* @param clear Whether to clear out the subsession histograms after snapshotting.
* @return An object as histogramSnapshots, except this contains the internally duplicated histograms for subsession telemetry.
*/
[implicit_jscontext]
jsval snapshotSubsessionHistograms([optional] in boolean clear);
/**
* The amount of time, in milliseconds, that the last session took
* to shutdown. Reads as 0 to indicate failure.
*/
readonly attribute uint32_t lastShutdownDuration;
/**
* The number of failed profile lock attempts that have occurred prior to
* successfully locking the profile
*/
readonly attribute uint32_t failedProfileLockCount;
/*
* An object containing information about slow SQL statements.
*
* {
* mainThread: { "sqlString1": [<hit count>, <total time>], "sqlString2": [...], ... },
* otherThreads: { "sqlString3": [<hit count>, <total time>], "sqlString4": [...], ... }
* }
*
* where:
* mainThread: Slow statements that executed on the main thread
* otherThreads: Slow statements that executed on a non-main thread
* sqlString - String of the offending statement (see note)
* hit count - The number of times this statement required longer than the threshold time to execute
* total time - The sum of all execution times above the threshold time for this statement
*
* Note that dynamic SQL strings and SQL strings executed against addon DBs could contain private information.
* This property represents such SQL as aggregate database-level stats and the sqlString contains the database
* filename instead.
*/
[implicit_jscontext]
readonly attribute jsval slowSQL;
/*
* See slowSQL above.
*
* An object containing full strings of every slow SQL statement if toolkit.telemetry.debugSlowSql = true
* The returned SQL strings may contain private information and should not be reported to Telemetry.
*/
[implicit_jscontext]
readonly attribute jsval debugSlowSQL;
/*
* An object containing information about Webrtc related stats. For now it
* only contains local and remote ICE candidates avaiable when a Webrtc
* PeerConnection gets terminated.
*/
[implicit_jscontext]
readonly attribute jsval webrtcStats;
/**
* A number representing the highest number of concurrent threads
* reached during this session.
*/
readonly attribute uint32_t maximalNumberOfConcurrentThreads;
/*
* An array of chrome hang reports. Each element is a hang report represented
* as an object containing the hang duration, call stack PCs and information
* about modules in memory.
*/
[implicit_jscontext]
readonly attribute jsval chromeHangs;
/*
* An array of thread hang stats,
* [<thread>, <thread>, ...]
* <thread> represents a single thread,
* {"name": "<name>",
* "activity": <time>,
* "hangs": [<hang>, <hang>, ...]}
* <time> represents a histogram of time intervals in milliseconds,
* with the same format as histogramSnapshots
* <hang> represents a particular hang,
* {"stack": <stack>, "nativeStack": <stack>, "histogram": <time>}
* <stack> represents the hang's stack,
* ["<frame_0>", "<frame_1>", ...]
*/
[implicit_jscontext]
readonly attribute jsval threadHangStats;
/*
* An object with two fields: memoryMap and stacks.
* * memoryMap is a list of loaded libraries.
* * stacks is a list of stacks. Each stack is a list of pairs of the form
* [moduleIndex, offset]. The moduleIndex is an index into the memoryMap and
* offset is an offset in the library at memoryMap[moduleIndex].
* This format is used to make it easier to send the stacks to the
* symbolication server.
*/
[implicit_jscontext]
readonly attribute jsval lateWrites;
/**
* Returns an array whose values are the names of histograms defined
* in Histograms.json.
*
* @param dataset - DATASET_RELEASE_CHANNEL_OPTOUT or DATASET_RELEASE_CHANNEL_OPTIN
*/
void registeredHistograms(in uint32_t dataset,
out uint32_t count,
[retval, array, size_is(count)] out string histograms);
/**
* Create and return a histogram registered in TelemetryHistograms.h.
*
* @param id - unique identifier from TelemetryHistograms.h
* The returned object has the following functions:
* add(int) - Adds an int value to the appropriate bucket
* snapshot() - Returns a snapshot of the histogram with the same data fields as in histogramSnapshots()
* clear() - Zeros out the histogram's buckets and sum
* dataset() - identifies what dataset this is in: DATASET_RELEASE_CHANNEL_OPTOUT or ...OPTIN
*/
[implicit_jscontext]
jsval getHistogramById(in ACString id);
/*
* An object containing a snapshot from all of the currently registered keyed histograms.
* { name1: {histogramData1}, name2:{histogramData2}...}
* where the histogramData is as described in histogramSnapshots.
*/
[implicit_jscontext]
readonly attribute jsval keyedHistogramSnapshots;
/**
* Returns an array whose values are the names of histograms defined
* in Histograms.json.
*
* @param dataset - DATASET_RELEASE_CHANNEL_OPTOUT or ...OPTIN
*/
void registeredKeyedHistograms(in uint32_t dataset, out uint32_t count,
[retval, array, size_is(count)] out string histograms);
/**
* Create and return a histogram registered in TelemetryHistograms.h.
*
* @param id - unique identifier from TelemetryHistograms.h
* The returned object has the following functions:
* add(string key, [optional] int) - Add an int value to the histogram for that key. If no histogram for that key exists yet, it is created.
* snapshot([optional] string key) - If key is provided, returns a snapshot for the histogram with that key or null. If key is not provided, returns the snapshots of all the registered keys in the form {key1: snapshot1, key2: snapshot2, ...}.
* keys() - Returns an array with the string keys of the currently registered histograms
* clear() - Clears the registered histograms from this.
* dataset() - identifies what dataset this is in: DATASET_RELEASE_CHANNEL_OPTOUT or ...OPTIN
*/
[implicit_jscontext]
jsval getKeyedHistogramById(in ACString id);
/**
* A flag indicating if Telemetry can record base data (FHR data). This is true if the
* FHR data reporting service or self-support are enabled.
*
* In the unlikely event that adding a new base probe is needed, please check the data
* collection wiki at https://wiki.mozilla.org/Firefox/Data_Collection and talk to the
* Telemetry team.
*/
attribute boolean canRecordBase;
/**
* A flag indicating if Telemetry is allowed to record extended data. Returns false if
* the user hasn't opted into "extended Telemetry" on the Release channel, when the
* user has explicitly opted out of Telemetry on Nightly/Aurora/Beta or if manually
* set to false during tests.
*
* Set this to false in tests to disable gathering of extended telemetry statistics.
*/
attribute boolean canRecordExtended;
/**
* A flag indicating whether Telemetry can submit official results (for base or extended
* data). This is true on official, non-debug builds with built in support for Mozilla
* Telemetry reporting.
*/
readonly attribute boolean isOfficialTelemetry;
/** Addon telemetry hooks */
/**
* Register a histogram for an addon. Throws an error if the
* histogram name has been registered previously.
*
* @param addon_id - Unique ID of the addon
* @param name - Unique histogram name
* @param histogram_type - HISTOGRAM_EXPONENTIAL, HISTOGRAM_LINEAR,
* HISTOGRAM_BOOLEAN or HISTOGRAM_COUNT
* @param min - Minimal bucket size
* @param max - Maximum bucket size
* @param bucket_count - number of buckets in the histogram
*/
[optional_argc]
void registerAddonHistogram(in ACString addon_id, in ACString name,
in unsigned long histogram_type,
[optional] in uint32_t min,
[optional] in uint32_t max,
[optional] in uint32_t bucket_count);
/**
* Return a histogram previously registered via
* registerAddonHistogram. Throws an error if the id/name combo has
* not been registered via registerAddonHistogram.
*
* @param addon_id - Unique ID of the addon
* @param name - Registered histogram name
*
*/
[implicit_jscontext]
jsval getAddonHistogram(in ACString addon_id, in ACString name);
/**
* Delete all histograms associated with the given addon id.
*
* @param addon_id - Unique ID of the addon
*/
void unregisterAddonHistograms(in ACString addon_id);
/**
* Enable/disable recording for this histogram at runtime.
* Recording is enabled by default, unless listed at kRecordingInitiallyDisabledIDs[].
* Name must be a valid Histogram identifier, otherwise an assertion will be triggered.
*
* @param id - unique identifier from histograms.json
* @param enabled - whether or not to enable recording from now on.
*/
void setHistogramRecordingEnabled(in ACString id, in boolean enabled);
/**
* An object containing a snapshot from all of the currently
* registered addon histograms.
* { addon-id1 : data1, ... }
*
* where data is an object whose properties are the names of the
* addon's histograms and whose corresponding values are as in
* histogramSnapshots.
*/
[implicit_jscontext]
readonly attribute jsval addonHistogramSnapshots;
/**
* Read data from the previous run. After the callback is called, the last
* shutdown time is available in lastShutdownDuration and any late
* writes in lateWrites.
*/
void asyncFetchTelemetryData(in nsIFetchTelemetryDataCallback aCallback);
/**
* Get statistics of file IO reports, null, if not recorded.
*
* The statistics are returned as an object whose propoerties are the names
* of the files that have been accessed and whose corresponding values are
* arrays of size three, representing startup, normal, and shutdown stages.
* Each stage's entry is either null or an array with the layout
* [total_time, #creates, #reads, #writes, #fsyncs, #stats]
*/
[implicit_jscontext]
readonly attribute jsval fileIOReports;
/**
* Return the number of milliseconds since process start using monotonic
* timestamps (unaffected by system clock changes).
* @throws NS_ERROR_NOT_AVAILABLE if TimeStamp doesn't have the data.
*/
double msSinceProcessStart();
/**
* Adds the value to the given scalar.
*
* @param aName The scalar name.
* @param aValue The numeric value to add to the scalar. Only unsigned integers supported.
*/
[implicit_jscontext]
void scalarAdd(in ACString aName, in jsval aValue);
/**
* Sets the scalar to the given value.
*
* @param aName The scalar name.
* @param aValue The value to set the scalar to. If the type of aValue doesn't match the
* type of the scalar, the function will fail. For scalar string types, the this
* is truncated to 50 characters.
*/
[implicit_jscontext]
void scalarSet(in ACString aName, in jsval aValue);
/**
* Sets the scalar to the maximum of the current and the passed value.
*
* @param aName The scalar name.
* @param aValue The numeric value to set the scalar to. Only unsigned integers supported.
*/
[implicit_jscontext]
void scalarSetMaximum(in ACString aName, in jsval aValue);
/**
* Serializes the scalars from the given dataset to a JSON-style object and resets them.
* The returned structure looks like:
* { "group1.probe": 1, "group1.other_probe": false, ... }
*
* @param aDataset DATASET_RELEASE_CHANNEL_OPTOUT or DATASET_RELEASE_CHANNEL_OPTIN.
* @param [aClear=false] Whether to clear out the scalars after snapshotting.
*/
[implicit_jscontext, optional_argc]
jsval snapshotScalars(in uint32_t aDataset, [optional] in boolean aClear);
/**
* Adds the value to the given keyed scalar.
*
* @param aName The scalar name.
* @param aKey The key name.
* @param aValue The numeric value to add to the scalar. Only unsigned integers supported.
*/
[implicit_jscontext]
void keyedScalarAdd(in ACString aName, in AString aKey, in jsval aValue);
/**
* Sets the keyed scalar to the given value.
*
* @param aName The scalar name.
* @param aKey The key name.
* @param aValue The value to set the scalar to. If the type of aValue doesn't match the
* type of the scalar, the function will fail.
*/
[implicit_jscontext]
void keyedScalarSet(in ACString aName, in AString aKey, in jsval aValue);
/**
* Sets the keyed scalar to the maximum of the current and the passed value.
*
* @param aName The scalar name.
* @param aKey The key name.
* @param aValue The numeric value to set the scalar to. Only unsigned integers supported.
*/
[implicit_jscontext]
void keyedScalarSetMaximum(in ACString aName, in AString aKey, in jsval aValue);
/**
* Serializes the keyed scalars from the given dataset to a JSON-style object and
* resets them.
* The returned structure looks like:
* { "group1.probe": { "key_1": 2, "key_2": 1, ... }, ... }
*
* @param aDataset DATASET_RELEASE_CHANNEL_OPTOUT or DATASET_RELEASE_CHANNEL_OPTIN.
* @param [aClear=false] Whether to clear out the scalars after snapshotting.
*/
[implicit_jscontext, optional_argc]
jsval snapshotKeyedScalars(in uint32_t aDataset, [optional] in boolean aClear);
/**
* Resets all the stored scalars. This is intended to be only used in tests.
*/
void clearScalars();
/**
* Immediately sends any Telemetry batched on this process to the parent
* process. This is intended only to be used on process shutdown.
*/
void flushBatchedChildTelemetry();
/**
* Record an event in Telemetry.
*
* @param aCategory The category name.
* @param aMethod The method name.
* @param aMethod The object name.
* @param aValue An optional string value to record.
* @param aExtra An optional object of the form (string -> string).
* It should only contain registered extra keys.
*
* @throws NS_ERROR_INVALID_ARG When trying to record an unknown event.
*/
[implicit_jscontext, optional_argc]
void recordEvent(in ACString aCategory, in ACString aMethod, in ACString aObject, [optional] in jsval aValue, [optional] in jsval extra);
/**
* Serializes the recorded events to a JSON-appropriate array and optionally resets them.
* The returned structure looks like this:
* [
* // [timestamp, category, method, object, stringValue, extraValues]
* [43245, "category1", "method1", "object1", "string value", null],
* [43258, "category1", "method2", "object1", null, {"key1": "string value"}],
* ...
* ]
*
* @param aDataset DATASET_RELEASE_CHANNEL_OPTOUT or DATASET_RELEASE_CHANNEL_OPTIN.
* @param [aClear=false] Whether to clear out the scalars after snapshotting.
*/
[implicit_jscontext, optional_argc]
jsval snapshotBuiltinEvents(in uint32_t aDataset, [optional] in boolean aClear);
/**
* Resets all the stored events. This is intended to be only used in tests.
*/
void clearEvents();
};

271
toolkit/components/telemetry/parse_events.py
View file

@ -1,271 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import re
import yaml
import itertools
import datetime
import string
from shared_telemetry_utils import add_expiration_postfix
MAX_CATEGORY_NAME_LENGTH = 30
MAX_METHOD_NAME_LENGTH = 20
MAX_OBJECT_NAME_LENGTH = 20
MAX_EXTRA_KEYS_COUNT = 10
MAX_EXTRA_KEY_NAME_LENGTH = 15
IDENTIFIER_PATTERN = r'^[a-zA-Z][a-zA-Z0-9_.]+[a-zA-Z0-9]$'
DATE_PATTERN = r'^[0-9]{4}-[0-9]{2}-[0-9]{2}$'
def nice_type_name(t):
if isinstance(t, basestring):
return "string"
return t.__name__
def convert_to_cpp_identifier(s, sep):
return string.capwords(s, sep).replace(sep, "")
class OneOf:
"""This is a placeholder type for the TypeChecker below.
It signals that the checked value should match one of the following arguments
passed to the TypeChecker constructor.
"""
pass
class TypeChecker:
"""This implements a convenience type TypeChecker to make the validation code more readable."""
def __init__(self, kind, *args):
"""This takes 1-3 arguments, specifying the value type to check for.
It supports:
- atomic values, e.g.: TypeChecker(int)
- list values, e.g.: TypeChecker(list, basestring)
- dict values, e.g.: TypeChecker(dict, basestring, int)
- atomic values that can have different types, e.g.: TypeChecker(OneOf, int, date)"""
self._kind = kind
self._args = args
def check(self, key, value):
# Check fields that can be one of two different types.
if self._kind is OneOf:
if not isinstance(value, self._args[0]) and not isinstance(value, self._args[1]):
raise ValueError, "failed type check for %s - expected %s or %s, got %s" %\
(key,
nice_type_name(self._args[0]),
nice_type_name(self._args[1]),
nice_type_name(type(value)))
return
# Check basic type of value.
if not isinstance(value, self._kind):
raise ValueError, "failed type check for %s - expected %s, got %s" %\
(key,
nice_type_name(self._kind),
nice_type_name(type(value)))
# Check types of values in lists.
if self._kind is list:
if len(value) < 1:
raise ValueError, "failed check for %s - list should not be empty" % key
for x in value:
if not isinstance(x, self._args[0]):
raise ValueError, "failed type check for %s - expected list value type %s, got %s" %\
(key,
nice_type_name(self._args[0]),
nice_type_name(type(x)))
# Check types of keys and values in dictionaries.
elif self._kind is dict:
if len(value.keys()) < 1:
raise ValueError, "failed check for %s - dict should not be empty" % key
for x in value.iterkeys():
if not isinstance(x, self._args[0]):
raise ValueError, "failed dict type check for %s - expected key type %s, got %s" %\
(key,
nice_type_name(self._args[0]),
nice_type_name(type(x)))
for k,v in value.iteritems():
if not isinstance(x, self._args[1]):
raise ValueError, "failed dict type check for %s - expected value type %s for key %s, got %s" %\
(key,
nice_type_name(self._args[1]),
k,
nice_type_name(type(x)))
def type_check_event_fields(category, definition):
"""Perform a type/schema check on the event definition."""
REQUIRED_FIELDS = {
'methods': TypeChecker(list, basestring),
'objects': TypeChecker(list, basestring),
'bug_numbers': TypeChecker(list, int),
'notification_emails': TypeChecker(list, basestring),
'description': TypeChecker(basestring),
}
OPTIONAL_FIELDS = {
'release_channel_collection': TypeChecker(basestring),
'expiry_date': TypeChecker(OneOf, basestring, datetime.date),
'expiry_version': TypeChecker(basestring),
'extra_keys': TypeChecker(dict, basestring, basestring),
}
ALL_FIELDS = REQUIRED_FIELDS.copy()
ALL_FIELDS.update(OPTIONAL_FIELDS)
# Check that all the required fields are available.
missing_fields = [f for f in REQUIRED_FIELDS.keys() if f not in definition]
if len(missing_fields) > 0:
raise KeyError(category + ' - missing required fields: ' + ', '.join(missing_fields))
# Is there any unknown field?
unknown_fields = [f for f in definition.keys() if f not in ALL_FIELDS]
if len(unknown_fields) > 0:
raise KeyError(category + ' - unknown fields: ' + ', '.join(unknown_fields))
# Type-check fields.
for k,v in definition.iteritems():
ALL_FIELDS[k].check(k, v)
def string_check(category, field_name, value, min_length, max_length, regex=None):
# Length check.
if len(value) > max_length:
raise ValueError("Value '%s' for %s in %s exceeds maximum length of %d" %\
(value, field_name, category, max_length))
# Regex check.
if regex and not re.match(regex, value):
raise ValueError, 'String value for %s in %s is not matching pattern "%s": %s' % \
(field_name, category, regex, value)
class EventData:
"""A class representing one event."""
def __init__(self, category, definition):
type_check_event_fields(category, definition)
string_check(category, 'methods', definition.get('methods')[0], 1, MAX_METHOD_NAME_LENGTH, regex=IDENTIFIER_PATTERN)
string_check(category, 'objects', definition.get('objects')[0], 1, MAX_OBJECT_NAME_LENGTH, regex=IDENTIFIER_PATTERN)
# Check release_channel_collection
rcc_key = 'release_channel_collection'
rcc = definition.get(rcc_key, 'opt-in')
allowed_rcc = ["opt-in", "opt-out"]
if not rcc in allowed_rcc:
raise ValueError, "Value for %s in %s should be one of: %s" %\
(rcc_key, category, ", ".join(allowed_rcc))
# Check extra_keys.
extra_keys = definition.get('extra_keys', {})
if len(extra_keys.keys()) > MAX_EXTRA_KEYS_COUNT:
raise ValueError, "Number of extra_keys in %s exceeds limit %d" %\
(category, MAX_EXTRA_KEYS_COUNT)
for key in extra_keys.iterkeys():
string_check(category, 'extra_keys', key, 1, MAX_EXTRA_KEY_NAME_LENGTH, regex=IDENTIFIER_PATTERN)
# Check expiry.
if not 'expiry_version' in definition and not 'expiry_date' in definition:
raise KeyError, "Event in %s is missing an expiration - either expiry_version or expiry_date is required" %\
(category)
expiry_date = definition.get('expiry_date')
if expiry_date and isinstance(expiry_date, basestring) and expiry_date != 'never':
if not re.match(DATE_PATTERN, expiry_date):
raise ValueError, "Event in %s has invalid expiry_date, it should be either 'never' or match this format: %s" %\
(category, DATE_PATTERN)
# Parse into date.
definition['expiry_date'] = datetime.datetime.strptime(expiry_date, '%Y-%m-%d')
# Finish setup.
self._category = category
self._definition = definition
definition['expiry_version'] = add_expiration_postfix(definition.get('expiry_version', 'never'))
@property
def category(self):
return self._category
@property
def category_cpp(self):
# Transform e.g. category.example into CategoryExample.
return convert_to_cpp_identifier(self._category, ".")
@property
def methods(self):
return self._definition.get('methods')
@property
def objects(self):
return self._definition.get('objects')
@property
def expiry_version(self):
return self._definition.get('expiry_version')
@property
def expiry_day(self):
date = self._definition.get('expiry_date')
if not date:
return 0
if isinstance(date, basestring) and date == 'never':
return 0
# Convert date to days since UNIX epoch.
epoch = datetime.date(1970, 1, 1)
days = (date - epoch).total_seconds() / (24 * 60 * 60)
return round(days)
@property
def cpp_guard(self):
return self._definition.get('cpp_guard')
@property
def enum_labels(self):
def enum(method_name, object_name):
m = convert_to_cpp_identifier(method_name, "_")
o = convert_to_cpp_identifier(object_name, "_")
return m + '_' + o
combinations = itertools.product(self.methods, self.objects)
return [enum(t[0], t[1]) for t in combinations]
@property
def dataset(self):
"""Get the nsITelemetry constant equivalent for release_channel_collection.
"""
rcc = self._definition.get('release_channel_collection', 'opt-in')
if rcc == 'opt-out':
return 'nsITelemetry::DATASET_RELEASE_CHANNEL_OPTOUT'
else:
return 'nsITelemetry::DATASET_RELEASE_CHANNEL_OPTIN'
@property
def extra_keys(self):
return self._definition.get('extra_keys', {}).keys()
def load_events(filename):
"""Parses a YAML file containing the event definitions.
:param filename: the YAML file containing the event definitions.
:raises Exception: if the event file cannot be opened or parsed.
"""
# Parse the event definitions from the YAML file.
events = None
try:
with open(filename, 'r') as f:
events = yaml.safe_load(f)
except IOError, e:
raise Exception('Error opening ' + filename + ': ' + e.message)
except ValueError, e:
raise Exception('Error parsing events in ' + filename + ': ' + e.message)
event_list = []
# Events are defined in a fixed two-level hierarchy within the definition file.
# The first level contains the category (group name), while the second level contains the
# event definitions (e.g. "category.name: [<event definition>, ...], ...").
for category_name,category in events.iteritems():
string_check('', 'category', category_name, 1, MAX_CATEGORY_NAME_LENGTH, regex=IDENTIFIER_PATTERN)
# Make sure that the category has at least one entry in it.
if not category or len(category) == 0:
raise ValueError(category_name + ' must contain at least one entry')
for entry in category:
event_list.append(EventData(category_name, entry))
return event_list

262
toolkit/components/telemetry/parse_scalars.py
View file

@ -1,262 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import re
import yaml
from shared_telemetry_utils import add_expiration_postfix
# The map of containing the allowed scalar types and their mapping to
# nsITelemetry::SCALAR_* type constants.
SCALAR_TYPES_MAP = {
'uint': 'nsITelemetry::SCALAR_COUNT',
'string': 'nsITelemetry::SCALAR_STRING',
'boolean': 'nsITelemetry::SCALAR_BOOLEAN'
}
class ScalarType:
"""A class for representing a scalar definition."""
def __init__(self, group_name, probe_name, definition):
# Validate and set the name, so we don't need to pass it to the other
# validation functions.
self.validate_names(group_name, probe_name)
self._name = probe_name
self._group_name = group_name
# Validating the scalar definition.
self.validate_types(definition)
self.validate_values(definition)
# Everything is ok, set the rest of the data.
self._definition = definition
definition['expires'] = add_expiration_postfix(definition['expires'])
def validate_names(self, group_name, probe_name):
"""Validate the group and probe name:
- Group name must be alpha-numeric + '.', no leading/trailing digit or '.'.
- Probe name must be alpha-numeric + '_', no leading/trailing digit or '_'.
:param group_name: the name of the group the probe is in.
:param probe_name: the name of the scalar probe.
:raises ValueError: if the length of the names exceeds the limit or they don't
conform our name specification.
"""
# Enforce a maximum length on group and probe names.
MAX_NAME_LENGTH = 40
for n in [group_name, probe_name]:
if len(n) > MAX_NAME_LENGTH:
raise ValueError("Name '{}' exceeds maximum name length of {} characters."\
.format(n, MAX_NAME_LENGTH))
def check_name(name, error_msg_prefix, allowed_char_regexp):
# Check if we only have the allowed characters.
chars_regxp = r'^[a-zA-Z0-9' + allowed_char_regexp + r']+$'
if not re.search(chars_regxp, name):
raise ValueError(error_msg_prefix + " name must be alpha-numeric. Got: '{}'".format(name))
# Don't allow leading/trailing digits, '.' or '_'.
if re.search(r'(^[\d\._])|([\d\._])$', name):
raise ValueError(error_msg_prefix +
" name must not have a leading/trailing digit, a dot or underscore. Got: '{}'"\
.format(name))
check_name(group_name, 'Group', r'\.')
check_name(probe_name, 'Probe', r'_')
def validate_types(self, definition):
"""This function performs some basic sanity checks on the scalar definition:
- Checks that all the required fields are available.
- Checks that all the fields have the expected types.
:param definition: the dictionary containing the scalar properties.
:raises TypeError: if a scalar definition field is of the wrong type.
:raise KeyError: if a required field is missing or unknown fields are present.
"""
# The required and optional fields in a scalar type definition.
REQUIRED_FIELDS = {
'bug_numbers': list, # This contains ints. See LIST_FIELDS_CONTENT.
'description': basestring,
'expires': basestring,
'kind': basestring,
'notification_emails': list # This contains strings. See LIST_FIELDS_CONTENT.
}
OPTIONAL_FIELDS = {
'cpp_guard': basestring,
'release_channel_collection': basestring,
'keyed': bool
}
# The types for the data within the fields that hold lists.
LIST_FIELDS_CONTENT = {
'bug_numbers': int,
'notification_emails': basestring
}
# Concatenate the required and optional field definitions.
ALL_FIELDS = REQUIRED_FIELDS.copy()
ALL_FIELDS.update(OPTIONAL_FIELDS)
# Checks that all the required fields are available.
missing_fields = [f for f in REQUIRED_FIELDS.keys() if f not in definition]
if len(missing_fields) > 0:
raise KeyError(self._name + ' - missing required fields: ' + ', '.join(missing_fields))
# Do we have any unknown field?
unknown_fields = [f for f in definition.keys() if f not in ALL_FIELDS]
if len(unknown_fields) > 0:
raise KeyError(self._name + ' - unknown fields: ' + ', '.join(unknown_fields))
# Checks the type for all the fields.
wrong_type_names = ['{} must be {}'.format(f, ALL_FIELDS[f].__name__) \
for f in definition.keys() if not isinstance(definition[f], ALL_FIELDS[f])]
if len(wrong_type_names) > 0:
raise TypeError(self._name + ' - ' + ', '.join(wrong_type_names))
# Check that the lists are not empty and that data in the lists
# have the correct types.
list_fields = [f for f in definition if isinstance(definition[f], list)]
for field in list_fields:
# Check for empty lists.
if len(definition[field]) == 0:
raise TypeError("Field '{}' for probe '{}' must not be empty."
.format(field, self._name))
# Check the type of the list content.
broken_types =\
[not isinstance(v, LIST_FIELDS_CONTENT[field]) for v in definition[field]]
if any(broken_types):
raise TypeError("Field '{}' for probe '{}' must only contain values of type {}"
.format(field, self._name, LIST_FIELDS_CONTENT[field].__name__))
def validate_values(self, definition):
"""This function checks that the fields have the correct values.
:param definition: the dictionary containing the scalar properties.
:raises ValueError: if a scalar definition field contains an unexpected value.
"""
# Validate the scalar kind.
scalar_kind = definition.get('kind')
if scalar_kind not in SCALAR_TYPES_MAP.keys():
raise ValueError(self._name + ' - unknown scalar kind: ' + scalar_kind)
# Validate the collection policy.
collection_policy = definition.get('release_channel_collection', None)
if collection_policy and collection_policy not in ['opt-in', 'opt-out']:
raise ValueError(self._name + ' - unknown collection policy: ' + collection_policy)
# Validate the cpp_guard.
cpp_guard = definition.get('cpp_guard')
if cpp_guard and re.match(r'\W', cpp_guard):
raise ValueError(self._name + ' - invalid cpp_guard: ' + cpp_guard)
@property
def name(self):
"""Get the scalar name"""
return self._name
@property
def label(self):
"""Get the scalar label generated from the scalar and group names."""
return self._group_name + '.' + self._name
@property
def enum_label(self):
"""Get the enum label generated from the scalar and group names. This is used to
generate the enum tables."""
# The scalar name can contain informations about its hierarchy (e.g. 'a.b.scalar').
# We can't have dots in C++ enums, replace them with an underscore. Also, make the
# label upper case for consistency with the histogram enums.
return self.label.replace('.', '_').upper()
@property
def bug_numbers(self):
"""Get the list of related bug numbers"""
return self._definition['bug_numbers']
@property
def description(self):
"""Get the scalar description"""
return self._definition['description']
@property
def expires(self):
"""Get the scalar expiration"""
return self._definition['expires']
@property
def kind(self):
"""Get the scalar kind"""
return self._definition['kind']
@property
def keyed(self):
"""Boolean indicating whether this is a keyed scalar"""
return self._definition.get('keyed', False)
@property
def nsITelemetry_kind(self):
"""Get the scalar kind constant defined in nsITelemetry"""
return SCALAR_TYPES_MAP.get(self.kind)
@property
def notification_emails(self):
"""Get the list of notification emails"""
return self._definition['notification_emails']
@property
def dataset(self):
"""Get the nsITelemetry constant equivalent to the chose release channel collection
policy for the scalar.
"""
# The collection policy is optional, but we still define a default
# behaviour for it.
release_channel_collection = \
self._definition.get('release_channel_collection', 'opt-in')
return 'nsITelemetry::' + ('DATASET_RELEASE_CHANNEL_OPTOUT' \
if release_channel_collection == 'opt-out' else 'DATASET_RELEASE_CHANNEL_OPTIN')
@property
def cpp_guard(self):
"""Get the cpp guard for this scalar"""
return self._definition.get('cpp_guard')
def load_scalars(filename):
"""Parses a YAML file containing the scalar definition.
:param filename: the YAML file containing the scalars definition.
:raises Exception: if the scalar file cannot be opened or parsed.
"""
# Parse the scalar definitions from the YAML file.
scalars = None
try:
with open(filename, 'r') as f:
scalars = yaml.safe_load(f)
except IOError, e:
raise Exception('Error opening ' + filename + ': ' + e.message)
except ValueError, e:
raise Exception('Error parsing scalars in ' + filename + ': ' + e.message)
scalar_list = []
# Scalars are defined in a fixed two-level hierarchy within the definition file.
# The first level contains the group name, while the second level contains the
# probe name (e.g. "group.name: probe: ...").
for group_name in scalars:
group = scalars[group_name]
# Make sure that the group has at least one probe in it.
if not group or len(group) == 0:
raise ValueError(group_name + ' must have at least a probe in it')
for probe_name in group:
# We found a scalar type. Go ahead and parse it.
scalar_info = group[probe_name]
scalar_list.append(ScalarType(group_name, probe_name, scalar_info))
return scalar_list

41
toolkit/components/telemetry/schemas/core.schema.json
View file

@ -1,41 +0,0 @@
{
"$schema" : "http://json-schema.org/draft-04/schema#",
"type" : "object",
"name" : "core",
"properties" : {
"arch" : {
"type" : "string"
},
"clientId" : {
"type" : "string",
"pattern": "^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
},
"device" : {
"type" : "string"
},
"experiments" : {
"type" : "array",
"items" : {
"type" : "string"
}
},
"locale" : {
"type" : "string"
},
"os" : {
"type" : "string"
},
"osversion" : {
"type" : "string"
},
"seq" : {
"type" : "integer",
"minimum": 0
},
"v" : {
"type" : "integer",
"enum" : [ 1 ]
}
},
"required" : ["arch", "clientId", "device", "locale", "os", "osversion", "seq", "v"]
}

103
toolkit/components/telemetry/shared_telemetry_utils.py
View file

@ -1,103 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# This file contains utility functions shared by the scalars and the histogram generation
# scripts.
from __future__ import print_function
import re
class StringTable:
"""Manages a string table and allows C style serialization to a file."""
def __init__(self):
self.current_index = 0;
self.table = {}
def c_strlen(self, string):
"""The length of a string including the null terminating character.
:param string: the input string.
"""
return len(string) + 1
def stringIndex(self, string):
"""Returns the index in the table of the provided string. Adds the string to
the table if it's not there.
:param string: the input string.
"""
if string in self.table:
return self.table[string]
else:
result = self.current_index
self.table[string] = result
self.current_index += self.c_strlen(string)
return result
def stringIndexes(self, strings):
""" Returns a list of indexes for the provided list of strings.
Adds the strings to the table if they are not in it yet.
:param strings: list of strings to put into the table.
"""
return [self.stringIndex(s) for s in strings]
def writeDefinition(self, f, name):
"""Writes the string table to a file as a C const char array.
This writes out the string table as one single C char array for memory
size reasons, separating the individual strings with '\0' characters.
This way we can index directly into the string array and avoid the additional
storage costs for the pointers to them (and potential extra relocations for those).
:param f: the output stream.
:param name: the name of the output array.
"""
entries = self.table.items()
entries.sort(key=lambda x:x[1])
# Avoid null-in-string warnings with GCC and potentially
# overlong string constants; write everything out the long way.
def explodeToCharArray(string):
def toCChar(s):
if s == "'":
return "'\\''"
else:
return "'%s'" % s
return ", ".join(map(toCChar, string))
f.write("const char %s[] = {\n" % name)
for (string, offset) in entries:
if "*/" in string:
raise ValueError, "String in string table contains unexpected sequence '*/': %s" % string
e = explodeToCharArray(string)
if e:
f.write(" /* %5d - \"%s\" */ %s, '\\0',\n"
% (offset, string, explodeToCharArray(string)))
else:
f.write(" /* %5d - \"%s\" */ '\\0',\n" % (offset, string))
f.write("};\n\n")
def static_assert(output, expression, message):
"""Writes a C++ compile-time assertion expression to a file.
:param output: the output stream.
:param expression: the expression to check.
:param message: the string literal that will appear if the expression evaluates to
false.
"""
print("static_assert(%s, \"%s\");" % (expression, message), file=output)
def add_expiration_postfix(expiration):
""" Formats the expiration version and adds a version postfix if needed.
:param expiration: the expiration version string.
:return: the modified expiration string.
"""
if re.match(r'^[1-9][0-9]*$', expiration):
return expiration + ".0a1"
if re.match(r'^[1-9][0-9]*\.0$', expiration):
return expiration + "a1"
return expiration

1
toolkit/components/terminator/moz.build
View file

@ -8,7 +8,6 @@ EXPORTS += ['nsTerminator.h']
SOURCES += ['nsTerminator.cpp']
EXTRA_COMPONENTS += [
'nsTerminatorTelemetry.js',
'terminator.manifest',
]

248
toolkit/components/terminator/nsTerminator.cpp
View file

@ -1,5 +1,4 @@
/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -44,7 +43,6 @@
#include "mozilla/Services.h"
#include "mozilla/UniquePtr.h"
#include "mozilla/Unused.h"
#include "mozilla/Telemetry.h"
// Normally, the number of milliseconds that AsyncShutdown waits until
// it decides to crash is specified as a preference. We use the
@ -156,143 +154,6 @@ RunWatchdog(void* arg)
}
}
////////////////////////////////////////////
//
// Writer thread
//
// This nspr thread is in charge of writing to disk statistics produced by the
// watchdog thread and collected by the main thread. Note that we use a nspr
// thread rather than usual XPCOM I/O simply because we outlive XPCOM and its
// threads.
//
// Utility class, used by UniquePtr<> to close nspr files.
class PR_CloseDelete
{
public:
constexpr PR_CloseDelete() {}
PR_CloseDelete(const PR_CloseDelete& aOther)
{}
void operator()(PRFileDesc* aPtr) const
{
PR_Close(aPtr);
}
};
//
// Communication between the main thread and the writer thread.
//
// Main thread:
//
// * Whenever a shutdown step has been completed, the main thread
// obtains the number of ticks from the watchdog threads, builds
// a string representing all the data gathered so far, places
// this string in `gWriteData`, and wakes up the writer thread
// using `gWriteReady`. If `gWriteData` already contained a non-null
// pointer, this means that the writer thread is lagging behind the
// main thread, and the main thread cleans up the memory.
//
// Writer thread:
//
// * When awake, the writer thread swaps `gWriteData` to nullptr. If
// `gWriteData` contained data to write, the . If so, the writer
// thread writes the data to a file named "ShutdownDuration.json.tmp",
// then moves that file to "ShutdownDuration.json" and cleans up the
// data. If `gWriteData` contains a nullptr, the writer goes to sleep
// until it is awkened using `gWriteReady`.
//
//
// The data written by the writer thread will be read by another
// module upon the next restart and fed to Telemetry.
//
Atomic<nsCString*> gWriteData(nullptr);
PRMonitor* gWriteReady = nullptr;
void RunWriter(void* arg)
{
PR_SetCurrentThreadName("Shutdown Statistics Writer");
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(arg);
// Shutdown will generally complete before we have a chance to
// deallocate. This is not a leak.
// Setup destinationPath and tmpFilePath
nsCString destinationPath(static_cast<char*>(arg));
nsAutoCString tmpFilePath;
tmpFilePath.Append(destinationPath);
tmpFilePath.AppendLiteral(".tmp");
// Cleanup any file leftover from a previous run
Unused << PR_Delete(tmpFilePath.get());
Unused << PR_Delete(destinationPath.get());
while (true) {
//
// Check whether we have received data from the main thread.
//
// We perform the check before waiting on `gWriteReady` as we may
// have received data while we were busy writing.
//
// Also note that gWriteData may have been modified several times
// since we last checked. That's ok, we are not losing any important
// data (since we keep adding data), and we are not leaking memory
// (since the main thread deallocates any data that hasn't been
// consumed by the writer thread).
//
UniquePtr<nsCString> data(gWriteData.exchange(nullptr));
if (!data) {
// Data is not available yet.
// Wait until the main thread provides it.
PR_EnterMonitor(gWriteReady);
PR_Wait(gWriteReady, PR_INTERVAL_NO_TIMEOUT);
PR_ExitMonitor(gWriteReady);
continue;
}
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(data.get());
// Shutdown may complete before we have a chance to deallocate.
// This is not a leak.
//
// Write to a temporary file
//
// In case of any error, we simply give up. Since the data is
// hardly critical, we don't want to spend too much effort
// salvaging it.
//
UniquePtr<PRFileDesc, PR_CloseDelete>
tmpFileDesc(PR_Open(tmpFilePath.get(),
PR_WRONLY | PR_TRUNCATE | PR_CREATE_FILE,
00600));
// Shutdown may complete before we have a chance to close the file.
// This is not a leak.
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(tmpFileDesc.get());
if (tmpFileDesc == nullptr) {
break;
}
if (PR_Write(tmpFileDesc.get(), data->get(), data->Length()) == -1) {
break;
}
tmpFileDesc.reset();
//
// Rename on top of destination file.
//
// This is not sufficient to guarantee that the destination file
// will be written correctly, but, again, we don't care enough
// about the data to make more efforts.
//
if (PR_Rename(tmpFilePath.get(), destinationPath.get()) != PR_SUCCESS) {
break;
}
}
}
/**
* A step during shutdown.
*
@ -347,18 +208,12 @@ nsTerminator::SelfInit()
return NS_OK;
}
// Actually launch these threads. This takes place at the first sign of shutdown.
// Actually launch the thread. This takes place at the first sign of shutdown.
void
nsTerminator::Start()
{
MOZ_ASSERT(!mInitialized);
StartWatchdog();
#if !defined(DEBUG)
// Only allow nsTerminator to write on non-debug builds so we don't get leak warnings on
// shutdown for intentional leaks (see bug 1242084). This will be enabled again by bug
// 1255484 when 1255478 lands.
StartWriter();
#endif // !defined(DEBUG)
mInitialized = true;
}
@ -397,43 +252,6 @@ nsTerminator::StartWatchdog()
MOZ_ASSERT(watchdogThread);
}
// Prepare, allocate and start the writer thread. By design, it will never
// finish, nor be deallocated. In case of error, we degrade
// gracefully to not writing Telemetry data.
void
nsTerminator::StartWriter()
{
if (!Telemetry::CanRecordExtended()) {
return;
}
nsCOMPtr<nsIFile> profLD;
nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
getter_AddRefs(profLD));
if (NS_FAILED(rv)) {
return;
}
rv = profLD->Append(NS_LITERAL_STRING("ShutdownDuration.json"));
if (NS_FAILED(rv)) {
return;
}
nsAutoString path;
rv = profLD->GetPath(path);
if (NS_FAILED(rv)) {
return;
}
gWriteReady = PR_NewMonitor();
MOZ_LSAN_INTENTIONALLY_LEAK_OBJECT(gWriteReady); // We will never deallocate this object
PRThread* writerThread = CreateSystemThread(RunWriter,
ToNewUTF8String(path));
if (!writerThread) {
return;
}
}
NS_IMETHODIMP
nsTerminator::Observe(nsISupports *, const char *aTopic, const char16_t *)
{
@ -451,13 +269,6 @@ nsTerminator::Observe(nsISupports *, const char *aTopic, const char16_t *)
}
UpdateHeartbeat(aTopic);
#if !defined(DEBUG)
// Only allow nsTerminator to write on non-debug builds so we don't get leak warnings on
// shutdown for intentional leaks (see bug 1242084). This will be enabled again by bug
// 1255484 when 1255478 lands.
UpdateTelemetry();
#endif // !defined(DEBUG)
UpdateCrashReport(aTopic);
// Perform a little cleanup
nsCOMPtr<nsIObserverService> os = mozilla::services::GetObserverService();
@ -489,61 +300,4 @@ nsTerminator::UpdateHeartbeat(const char* aTopic)
mCurrentStep = nextStep;
}
void
nsTerminator::UpdateTelemetry()
{
if (!Telemetry::CanRecordExtended() || !gWriteReady) {
return;
}
//
// We need Telemetry data on the effective duration of each step,
// to be able to tune the time-to-crash of each of both the
// Terminator and AsyncShutdown. However, at this stage, it is too
// late to record such data into Telemetry, so we write it to disk
// and read it upon the next startup.
//
// Build JSON.
UniquePtr<nsCString> telemetryData(new nsCString());
telemetryData->AppendLiteral("{");
size_t fields = 0;
for (size_t i = 0; i < ArrayLength(sShutdownSteps); ++i) {
if (sShutdownSteps[i].mTicks < 0) {
// Ignore this field.
continue;
}
if (fields++ > 0) {
telemetryData->Append(", ");
}
telemetryData->AppendLiteral("\"");
telemetryData->Append(sShutdownSteps[i].mTopic);
telemetryData->AppendLiteral("\": ");
telemetryData->AppendInt(sShutdownSteps[i].mTicks);
}
telemetryData->AppendLiteral("}");
if (fields == 0) {
// Nothing to write
return;
}
//
// Send data to the worker thread.
//
delete gWriteData.exchange(telemetryData.release()); // Clear any data that hasn't been written yet
// In case the worker thread was sleeping, wake it up.
PR_EnterMonitor(gWriteReady);
PR_Notify(gWriteReady);
PR_ExitMonitor(gWriteReady);
}
void
nsTerminator::UpdateCrashReport(const char* aTopic)
{
/*** STUB ***/
}
} // namespace mozilla

4
toolkit/components/terminator/nsTerminator.h
View file

@ -1,5 +1,4 @@
/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-
* vim: sw=2 ts=2 et lcs=trail\:.,tab\:>~ :
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
@ -23,11 +22,8 @@ private:
nsresult SelfInit();
void Start();
void StartWatchdog();
void StartWriter();
void UpdateHeartbeat(const char* aTopic);
void UpdateTelemetry();
void UpdateCrashReport(const char* aTopic);
~nsTerminator() {}

104
toolkit/components/terminator/nsTerminatorTelemetry.js
View file

@ -1,104 +0,0 @@
/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
"use strict";
/**
* Read the data saved by nsTerminator during shutdown and feed it to the
* relevant telemetry histograms.
*/
const Cc = Components.classes;
const Ci = Components.interfaces;
const Cu = Components.utils;
const Cr = Components.results;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "OS",
"resource://gre/modules/osfile.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Promise",
"resource://gre/modules/Promise.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Task",
"resource://gre/modules/Task.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "setTimeout",
"resource://gre/modules/Timer.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Services",
"resource://gre/modules/Services.jsm");
function nsTerminatorTelemetry() {}
var HISTOGRAMS = {
"quit-application": "SHUTDOWN_PHASE_DURATION_TICKS_QUIT_APPLICATION",
"profile-change-teardown": "SHUTDOWN_PHASE_DURATION_TICKS_PROFILE_CHANGE_TEARDOWN",
"profile-before-change": "SHUTDOWN_PHASE_DURATION_TICKS_PROFILE_BEFORE_CHANGE",
"xpcom-will-shutdown": "SHUTDOWN_PHASE_DURATION_TICKS_XPCOM_WILL_SHUTDOWN",
};
nsTerminatorTelemetry.prototype = {
classID: Components.ID("{3f78ada1-cba2-442a-82dd-d5fb300ddea7}"),
_xpcom_factory: XPCOMUtils.generateSingletonFactory(nsTerminatorTelemetry),
// nsISupports
QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
// nsIObserver
observe: function DS_observe(aSubject, aTopic, aData)
{
Task.spawn(function*() {
//
// This data is hardly critical, reading it can wait for a few seconds.
//
yield new Promise(resolve => setTimeout(resolve, 3000));
let PATH = OS.Path.join(OS.Constants.Path.localProfileDir,
"ShutdownDuration.json");
let raw;
try {
raw = yield OS.File.read(PATH, { encoding: "utf-8" });
} catch (ex) {
if (!ex.becauseNoSuchFile) {
throw ex;
}
return;
}
// Let other errors be reported by Promise's error-reporting.
// Clean up
OS.File.remove(PATH);
OS.File.remove(PATH + ".tmp");
let data = JSON.parse(raw);
for (let k of Object.keys(data)) {
let id = HISTOGRAMS[k];
try {
let histogram = Services.telemetry.getHistogramById(id);
if (!histogram) {
throw new Error("Unknown histogram " + id);
}
histogram.add(Number.parseInt(data[k]));
} catch (ex) {
// Make sure that the error is reported and causes test failures,
// but otherwise, ignore it.
Promise.reject(ex);
continue;
}
}
// Inform observers that we are done.
Services.obs.notifyObservers(null,
"shutdown-terminator-telemetry-updated",
"");
});
},
};
// Module
this.NSGetFactory = XPCOMUtils.generateNSGetFactory([nsTerminatorTelemetry]);