wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-06 15:58:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

pt 1 in reviving the android build (copied from pm 28a1, wish me luck)

Browse source
This commit is contained in:
wuggy 2026-04-08 15:43:25 -07:00
commit d7788a6d6d
4249 changed files with 468189 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

16
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryConstants.java Normal file
View file

@ -0,0 +1,16 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.telemetry;
import org.mozilla.gecko.AppConstants;
public class TelemetryConstants {
// To test, set this to true & change "toolkit.telemetry.server" in about:config.
public static final boolean UPLOAD_ENABLED = AppConstants.MOZILLA_OFFICIAL; // Disabled for developer builds.
public static final String USER_AGENT =
"Firefox-Android-Telemetry/" + AppConstants.MOZ_APP_VERSION + " (" + AppConstants.MOZ_APP_UA_NAME + ")";
}

188
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryCorePingDelegate.java Normal file
View file

@ -0,0 +1,188 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry;
import android.content.Context;
import android.content.SharedPreferences;
import android.support.annotation.Nullable;
import android.support.annotation.WorkerThread;
import android.util.Log;
import org.mozilla.gecko.BrowserApp;
import org.mozilla.gecko.GeckoProfile;
import org.mozilla.gecko.GeckoSharedPrefs;
import org.mozilla.gecko.adjust.AttributionHelperListener;
import org.mozilla.gecko.telemetry.measurements.CampaignIdMeasurements;
import org.mozilla.gecko.delegates.BrowserAppDelegateWithReference;
import org.mozilla.gecko.distribution.DistributionStoreCallback;
import org.mozilla.gecko.search.SearchEngineManager;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.telemetry.measurements.SearchCountMeasurements;
import org.mozilla.gecko.telemetry.measurements.SessionMeasurements;
import org.mozilla.gecko.telemetry.pingbuilders.TelemetryCorePingBuilder;
import org.mozilla.gecko.util.StringUtils;
import org.mozilla.gecko.util.ThreadUtils;
import java.io.IOException;
/**
* An activity-lifecycle delegate for uploading the core ping.
*/
public class TelemetryCorePingDelegate extends BrowserAppDelegateWithReference
implements SearchEngineManager.SearchEngineCallback, AttributionHelperListener {
private static final String LOGTAG = StringUtils.safeSubstring(
"Gecko" + TelemetryCorePingDelegate.class.getSimpleName(), 0, 23);
private static final String PREF_IS_FIRST_RUN = "telemetry-isFirstRun";
private TelemetryDispatcher telemetryDispatcher; // lazy
private final SessionMeasurements sessionMeasurements = new SessionMeasurements();
@Override
public void onStart(final BrowserApp browserApp) {
TelemetryPreferences.initPreferenceObserver(browserApp, browserApp.getProfile().getName());
// We don't upload in onCreate because that's only called when the Activity needs to be instantiated
// and it's possible the system will never free the Activity from memory.
//
// We don't upload in onResume/onPause because that will be called each time the Activity is obscured,
// including by our own Activities/dialogs, and there is no reason to upload each time we're unobscured.
//
// We're left with onStart/onStop and we upload in onStart because onStop is not guaranteed to be called
// and we want to upload the first run ASAP (e.g. to get install data before the app may crash).
uploadPing(browserApp);
}
@Override
public void onStop(final BrowserApp browserApp) {
// We've decided to upload primarily in onStart (see note there). However, if it's the first run,
// it's possible a user used fennec and decided never to return to it again - it'd be great to get
// their session information before they decided to give it up so we upload here on first run.
//
// Caveats:
// * onStop is not guaranteed to be called in low memory conditions so it's possible we won't upload,
// but it's better than it was before.
// * Besides first run (because of this call), we can never get the user's *last* session data.
//
// If we are really interested in the user's last session data, we could consider uploading in onStop
// but it's less robust (see discussion in bug 1277091).
final SharedPreferences sharedPrefs = getSharedPreferences(browserApp);
if (sharedPrefs.getBoolean(PREF_IS_FIRST_RUN, true)) {
sharedPrefs.edit()
.putBoolean(PREF_IS_FIRST_RUN, false)
.apply();
uploadPing(browserApp);
}
}
private void uploadPing(final BrowserApp browserApp) {
final SearchEngineManager searchEngineManager = browserApp.getSearchEngineManager();
searchEngineManager.getEngine(this);
}
@Override
public void onResume(BrowserApp browserApp) {
sessionMeasurements.recordSessionStart();
}
@Override
public void onPause(BrowserApp browserApp) {
// onStart/onStop is ideal over onResume/onPause. However, onStop is not guaranteed to be called and
// dealing with that possibility adds a lot of complexity that we don't want to handle at this point.
sessionMeasurements.recordSessionEnd(browserApp);
}
@WorkerThread // via constructor
private TelemetryDispatcher getTelemetryDispatcher(final BrowserApp browserApp) {
if (telemetryDispatcher == null) {
final GeckoProfile profile = browserApp.getProfile();
final String profilePath = profile.getDir().getAbsolutePath();
final String profileName = profile.getName();
telemetryDispatcher = new TelemetryDispatcher(profilePath, profileName);
}
return telemetryDispatcher;
}
private SharedPreferences getSharedPreferences(final BrowserApp activity) {
return GeckoSharedPrefs.forProfileName(activity, activity.getProfile().getName());
}
// via SearchEngineCallback - may be called from any thread.
@Override
public void execute(@Nullable final org.mozilla.gecko.search.SearchEngine engine) {
// Don't waste resources queueing to the background thread if we don't have a reference.
if (getBrowserApp() == null) {
return;
}
// The containing method can be called from onStart: queue this work so that
// the first launch of the activity doesn't trigger profile init too early.
//
// Additionally, getAndIncrementSequenceNumber must be called from a worker thread.
ThreadUtils.postToBackgroundThread(new Runnable() {
@WorkerThread
@Override
public void run() {
final BrowserApp activity = getBrowserApp();
if (activity == null) {
return;
}
final GeckoProfile profile = activity.getProfile();
if (!TelemetryUploadService.isUploadEnabledByProfileConfig(activity, profile)) {
Log.d(LOGTAG, "Core ping upload disabled by profile config. Returning.");
return;
}
final String clientID;
try {
clientID = profile.getClientId();
} catch (final IOException e) {
Log.w(LOGTAG, "Unable to get client ID to generate core ping: " + e);
return;
}
// Each profile can have different telemetry data so we intentionally grab the shared prefs for the profile.
final SharedPreferences sharedPrefs = getSharedPreferences(activity);
final SessionMeasurements.SessionMeasurementsContainer sessionMeasurementsContainer =
sessionMeasurements.getAndResetSessionMeasurements(activity);
final TelemetryCorePingBuilder pingBuilder = new TelemetryCorePingBuilder(activity)
.setClientID(clientID)
.setDefaultSearchEngine(TelemetryCorePingBuilder.getEngineIdentifier(engine))
.setProfileCreationDate(TelemetryCorePingBuilder.getProfileCreationDate(activity, profile))
.setSequenceNumber(TelemetryCorePingBuilder.getAndIncrementSequenceNumber(sharedPrefs))
.setSessionCount(sessionMeasurementsContainer.sessionCount)
.setSessionDuration(sessionMeasurementsContainer.elapsedSeconds);
maybeSetOptionalMeasurements(activity, sharedPrefs, pingBuilder);
getTelemetryDispatcher(activity).queuePingForUpload(activity, pingBuilder);
}
});
}
private void maybeSetOptionalMeasurements(final Context context, final SharedPreferences sharedPrefs,
final TelemetryCorePingBuilder pingBuilder) {
final String distributionId = sharedPrefs.getString(DistributionStoreCallback.PREF_DISTRIBUTION_ID, null);
if (distributionId != null) {
pingBuilder.setOptDistributionID(distributionId);
}
final ExtendedJSONObject searchCounts = SearchCountMeasurements.getAndZeroSearch(sharedPrefs);
if (searchCounts.size() > 0) {
pingBuilder.setOptSearchCounts(searchCounts);
}
final String campaignId = CampaignIdMeasurements.getCampaignIdFromPrefs(context);
if (campaignId != null) {
pingBuilder.setOptCampaignId(campaignId);
}
}
@Override
public void onCampaignIdChanged(String campaignId) {
CampaignIdMeasurements.updateCampaignIdPref(getBrowserApp(), campaignId);
}
}

118
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryDispatcher.java Normal file
View file

@ -0,0 +1,118 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry;
import android.content.Context;
import android.support.annotation.WorkerThread;
import android.util.Log;
import org.mozilla.gecko.telemetry.pingbuilders.TelemetryCorePingBuilder;
import org.mozilla.gecko.telemetry.schedulers.TelemetryUploadScheduler;
import org.mozilla.gecko.telemetry.schedulers.TelemetryUploadAllPingsImmediatelyScheduler;
import org.mozilla.gecko.telemetry.stores.TelemetryJSONFilePingStore;
import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
import org.mozilla.gecko.util.ThreadUtils;
import java.io.File;
import java.io.IOException;
/**
* The entry-point for Java-based telemetry. This class handles:
* * Initializing the Stores & Schedulers.
* * Queueing upload requests for a given ping.
*
* To test Telemetry , see {@link TelemetryConstants} &
* https://wiki.mozilla.org/Mobile/Fennec/Android/Java_telemetry.
*
* The full architecture is:
*
* Fennec -(PingBuilder)-> Dispatcher -2-> Scheduler -> UploadService
* | 1 |
* Store <--------------------------
*
* The store acts as a single store of truth and contains a list of all
* pings waiting to be uploaded. The dispatcher will queue a ping to upload
* by writing it to the store. Later, the UploadService will try to upload
* this queued ping by reading directly from the store.
*
* To implement a new ping type, you should:
* 1) Implement a {@link org.mozilla.gecko.telemetry.pingbuilders.TelemetryPingBuilder} for your ping type.
* 2) Re-use a ping store in .../stores/ or implement a new one: {@link TelemetryPingStore}. The
* type of store may be affected by robustness requirements (e.g. do you have data in addition to
* pings that need to be atomically updated when a ping is stored?) and performance requirements.
* 3) Re-use an upload scheduler in .../schedulers/ or implement a new one: {@link TelemetryUploadScheduler}.
* 4) Initialize your Store & (if new) Scheduler in the constructor of this class
* 5) Add a queuePingForUpload method for your PingBuilder class (see
* {@link #queuePingForUpload(Context, TelemetryCorePingBuilder)})
* 6) In Fennec, where you want to store a ping and attempt upload, create a PingBuilder and
* pass it to the new queuePingForUpload method.
*/
public class TelemetryDispatcher {
private static final String LOGTAG = "Gecko" + TelemetryDispatcher.class.getSimpleName();
private static final String STORE_CONTAINER_DIR_NAME = "telemetry_java";
private static final String CORE_STORE_DIR_NAME = "core";
private final TelemetryJSONFilePingStore coreStore;
private final TelemetryUploadAllPingsImmediatelyScheduler uploadAllPingsImmediatelyScheduler;
@WorkerThread // via TelemetryJSONFilePingStore
public TelemetryDispatcher(final String profilePath, final String profileName) {
final String storePath = profilePath + File.separator + STORE_CONTAINER_DIR_NAME;
// There are measurements in the core ping (e.g. seq #) that would ideally be atomically updated
// when the ping is stored. However, for simplicity, we use the json store and accept the possible
// loss of data (see bug 1243585 comment 16+ for more).
coreStore = new TelemetryJSONFilePingStore(new File(storePath, CORE_STORE_DIR_NAME), profileName);
uploadAllPingsImmediatelyScheduler = new TelemetryUploadAllPingsImmediatelyScheduler();
}
private void queuePingForUpload(final Context context, final TelemetryPing ping, final TelemetryPingStore store,
final TelemetryUploadScheduler scheduler) {
final QueuePingRunnable runnable = new QueuePingRunnable(context, ping, store, scheduler);
ThreadUtils.postToBackgroundThread(runnable); // TODO: Investigate how busy this thread is. See if we want another.
}
/**
* Queues the given ping for upload and potentially schedules upload. This method can be called from any thread.
*/
public void queuePingForUpload(final Context context, final TelemetryCorePingBuilder pingBuilder) {
final TelemetryPing ping = pingBuilder.build();
queuePingForUpload(context, ping, coreStore, uploadAllPingsImmediatelyScheduler);
}
private static class QueuePingRunnable implements Runnable {
private final Context applicationContext;
private final TelemetryPing ping;
private final TelemetryPingStore store;
private final TelemetryUploadScheduler scheduler;
public QueuePingRunnable(final Context context, final TelemetryPing ping, final TelemetryPingStore store,
final TelemetryUploadScheduler scheduler) {
this.applicationContext = context.getApplicationContext();
this.ping = ping;
this.store = store;
this.scheduler = scheduler;
}
@Override
public void run() {
// We block while storing the ping so the scheduled upload is guaranteed to have the newly-stored value.
try {
store.storePing(ping);
} catch (final IOException e) {
// Don't log exception to avoid leaking profile path.
Log.e(LOGTAG, "Unable to write ping to disk. Continuing with upload attempt");
}
if (scheduler.isReadyToUpload(store)) {
scheduler.scheduleUpload(applicationContext, store);
}
}
}
}

34
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryPing.java Normal file
View file

@ -0,0 +1,34 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.telemetry;
import org.mozilla.gecko.sync.ExtendedJSONObject;
/**
* Container for telemetry data and the data necessary to upload it.
*
* The doc ID is used by a Store to manipulate its internal pings and should
* be the same value found in the urlPath.
*
* If you want to create one of these, consider extending
* {@link org.mozilla.gecko.telemetry.pingbuilders.TelemetryPingBuilder}
* or one of its descendants.
*/
public class TelemetryPing {
private final String urlPath;
private final ExtendedJSONObject payload;
private final String docID;
public TelemetryPing(final String urlPath, final ExtendedJSONObject payload, final String docID) {
this.urlPath = urlPath;
this.payload = payload;
this.docID = docID;
}
public String getURLPath() { return urlPath; }
public ExtendedJSONObject getPayload() { return payload; }
public String getDocID() { return docID; }
}

73
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryPreferences.java Normal file
View file

@ -0,0 +1,73 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.telemetry;
import android.content.Context;
import android.content.SharedPreferences;
import org.mozilla.gecko.GeckoSharedPrefs;
import org.mozilla.gecko.PrefsHelper;
import org.mozilla.gecko.PrefsHelper.PrefHandler;
import java.lang.ref.WeakReference;
/**
* Manages getting and setting any preferences related to telemetry.
*
* This class persists any Gecko preferences beyond shutdown so that these values
* can be accessed on the next run before Gecko is started as we expect Telemetry
* to run before Gecko is available.
*/
public class TelemetryPreferences {
private TelemetryPreferences() {}
private static final String GECKO_PREF_SERVER_URL = "toolkit.telemetry.server";
private static final String SHARED_PREF_SERVER_URL = "telemetry-serverUrl";
// Defaults are a mirror of about:config defaults so we can access them before Gecko is available.
private static final String DEFAULT_SERVER_URL = "https://incoming.telemetry.mozilla.org";
private static final String[] OBSERVED_PREFS = {
GECKO_PREF_SERVER_URL,
};
public static String getServerSchemeHostPort(final Context context, final String profileName) {
return getSharedPrefs(context, profileName).getString(SHARED_PREF_SERVER_URL, DEFAULT_SERVER_URL);
}
public static void initPreferenceObserver(final Context context, final String profileName) {
final PrefHandler prefHandler = new TelemetryPrefHandler(context, profileName);
PrefsHelper.addObserver(OBSERVED_PREFS, prefHandler); // gets preference value when gecko starts.
}
private static SharedPreferences getSharedPrefs(final Context context, final String profileName) {
return GeckoSharedPrefs.forProfileName(context, profileName);
}
private static class TelemetryPrefHandler extends PrefsHelper.PrefHandlerBase {
private final WeakReference<Context> contextWeakReference;
private final String profileName;
private TelemetryPrefHandler(final Context context, final String profileName) {
contextWeakReference = new WeakReference<>(context);
this.profileName = profileName;
}
@Override
public void prefValue(final String pref, final String value) {
final Context context = contextWeakReference.get();
if (context == null) {
return;
}
if (!pref.equals(GECKO_PREF_SERVER_URL)) {
throw new IllegalStateException("Unknown preference: " + pref);
}
getSharedPrefs(context, profileName).edit()
.putString(SHARED_PREF_SERVER_URL, value)
.apply();
}
}
}

347
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java Normal file
View file

@ -0,0 +1,347 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.telemetry;
import android.app.IntentService;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
import ch.boye.httpclientandroidlib.HttpHeaders;
import ch.boye.httpclientandroidlib.HttpResponse;
import ch.boye.httpclientandroidlib.client.ClientProtocolException;
import ch.boye.httpclientandroidlib.client.methods.HttpRequestBase;
import ch.boye.httpclientandroidlib.impl.client.DefaultHttpClient;
import org.mozilla.gecko.GeckoProfile;
import org.mozilla.gecko.preferences.GeckoPreferences;
import org.mozilla.gecko.restrictions.Restrictable;
import org.mozilla.gecko.restrictions.Restrictions;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.sync.net.BaseResource;
import org.mozilla.gecko.sync.net.BaseResourceDelegate;
import org.mozilla.gecko.sync.net.Resource;
import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
import org.mozilla.gecko.util.DateUtil;
import org.mozilla.gecko.util.NetworkUtils;
import org.mozilla.gecko.util.StringUtils;
import java.io.IOException;
import java.net.URISyntaxException;
import java.security.GeneralSecurityException;
import java.util.Calendar;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
/**
* The service that handles retrieving a list of telemetry pings to upload from the given
* {@link TelemetryPingStore}, uploading those payloads to the associated server, and reporting
* back to the Store which uploads were a success.
*/
public class TelemetryUploadService extends IntentService {
private static final String LOGTAG = StringUtils.safeSubstring("Gecko" + TelemetryUploadService.class.getSimpleName(), 0, 23);
private static final String WORKER_THREAD_NAME = LOGTAG + "Worker";
public static final String ACTION_UPLOAD = "upload";
public static final String EXTRA_STORE = "store";
// TelemetryUploadService can run in a background thread so for future proofing, we set it volatile.
private static volatile boolean isDisabled = false;
public static void setDisabled(final boolean isDisabled) {
TelemetryUploadService.isDisabled = isDisabled;
if (isDisabled) {
Log.d(LOGTAG, "Telemetry upload disabled (env var?");
}
}
public TelemetryUploadService() {
super(WORKER_THREAD_NAME);
// Intent redelivery can fail hard (e.g. we OOM as we try to upload, the Intent gets redelivered, repeat)
// so for simplicity, we avoid it. We expect the upload service to eventually get called again by the caller.
setIntentRedelivery(false);
}
/**
* Handles a ping with the mandatory extras:
* * EXTRA_STORE: A {@link TelemetryPingStore} where the pings to upload are located
*/
@Override
public void onHandleIntent(final Intent intent) {
Log.d(LOGTAG, "Service started");
if (!isReadyToUpload(this, intent)) {
return;
}
final TelemetryPingStore store = intent.getParcelableExtra(EXTRA_STORE);
final boolean wereAllUploadsSuccessful = uploadPendingPingsFromStore(this, store);
store.maybePrunePings();
Log.d(LOGTAG, "Service finished: upload and prune attempts completed");
if (!wereAllUploadsSuccessful) {
// If we had an upload failure, we should stop the IntentService and drop any
// pending Intents in the queue so we don't waste resources (e.g. battery)
// trying to upload when there's likely to be another connection failure.
Log.d(LOGTAG, "Clearing Intent queue due to connection failures");
stopSelf();
}
}
/**
* @return true if all pings were uploaded successfully, false otherwise.
*/
private static boolean uploadPendingPingsFromStore(final Context context, final TelemetryPingStore store) {
final List<TelemetryPing> pingsToUpload = store.getAllPings();
if (pingsToUpload.isEmpty()) {
return true;
}
final String serverSchemeHostPort = TelemetryPreferences.getServerSchemeHostPort(context, store.getProfileName());
final HashSet<String> successfulUploadIDs = new HashSet<>(pingsToUpload.size()); // used for side effects.
final PingResultDelegate delegate = new PingResultDelegate(successfulUploadIDs);
for (final TelemetryPing ping : pingsToUpload) {
// TODO: It'd be great to re-use the same HTTP connection for each upload request.
delegate.setDocID(ping.getDocID());
final String url = serverSchemeHostPort + "/" + ping.getURLPath();
uploadPayload(url, ping.getPayload(), delegate);
// There are minimal gains in trying to upload if we already failed one attempt.
if (delegate.hadConnectionError()) {
break;
}
}
final boolean wereAllUploadsSuccessful = !delegate.hadConnectionError();
if (wereAllUploadsSuccessful) {
// We don't log individual successful uploads to avoid log spam.
Log.d(LOGTAG, "Telemetry upload success!");
}
store.onUploadAttemptComplete(successfulUploadIDs);
return wereAllUploadsSuccessful;
}
private static void uploadPayload(final String url, final ExtendedJSONObject payload, final ResultDelegate delegate) {
final BaseResource resource;
try {
resource = new BaseResource(url);
} catch (final URISyntaxException e) {
Log.w(LOGTAG, "URISyntaxException for server URL when creating BaseResource: returning.");
return;
}
delegate.setResource(resource);
resource.delegate = delegate;
resource.setShouldCompressUploadedEntity(true);
resource.setShouldChunkUploadsHint(false); // Telemetry servers don't support chunking.
// We're in a background thread so we don't have any reason to do this asynchronously.
// If we tried, onStartCommand would return and IntentService might stop itself before we finish.
resource.postBlocking(payload);
}
private static boolean isReadyToUpload(final Context context, final Intent intent) {
// Sanity check: is upload enabled? Generally, the caller should check this before starting the service.
// Since we don't have the profile here, we rely on the caller to check the enabled state for the profile.
if (!isUploadEnabledByAppConfig(context)) {
Log.w(LOGTAG, "Upload is not available by configuration; returning");
return false;
}
if (!NetworkUtils.isConnected(context)) {
Log.w(LOGTAG, "Network is not connected; returning");
return false;
}
if (!isIntentValid(intent)) {
Log.w(LOGTAG, "Received invalid Intent; returning");
return false;
}
if (!ACTION_UPLOAD.equals(intent.getAction())) {
Log.w(LOGTAG, "Unknown action: " + intent.getAction() + ". Returning");
return false;
}
return true;
}
/**
* Determines if the telemetry upload feature is enabled via the application configuration. Prefer to use
* {@link #isUploadEnabledByProfileConfig(Context, GeckoProfile)} if the profile is available as it takes into
* account more information.
*
* You may wish to also check if the network is connected when calling this method.
*
* Note that this method logs debug statements when upload is disabled.
*/
public static boolean isUploadEnabledByAppConfig(final Context context) {
if (!TelemetryConstants.UPLOAD_ENABLED) {
Log.d(LOGTAG, "Telemetry upload feature is compile-time disabled");
return false;
}
if (isDisabled) {
Log.d(LOGTAG, "Telemetry upload feature is disabled by intent (in testing?)");
return false;
}
if (!GeckoPreferences.getBooleanPref(context, GeckoPreferences.PREFS_HEALTHREPORT_UPLOAD_ENABLED, true)) {
Log.d(LOGTAG, "Telemetry upload opt-out");
return false;
}
if (Restrictions.isRestrictedProfile(context) &&
!Restrictions.isAllowed(context, Restrictable.HEALTH_REPORT)) {
Log.d(LOGTAG, "Telemetry upload feature disabled by admin profile");
return false;
}
return true;
}
/**
* Determines if the telemetry upload feature is enabled via profile & application level configurations. This is the
* preferred method.
*
* You may wish to also check if the network is connected when calling this method.
*
* Note that this method logs debug statements when upload is disabled.
*/
public static boolean isUploadEnabledByProfileConfig(final Context context, final GeckoProfile profile) {
if (profile.inGuestMode()) {
Log.d(LOGTAG, "Profile is in guest mode");
return false;
}
return isUploadEnabledByAppConfig(context);
}
private static boolean isIntentValid(final Intent intent) {
// Intent can be null. Bug 1025937.
if (intent == null) {
Log.d(LOGTAG, "Received null intent");
return false;
}
if (intent.getParcelableExtra(EXTRA_STORE) == null) {
Log.d(LOGTAG, "Received invalid store in Intent");
return false;
}
return true;
}
/**
* Logs on success & failure and appends the set ID to the given Set on success.
*
* Note: you *must* set the ping ID before attempting upload or we'll throw!
*
* We use mutation on the set ID and the successful upload array to avoid object allocation.
*/
private static class PingResultDelegate extends ResultDelegate {
// We persist pings and don't need to worry about losing data so we keep these
// durations short to save resources (e.g. battery).
private static final int SOCKET_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(30);
private static final int CONNECTION_TIMEOUT_MILLIS = (int) TimeUnit.SECONDS.toMillis(30);
/** The store ID of the ping currently being uploaded. Use {@link #getDocID()} to access it. */
private String docID = null;
private final Set<String> successfulUploadIDs;
private boolean hadConnectionError = false;
public PingResultDelegate(final Set<String> successfulUploadIDs) {
super();
this.successfulUploadIDs = successfulUploadIDs;
}
@Override
public int socketTimeout() {
return SOCKET_TIMEOUT_MILLIS;
}
@Override
public int connectionTimeout() {
return CONNECTION_TIMEOUT_MILLIS;
}
private String getDocID() {
if (docID == null) {
throw new IllegalStateException("Expected ping ID to have been updated before retrieval");
}
return docID;
}
public void setDocID(final String id) {
docID = id;
}
@Override
public String getUserAgent() {
return TelemetryConstants.USER_AGENT;
}
@Override
public void handleHttpResponse(final HttpResponse response) {
final int status = response.getStatusLine().getStatusCode();
switch (status) {
case 200:
case 201:
successfulUploadIDs.add(getDocID());
break;
default:
Log.w(LOGTAG, "Telemetry upload failure. HTTP status: " + status);
hadConnectionError = true;
}
}
@Override
public void handleHttpProtocolException(final ClientProtocolException e) {
// We don't log the exception to prevent leaking user data.
Log.w(LOGTAG, "HttpProtocolException when trying to upload telemetry");
hadConnectionError = true;
}
@Override
public void handleHttpIOException(final IOException e) {
// We don't log the exception to prevent leaking user data.
Log.w(LOGTAG, "HttpIOException when trying to upload telemetry");
hadConnectionError = true;
}
@Override
public void handleTransportException(final GeneralSecurityException e) {
// We don't log the exception to prevent leaking user data.
Log.w(LOGTAG, "Transport exception when trying to upload telemetry");
hadConnectionError = true;
}
private boolean hadConnectionError() {
return hadConnectionError;
}
@Override
public void addHeaders(final HttpRequestBase request, final DefaultHttpClient client) {
super.addHeaders(request, client);
request.addHeader(HttpHeaders.DATE, DateUtil.getDateInHTTPFormat(Calendar.getInstance().getTime()));
}
}
/**
* A hack because I want to set the resource after the Delegate is constructed.
* Be sure to call {@link #setResource(Resource)}!
*/
private static abstract class ResultDelegate extends BaseResourceDelegate {
public ResultDelegate() {
super(null);
}
protected void setResource(final Resource resource) {
this.resource = resource;
}
}
}

37
mobile/android/base/java/org/mozilla/gecko/telemetry/measurements/CampaignIdMeasurements.java Normal file
View file

@ -0,0 +1,37 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.measurements;
import android.content.Context;
import android.support.annotation.NonNull;
import android.text.TextUtils;
import org.mozilla.gecko.GeckoSharedPrefs;
import org.mozilla.gecko.adjust.AttributionHelperListener;
/**
* A class to retrieve and store the campaign Id pref that is used when the Adjust SDK gives us
* new attribution from the {@link AttributionHelperListener}.
*/
public class CampaignIdMeasurements {
private static final String PREF_CAMPAIGN_ID = "measurements-campaignId";
public static String getCampaignIdFromPrefs(@NonNull final Context context) {
return GeckoSharedPrefs.forProfile(context)
.getString(PREF_CAMPAIGN_ID, null);
}
public static void updateCampaignIdPref(@NonNull final Context context, @NonNull final String campaignId) {
if (TextUtils.isEmpty(campaignId)) {
return;
}
GeckoSharedPrefs.forProfile(context)
.edit()
.putString(PREF_CAMPAIGN_ID, campaignId)
.apply();
}
}

100
mobile/android/base/java/org/mozilla/gecko/telemetry/measurements/SearchCountMeasurements.java Normal file
View file

@ -0,0 +1,100 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.measurements;
import android.content.SharedPreferences;
import android.support.annotation.NonNull;
import android.support.annotation.VisibleForTesting;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
/**
* A place to store and retrieve the number of times a user has searched with a specific engine from a
* specific location. This is designed for use as a telemetry core ping measurement.
*
* The implementation works by storing a preference for each engine-location pair and incrementing them
* each time {@link #incrementSearch(SharedPreferences, String, String)} is called. In order to
* retrieve the full set of keys later, we store all the available key names in another preference.
*
* When we retrieve the keys in {@link #getAndZeroSearch(SharedPreferences)} (using the set of keys
* preference), the values saved to the preferences are returned and the preferences are removed
* (i.e. zeroed) from Shared Preferences. The reason we remove the preferences (rather than actually
* zeroing them) is to avoid bloating shared preferences if 1) the set of engines ever changes or
* 2) we remove this feature.
*
* Since we increment a value on each successive search, which doesn't take up more space, we don't
* have to worry about using excess disk space if the measurements are never zeroed (e.g. telemetry
* upload is disabled). In the worst case, we overflow the integer and may return negative values.
*
* This class is thread-safe by locking access to its public methods. When this class was written, incrementing &
* retrieval were called from multiple threads so rather than enforcing the callers keep their threads straight, it
* was simpler to lock all access.
*/
public class SearchCountMeasurements {
/** The set of "engine + where" keys we've stored; used for retrieving stored engines. */
@VisibleForTesting static final String PREF_SEARCH_KEYSET = "measurements-search-count-keyset";
private static final String PREF_SEARCH_PREFIX = "measurements-search-count-engine-"; // + "engine.where"
private SearchCountMeasurements() {}
public static synchronized void incrementSearch(@NonNull final SharedPreferences prefs,
@NonNull final String engineIdentifier, @NonNull final String where) {
final String engineWhereStr = engineIdentifier + "." + where;
final String key = getEngineSearchCountKey(engineWhereStr);
final int count = prefs.getInt(key, 0);
prefs.edit().putInt(key, count + 1).apply();
unionKeyToSearchKeyset(prefs, engineWhereStr);
}
/**
* @param key Engine of the form, "engine.where"
*/
private static void unionKeyToSearchKeyset(@NonNull final SharedPreferences prefs, @NonNull final String key) {
final Set<String> keysFromPrefs = prefs.getStringSet(PREF_SEARCH_KEYSET, Collections.<String>emptySet());
if (keysFromPrefs.contains(key)) {
return;
}
// String set returned by shared prefs cannot be modified so we copy.
final Set<String> keysToSave = new HashSet<>(keysFromPrefs);
keysToSave.add(key);
prefs.edit().putStringSet(PREF_SEARCH_KEYSET, keysToSave).apply();
}
/**
* Gets and zeroes search counts.
*
* We return ExtendedJSONObject for now because that's the format needed by the core telemetry ping.
*/
public static synchronized ExtendedJSONObject getAndZeroSearch(@NonNull final SharedPreferences prefs) {
final ExtendedJSONObject out = new ExtendedJSONObject();
final SharedPreferences.Editor editor = prefs.edit();
final Set<String> keysFromPrefs = prefs.getStringSet(PREF_SEARCH_KEYSET, Collections.<String>emptySet());
for (final String engineWhereStr : keysFromPrefs) {
final String key = getEngineSearchCountKey(engineWhereStr);
out.put(engineWhereStr, prefs.getInt(key, 0));
editor.remove(key);
}
editor.remove(PREF_SEARCH_KEYSET)
.apply();
return out;
}
/**
* @param engineWhereStr string of the form "engine.where"
* @return the key for the engines' search counts in shared preferences
*/
@VisibleForTesting static String getEngineSearchCountKey(final String engineWhereStr) {
return PREF_SEARCH_PREFIX + engineWhereStr;
}
}

99
mobile/android/base/java/org/mozilla/gecko/telemetry/measurements/SessionMeasurements.java Normal file
View file

@ -0,0 +1,99 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.measurements;
import android.content.Context;
import android.content.SharedPreferences;
import android.support.annotation.UiThread;
import android.support.annotation.VisibleForTesting;
import org.mozilla.gecko.GeckoSharedPrefs;
import java.util.concurrent.TimeUnit;
/**
* A class to measure the number of user sessions & their durations. It was created for use with the
* telemetry core ping. A session is the time between {@link #recordSessionStart()} and
* {@link #recordSessionEnd(Context)}.
*
* This class is thread-safe, provided the thread annotations are followed. Under the hood, this class uses
* SharedPreferences & because there is no atomic getAndSet operation, we synchronize access to it.
*/
public class SessionMeasurements {
@VisibleForTesting static final String PREF_SESSION_COUNT = "measurements-session-count";
@VisibleForTesting static final String PREF_SESSION_DURATION = "measurements-session-duration";
private boolean sessionStarted = false;
private long timeAtSessionStartNano = -1;
@UiThread // we assume this will be called on the same thread as session end so we don't have to synchronize sessionStarted.
public void recordSessionStart() {
if (sessionStarted) {
throw new IllegalStateException("Trying to start session but it is already started");
}
sessionStarted = true;
timeAtSessionStartNano = getSystemTimeNano();
}
@UiThread // we assume this will be called on the same thread as session start so we don't have to synchronize sessionStarted.
public void recordSessionEnd(final Context context) {
if (!sessionStarted) {
throw new IllegalStateException("Expected session to be started before session end is called");
}
sessionStarted = false;
final long sessionElapsedSeconds = TimeUnit.NANOSECONDS.toSeconds(getSystemTimeNano() - timeAtSessionStartNano);
final SharedPreferences sharedPrefs = getSharedPreferences(context);
synchronized (this) {
final int sessionCount = sharedPrefs.getInt(PREF_SESSION_COUNT, 0);
final long totalElapsedSeconds = sharedPrefs.getLong(PREF_SESSION_DURATION, 0);
sharedPrefs.edit()
.putInt(PREF_SESSION_COUNT, sessionCount + 1)
.putLong(PREF_SESSION_DURATION, totalElapsedSeconds + sessionElapsedSeconds)
.apply();
}
}
/**
* Gets the session measurements since the last time the measurements were last retrieved.
*/
public synchronized SessionMeasurementsContainer getAndResetSessionMeasurements(final Context context) {
final SharedPreferences sharedPrefs = getSharedPreferences(context);
final int sessionCount = sharedPrefs.getInt(PREF_SESSION_COUNT, 0);
final long totalElapsedSeconds = sharedPrefs.getLong(PREF_SESSION_DURATION, 0);
sharedPrefs.edit()
.putInt(PREF_SESSION_COUNT, 0)
.putLong(PREF_SESSION_DURATION, 0)
.apply();
return new SessionMeasurementsContainer(sessionCount, totalElapsedSeconds);
}
@VisibleForTesting SharedPreferences getSharedPreferences(final Context context) {
return GeckoSharedPrefs.forProfile(context);
}
/**
* Returns (roughly) the system uptime in nanoseconds. A less coupled implementation would
* take this value from the caller of recordSession*, however, we do this internally to ensure
* the caller uses both a time system consistent between the start & end calls and uses the
* appropriate time system (i.e. not wall time, which can change when the clock is changed).
*/
@VisibleForTesting long getSystemTimeNano() { // TODO: necessary?
return System.nanoTime();
}
public static final class SessionMeasurementsContainer {
/** The number of sessions. */
public final int sessionCount;
/** The number of seconds elapsed in ALL sessions included in {@link #sessionCount}. */
public final long elapsedSeconds;
private SessionMeasurementsContainer(final int sessionCount, final long elapsedSeconds) {
this.sessionCount = sessionCount;
this.elapsedSeconds = elapsedSeconds;
}
}
}

247
mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryCorePingBuilder.java Normal file
View file

@ -0,0 +1,247 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.pingbuilders;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.Build;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.annotation.WorkerThread;
import android.text.TextUtils;
import android.util.Log;
import org.mozilla.gecko.AppConstants;
import org.mozilla.gecko.GeckoProfile;
import org.mozilla.gecko.Locales;
import org.mozilla.gecko.search.SearchEngine;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.telemetry.TelemetryPing;
import org.mozilla.gecko.util.DateUtil;
import org.mozilla.gecko.Experiments;
import org.mozilla.gecko.util.StringUtils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
/**
* Builds a {@link TelemetryPing} representing a core ping.
*
* See https://gecko.readthedocs.org/en/latest/toolkit/components/telemetry/telemetry/core-ping.html
* for details on the core ping.
*/
public class TelemetryCorePingBuilder extends TelemetryPingBuilder {
private static final String LOGTAG = StringUtils.safeSubstring(TelemetryCorePingBuilder.class.getSimpleName(), 0, 23);
// For legacy reasons, this preference key is not namespaced with "core".
private static final String PREF_SEQ_COUNT = "telemetry-seqCount";
private static final String NAME = "core";
private static final int VERSION_VALUE = 7; // For version history, see toolkit/components/telemetry/docs/core-ping.rst
private static final String OS_VALUE = "Android";
private static final String ARCHITECTURE = "arch";
private static final String CAMPAIGN_ID = "campaignId";
private static final String CLIENT_ID = "clientId";
private static final String DEFAULT_SEARCH_ENGINE = "defaultSearch";
private static final String DEVICE = "device";
private static final String DISTRIBUTION_ID = "distributionId";
private static final String EXPERIMENTS = "experiments";
private static final String LOCALE = "locale";
private static final String OS_ATTR = "os";
private static final String OS_VERSION = "osversion";
private static final String PING_CREATION_DATE = "created";
private static final String PROFILE_CREATION_DATE = "profileDate";
private static final String SEARCH_COUNTS = "searches";
private static final String SEQ = "seq";
private static final String SESSION_COUNT = "sessions";
private static final String SESSION_DURATION = "durations";
private static final String TIMEZONE_OFFSET = "tz";
private static final String VERSION_ATTR = "v";
public TelemetryCorePingBuilder(final Context context) {
initPayloadConstants(context);
}
private void initPayloadConstants(final Context context) {
payload.put(VERSION_ATTR, VERSION_VALUE);
payload.put(OS_ATTR, OS_VALUE);
// We limit the device descriptor to 32 characters because it can get long. We give fewer characters to the
// manufacturer because we're less likely to have manufacturers with similar names than we are for a
// manufacturer to have two devices with the similar names (e.g. Galaxy S6 vs. Galaxy Note 6).
final String deviceDescriptor =
StringUtils.safeSubstring(Build.MANUFACTURER, 0, 12) + '-' + StringUtils.safeSubstring(Build.MODEL, 0, 19);
final Calendar nowCalendar = Calendar.getInstance();
final DateFormat pingCreationDateFormat = new SimpleDateFormat("yyyy-MM-dd", Locale.US);
payload.put(ARCHITECTURE, AppConstants.ANDROID_CPU_ARCH);
payload.put(DEVICE, deviceDescriptor);
payload.put(LOCALE, Locales.getLanguageTag(Locale.getDefault()));
payload.put(OS_VERSION, Integer.toString(Build.VERSION.SDK_INT)); // A String for cross-platform reasons.
payload.put(PING_CREATION_DATE, pingCreationDateFormat.format(nowCalendar.getTime()));
payload.put(TIMEZONE_OFFSET, DateUtil.getTimezoneOffsetInMinutesForGivenDate(nowCalendar));
payload.putArray(EXPERIMENTS, Experiments.getActiveExperiments(context));
}
@Override
public String getDocType() {
return NAME;
}
@Override
public String[] getMandatoryFields() {
return new String[] {
ARCHITECTURE,
CLIENT_ID,
DEFAULT_SEARCH_ENGINE,
DEVICE,
LOCALE,
OS_ATTR,
OS_VERSION,
PING_CREATION_DATE,
PROFILE_CREATION_DATE,
SEQ,
TIMEZONE_OFFSET,
VERSION_ATTR,
};
}
public TelemetryCorePingBuilder setClientID(@NonNull final String clientID) {
if (clientID == null) {
throw new IllegalArgumentException("Expected non-null clientID");
}
payload.put(CLIENT_ID, clientID);
return this;
}
/**
* @param engine the default search engine identifier, or null if there is an error.
*/
public TelemetryCorePingBuilder setDefaultSearchEngine(@Nullable final String engine) {
if (engine != null && engine.isEmpty()) {
throw new IllegalArgumentException("Received empty string. Expected identifier or null.");
}
payload.put(DEFAULT_SEARCH_ENGINE, engine);
return this;
}
public TelemetryCorePingBuilder setOptDistributionID(@NonNull final String distributionID) {
if (distributionID == null) {
throw new IllegalArgumentException("Expected non-null distribution ID");
}
payload.put(DISTRIBUTION_ID, distributionID);
return this;
}
/**
* @param searchCounts non-empty JSON with {"engine.where": <int-count>}
*/
public TelemetryCorePingBuilder setOptSearchCounts(@NonNull final ExtendedJSONObject searchCounts) {
if (searchCounts == null) {
throw new IllegalStateException("Expected non-null search counts");
} else if (searchCounts.size() == 0) {
throw new IllegalStateException("Expected non-empty search counts");
}
payload.put(SEARCH_COUNTS, searchCounts);
return this;
}
public TelemetryCorePingBuilder setOptCampaignId(final String campaignId) {
if (campaignId == null) {
throw new IllegalStateException("Expected non-null campaign ID.");
}
payload.put(CAMPAIGN_ID, campaignId);
return this;
}
/**
* @param date The profile creation date in days to the unix epoch (not millis!), or null if there is an error.
*/
public TelemetryCorePingBuilder setProfileCreationDate(@Nullable final Long date) {
if (date != null && date < 0) {
throw new IllegalArgumentException("Expect positive date value. Received: " + date);
}
payload.put(PROFILE_CREATION_DATE, date);
return this;
}
/**
* @param seq a positive sequence number.
*/
public TelemetryCorePingBuilder setSequenceNumber(final int seq) {
if (seq < 0) {
// Since this is an increasing value, it's possible we can overflow into negative values and get into a
// crash loop so we don't crash on invalid arg - we can investigate if we see negative values on the server.
Log.w(LOGTAG, "Expected positive sequence number. Received: " + seq);
}
payload.put(SEQ, seq);
return this;
}
public TelemetryCorePingBuilder setSessionCount(final int sessionCount) {
if (sessionCount < 0) {
// Since this is an increasing value, it's possible we can overflow into negative values and get into a
// crash loop so we don't crash on invalid arg - we can investigate if we see negative values on the server.
Log.w(LOGTAG, "Expected positive session count. Received: " + sessionCount);
}
payload.put(SESSION_COUNT, sessionCount);
return this;
}
public TelemetryCorePingBuilder setSessionDuration(final long sessionDuration) {
if (sessionDuration < 0) {
// Since this is an increasing value, it's possible we can overflow into negative values and get into a
// crash loop so we don't crash on invalid arg - we can investigate if we see negative values on the server.
Log.w(LOGTAG, "Expected positive session duration. Received: " + sessionDuration);
}
payload.put(SESSION_DURATION, sessionDuration);
return this;
}
/**
* Gets the sequence number from shared preferences and increments it in the prefs. This method
* is not thread safe.
*/
@WorkerThread // synchronous shared prefs write.
public static int getAndIncrementSequenceNumber(final SharedPreferences sharedPrefsForProfile) {
final int seq = sharedPrefsForProfile.getInt(PREF_SEQ_COUNT, 1);
sharedPrefsForProfile.edit().putInt(PREF_SEQ_COUNT, seq + 1).apply();
return seq;
}
/**
* @return the profile creation date in the format expected by
* {@link TelemetryCorePingBuilder#setProfileCreationDate(Long)}.
*/
@WorkerThread
public static Long getProfileCreationDate(final Context context, final GeckoProfile profile) {
final long profileMillis = profile.getAndPersistProfileCreationDate(context);
if (profileMillis < 0) {
return null;
}
return (long) Math.floor((double) profileMillis / TimeUnit.DAYS.toMillis(1));
}
/**
* @return the search engine identifier in the format expected by the core ping.
*/
@Nullable
public static String getEngineIdentifier(@Nullable final SearchEngine searchEngine) {
if (searchEngine == null) {
return null;
}
final String identifier = searchEngine.getIdentifier();
return TextUtils.isEmpty(identifier) ? null : identifier;
}
}

87
mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryPingBuilder.java Normal file
View file

@ -0,0 +1,87 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.pingbuilders;
import org.mozilla.gecko.AppConstants;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.telemetry.TelemetryPing;
import java.util.Set;
import java.util.UUID;
/**
* A generic Builder for {@link TelemetryPing} instances. Each overriding class is
* expected to create a specific type of ping (e.g. "core").
*
* This base class handles the common ping operations under the hood:
* * Validating mandatory fields
* * Forming the server url
*/
abstract class TelemetryPingBuilder {
// In the server url, the initial path directly after the "scheme://host:port/"
private static final String SERVER_INITIAL_PATH = "submit/telemetry";
private final String serverPath;
protected final ExtendedJSONObject payload;
private final String docID;
public TelemetryPingBuilder() {
docID = UUID.randomUUID().toString();
serverPath = getTelemetryServerPath(getDocType(), docID);
payload = new ExtendedJSONObject();
}
/**
* @return the name of the ping (e.g. "core")
*/
public abstract String getDocType();
/**
* @return the fields that are mandatory for the resultant ping to be uploaded to
* the server. These will be validated before the ping is built.
*/
public abstract String[] getMandatoryFields();
public TelemetryPing build() {
validatePayload();
return new TelemetryPing(serverPath, payload, docID);
}
private void validatePayload() {
final Set<String> keySet = payload.keySet();
for (final String mandatoryField : getMandatoryFields()) {
if (!keySet.contains(mandatoryField)) {
throw new IllegalArgumentException("Builder does not contain mandatory field: " +
mandatoryField);
}
}
}
/**
* Returns a url of the format:
* http://hostname/submit/telemetry/docId/docType/appName/appVersion/appUpdateChannel/appBuildID
*
* @param docType The name of the ping (e.g. "main")
* @return a url at which to POST the telemetry data to
*/
private static String getTelemetryServerPath(final String docType, final String docID) {
final String appName = AppConstants.MOZ_APP_BASENAME;
final String appVersion = AppConstants.MOZ_APP_VERSION;
final String appUpdateChannel = AppConstants.MOZ_UPDATE_CHANNEL;
final String appBuildId = AppConstants.MOZ_APP_BUILDID;
// The compiler will optimize a single String concatenation into a StringBuilder statement.
// If you change this `return`, be sure to keep it as a single statement to keep it optimized!
return SERVER_INITIAL_PATH + '/' +
docID + '/' +
docType + '/' +
appName + '/' +
appVersion + '/' +
appUpdateChannel + '/' +
appBuildId;
}
}

32
mobile/android/base/java/org/mozilla/gecko/telemetry/schedulers/TelemetryUploadAllPingsImmediatelyScheduler.java Normal file
View file

@ -0,0 +1,32 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.schedulers;
import android.content.Context;
import android.content.Intent;
import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
import org.mozilla.gecko.telemetry.TelemetryUploadService;
/**
* Schedules an upload with all pings to be sent immediately.
*/
public class TelemetryUploadAllPingsImmediatelyScheduler implements TelemetryUploadScheduler {
@Override
public boolean isReadyToUpload(final TelemetryPingStore store) {
// We're ready since we don't have any conditions to wait on (e.g. on wifi, accumulated X pings).
return true;
}
@Override
public void scheduleUpload(final Context applicationContext, final TelemetryPingStore store) {
final Intent i = new Intent(TelemetryUploadService.ACTION_UPLOAD);
i.setClass(applicationContext, TelemetryUploadService.class);
i.putExtra(TelemetryUploadService.EXTRA_STORE, store);
applicationContext.startService(i);
}
}

26
mobile/android/base/java/org/mozilla/gecko/telemetry/schedulers/TelemetryUploadScheduler.java Normal file
View file

@ -0,0 +1,26 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.schedulers;
import android.content.Context;
import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
/**
* An implementation of this class can investigate the given {@link TelemetryPingStore} to
* decide if it's ready to upload the pings inside that Store (e.g. on wifi? have we
* accumulated X pings?) and can schedule that upload. Typically, the upload will be
* scheduled by sending an {@link android.content.Intent} to the
* {@link org.mozilla.gecko.telemetry.TelemetryUploadService}, either immediately or
* via an external scheduler (e.g. {@link android.app.job.JobScheduler}).
*
* N.B.: If the Store is not ready to upload, an implementation *should not* try to reschedule
* the check to see if it's time to upload - this is expected to be handled by the caller.
*/
public interface TelemetryUploadScheduler {
boolean isReadyToUpload(TelemetryPingStore store);
void scheduleUpload(Context applicationContext, TelemetryPingStore store);
}

301
mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java Normal file
View file

@ -0,0 +1,301 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.stores;
import android.os.Parcel;
import android.os.Parcelable;
import android.support.annotation.VisibleForTesting;
import android.support.annotation.WorkerThread;
import android.util.Log;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.sync.NonObjectJSONException;
import org.mozilla.gecko.telemetry.TelemetryPing;
import org.mozilla.gecko.util.FileUtils;
import org.mozilla.gecko.util.FileUtils.FileLastModifiedComparator;
import org.mozilla.gecko.util.FileUtils.FilenameRegexFilter;
import org.mozilla.gecko.util.FileUtils.FilenameWhitelistFilter;
import org.mozilla.gecko.util.StringUtils;
import org.mozilla.gecko.util.UUIDUtil;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.channels.FileLock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
/**
* An implementation of TelemetryPingStore that is backed by JSON files.
*
* This implementation seeks simplicity. Each ping to upload is stored in its own file with its doc ID
* as the filename. The doc ID is sent with a ping to be uploaded and is expected to be returned with
* {@link #onUploadAttemptComplete(Set)} so the associated file can be removed.
*
* During prune, the pings with the oldest modified time will be removed first. Different filesystems will
* handle clock skew (e.g. manual time changes, daylight savings time, changing timezones) in different ways
* and we accept that these modified times may not be consistent - newer data is not more important than
* older data and the choice to delete the oldest data first is largely arbitrary so we don't care if
* the timestamps are occasionally inconsistent.
*
* Using separate files for this store allows for less restrictive concurrency:
* * requires locking: {@link #storePing(TelemetryPing)} writes a new file
* * requires locking: {@link #getAllPings()} reads all files, including those potentially being written,
* hence locking
* * no locking: {@link #maybePrunePings()} deletes the least recently written pings, none of which should
* be currently written
* * no locking: {@link #onUploadAttemptComplete(Set)} deletes the given pings, none of which should be
* currently written
*/
public class TelemetryJSONFilePingStore extends TelemetryPingStore {
private static final String LOGTAG = StringUtils.safeSubstring(
"Gecko" + TelemetryJSONFilePingStore.class.getSimpleName(), 0, 23);
@VisibleForTesting static final int MAX_PING_COUNT = 40; // TODO: value.
// We keep the key names short to reduce storage size impact.
@VisibleForTesting static final String KEY_PAYLOAD = "p";
@VisibleForTesting static final String KEY_URL_PATH = "u";
private final File storeDir;
private final FilenameFilter uuidFilenameFilter;
private final FileLastModifiedComparator fileLastModifiedComparator = new FileLastModifiedComparator();
@WorkerThread // Writes to disk
public TelemetryJSONFilePingStore(final File storeDir, final String profileName) {
super(profileName);
if (storeDir.exists() && !storeDir.isDirectory()) {
// An alternative is to create a new directory, but we wouldn't
// be able to access it later so it's better to throw.
throw new IllegalStateException("Store dir unexpectedly exists & is not a directory - cannot continue");
}
this.storeDir = storeDir;
this.storeDir.mkdirs();
uuidFilenameFilter = new FilenameRegexFilter(UUIDUtil.UUID_PATTERN);
if (!this.storeDir.canRead() || !this.storeDir.canWrite() || !this.storeDir.canExecute()) {
throw new IllegalStateException("Cannot read, write, or execute store dir: " +
this.storeDir.canRead() + " " + this.storeDir.canWrite() + " " + this.storeDir.canExecute());
}
}
@VisibleForTesting File getPingFile(final String docID) {
return new File(storeDir, docID);
}
@Override
public void storePing(final TelemetryPing ping) throws IOException {
final String output;
try {
output = new JSONObject()
.put(KEY_PAYLOAD, ping.getPayload())
.put(KEY_URL_PATH, ping.getURLPath())
.toString();
} catch (final JSONException e) {
// Do not log the exception to avoid leaking personal data.
throw new IOException("Unable to create JSON to store to disk");
}
final FileOutputStream outputStream = new FileOutputStream(getPingFile(ping.getDocID()), false);
blockForLockAndWriteFileAndCloseStream(outputStream, output);
}
@Override
public void maybePrunePings() {
final File[] files = storeDir.listFiles(uuidFilenameFilter);
if (files == null) {
return;
}
if (files.length < MAX_PING_COUNT) {
return;
}
// It's possible that multiple files will have the same timestamp: in this case they are treated
// as equal by the fileLastModifiedComparator. We therefore have to use a sorted list (as
// opposed to a set, or map).
final ArrayList<File> sortedFiles = new ArrayList<>(Arrays.asList(files));
Collections.sort(sortedFiles, fileLastModifiedComparator);
deleteSmallestFiles(sortedFiles, files.length - MAX_PING_COUNT);
}
private void deleteSmallestFiles(final ArrayList<File> files, final int numFilesToRemove) {
final Iterator<File> it = files.iterator();
int i = 0;
while (i < numFilesToRemove) {
i += 1;
// Sorted list so we're iterating over ascending files.
final File file = it.next(); // file count > files to remove so this should not throw.
file.delete();
}
}
@Override
public ArrayList<TelemetryPing> getAllPings() {
final File[] fileArray = storeDir.listFiles(uuidFilenameFilter);
if (fileArray == null) {
// Intentionally don't log all info for the store directory to prevent leaking the path.
Log.w(LOGTAG, "listFiles unexpectedly returned null - unable to retrieve pings. Debug: exists? " +
storeDir.exists() + "; directory? " + storeDir.isDirectory());
return new ArrayList<>(1);
}
final List<File> files = Arrays.asList(fileArray);
Collections.sort(files, fileLastModifiedComparator); // oldest to newest
final ArrayList<TelemetryPing> out = new ArrayList<>(files.size());
for (final File file : files) {
final JSONObject obj = lockAndReadJSONFromFile(file);
if (obj == null) {
// We log in the method to get the JSONObject if we return null.
continue;
}
try {
final String url = obj.getString(KEY_URL_PATH);
final ExtendedJSONObject payload = new ExtendedJSONObject(obj.getString(KEY_PAYLOAD));
out.add(new TelemetryPing(url, payload, file.getName()));
} catch (final IOException | JSONException | NonObjectJSONException e) {
Log.w(LOGTAG, "Bad json in ping. Ignoring.");
continue;
}
}
return out;
}
/**
* Logs if there is an error.
*
* @return the JSON object from the given file or null if there is an error.
*/
private JSONObject lockAndReadJSONFromFile(final File file) {
// lockAndReadFileAndCloseStream doesn't handle file size of 0.
if (file.length() == 0) {
Log.w(LOGTAG, "Unexpected empty file: " + file.getName() + ". Ignoring");
return null;
}
final FileInputStream inputStream;
try {
inputStream = new FileInputStream(file);
} catch (final FileNotFoundException e) {
// permission problem might also cause same exception. To get more debug information.
String fileInfo = String.format("existence: %b, can write: %b, size: %d.",
file.exists(), file.canWrite(), file.length());
String msg = String.format(
"Expected file to exist but got exception in thread: %s. File info - %s",
Thread.currentThread().getName(), fileInfo);
throw new IllegalStateException(msg);
}
final JSONObject obj;
try {
// Potential optimization: re-use the same buffer for reading from files.
obj = lockAndReadFileAndCloseStream(inputStream, (int) file.length());
} catch (final IOException | JSONException e) {
// We couldn't read this file so let's just skip it. These potentially
// corrupted files should be removed when the data is pruned.
Log.w(LOGTAG, "Error when reading file: " + file.getName() + " Likely corrupted. Ignoring");
return null;
}
if (obj == null) {
Log.d(LOGTAG, "Could not read given file: " + file.getName() + " File is locked. Ignoring");
}
return obj;
}
@Override
public void onUploadAttemptComplete(final Set<String> successfulRemoveIDs) {
if (successfulRemoveIDs.isEmpty()) {
return;
}
final File[] files = storeDir.listFiles(new FilenameWhitelistFilter(successfulRemoveIDs));
for (final File file : files) {
file.delete();
}
}
/**
* Locks the given {@link FileOutputStream} and writes the given String. This method will close the given stream.
*
* Note: this method blocks until a file lock can be acquired.
*/
private static void blockForLockAndWriteFileAndCloseStream(final FileOutputStream outputStream, final String str)
throws IOException {
try {
final FileLock lock = outputStream.getChannel().lock(0, Long.MAX_VALUE, false);
if (lock != null) {
// The file lock is released when the stream is closed. If we try to redundantly close it, we get
// a ClosedChannelException. To be safe, we could catch that every time but there is a performance
// hit to exception handling so instead we assume the file lock will be closed.
FileUtils.writeStringToOutputStreamAndCloseStream(outputStream, str);
}
} finally {
outputStream.close(); // redundant: closed when the stream is closed, but let's be safe.
}
}
/**
* Locks the given {@link FileInputStream} and reads the data. This method will close the given stream.
*
* Note: this method returns null when a lock could not be acquired.
*/
private static JSONObject lockAndReadFileAndCloseStream(final FileInputStream inputStream, final int fileSize)
throws IOException, JSONException {
try {
final FileLock lock = inputStream.getChannel().tryLock(0, Long.MAX_VALUE, true); // null when lock not acquired
if (lock == null) {
return null;
}
// The file lock is released when the stream is closed. If we try to redundantly close it, we get
// a ClosedChannelException. To be safe, we could catch that every time but there is a performance
// hit to exception handling so instead we assume the file lock will be closed.
return new JSONObject(FileUtils.readStringFromInputStreamAndCloseStream(inputStream, fileSize));
} finally {
inputStream.close(); // redundant: closed when the stream is closed, but let's be safe.
}
}
public static final Parcelable.Creator<TelemetryJSONFilePingStore> CREATOR = new Parcelable.Creator<TelemetryJSONFilePingStore>() {
@Override
public TelemetryJSONFilePingStore createFromParcel(final Parcel source) {
final String storeDirPath = source.readString();
final String profileName = source.readString();
return new TelemetryJSONFilePingStore(new File(storeDirPath), profileName);
}
@Override
public TelemetryJSONFilePingStore[] newArray(final int size) {
return new TelemetryJSONFilePingStore[size];
}
};
@Override
public int describeContents() {
return 0;
}
@Override
public void writeToParcel(final Parcel dest, final int flags) {
dest.writeString(storeDir.getAbsolutePath());
dest.writeString(getProfileName());
}
}

66
mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryPingStore.java Normal file
View file

@ -0,0 +1,66 @@
/*
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.stores;
import android.os.Parcelable;
import org.mozilla.gecko.telemetry.TelemetryPing;
import java.io.IOException;
import java.util.List;
import java.util.Set;
/**
* Persistent storage for TelemetryPings that are queued for upload.
*
* An implementation of this class is expected to be thread-safe. Additionally,
* multiple instances can be created and run simultaneously so they must be able
* to synchronize state (or be stateless!).
*
* The pings in {@link #getAllPings()} and {@link #maybePrunePings()} are returned in the
* same order in order to guarantee consistent results.
*/
public abstract class TelemetryPingStore implements Parcelable {
private final String profileName;
public TelemetryPingStore(final String profileName) {
this.profileName = profileName;
}
/**
* @return the profile name associated with this store.
*/
public String getProfileName() {
return profileName;
}
/**
* @return a list of all the telemetry pings in the store that are ready for upload, ascending oldest to newest.
*/
public abstract List<TelemetryPing> getAllPings();
/**
* Save a ping to the store.
*
* @param ping the ping to store
* @throws IOException for underlying store access errors
*/
public abstract void storePing(TelemetryPing ping) throws IOException;
/**
* Removes telemetry pings from the store if there are too many pings or they take up too much space.
* Pings should be removed from oldest to newest.
*/
public abstract void maybePrunePings();
/**
* Removes the successfully uploaded pings from the database and performs another other actions necessary
* for when upload is completed.
*
* @param successfulRemoveIDs doc ids of pings that were successfully uploaded
*/
public abstract void onUploadAttemptComplete(Set<String> successfulRemoveIDs);
}