wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-07 00:08:39 +09:00
Code Issues Projects Releases Packages Wiki Activity

import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

Browse source
This commit is contained in:
Roy Tam 2018-01-19 03:59:58 +08:00
commit dcd9973243
150858 changed files with 23884658 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

166
mobile/android/base/java/org/mozilla/gecko/dlc/BaseAction.java Normal file
View file

@ -0,0 +1,166 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import android.content.Context;
import android.support.annotation.IntDef;
import android.util.Log;
import org.mozilla.gecko.AppConstants;
import org.mozilla.gecko.background.nativecode.NativeCrypto;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import org.mozilla.gecko.sync.Utils;
import org.mozilla.gecko.util.HardwareUtils;
import org.mozilla.gecko.util.IOUtils;
import org.mozilla.gecko.util.ProxySelector;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
public abstract class BaseAction {
private static final String LOGTAG = "GeckoDLCBaseAction";
/**
* Exception indicating a recoverable error has happened. Download of the content will be retried later.
*/
/* package-private */ static class RecoverableDownloadContentException extends Exception {
private static final long serialVersionUID = -2246772819507370734L;
@IntDef({MEMORY, DISK_IO, SERVER, NETWORK})
public @interface ErrorType {}
public static final int MEMORY = 1;
public static final int DISK_IO = 2;
public static final int SERVER = 3;
public static final int NETWORK = 4;
private int errorType;
public RecoverableDownloadContentException(@ErrorType int errorType, String message) {
super(message);
this.errorType = errorType;
}
public RecoverableDownloadContentException(@ErrorType int errorType, Throwable cause) {
super(cause);
this.errorType = errorType;
}
@ErrorType
public int getErrorType() {
return errorType;
}
/**
* Should this error be counted as failure? If this type of error will happen multiple times in a row then this
* error will be treated as permanently and the operation will not be tried again until the content changes.
*/
public boolean shouldBeCountedAsFailure() {
if (NETWORK == errorType) {
return false; // Always retry after network errors
}
return true;
}
}
/**
* If this exception is thrown the content will be marked as unrecoverable, permanently failed and we will not try
* downloading it again - until a newer version of the content is available.
*/
/* package-private */ static class UnrecoverableDownloadContentException extends Exception {
private static final long serialVersionUID = 8956080754787367105L;
public UnrecoverableDownloadContentException(String message) {
super(message);
}
public UnrecoverableDownloadContentException(Throwable cause) {
super(cause);
}
}
public abstract void perform(Context context, DownloadContentCatalog catalog);
protected File getDestinationFile(Context context, DownloadContent content)
throws UnrecoverableDownloadContentException, RecoverableDownloadContentException {
if (content.isFont()) {
File destinationDirectory = new File(context.getApplicationInfo().dataDir, "fonts");
if (!destinationDirectory.exists() && !destinationDirectory.mkdirs()) {
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO,
"Destination directory does not exist and cannot be created");
}
return new File(destinationDirectory, content.getFilename());
}
// Unrecoverable: We downloaded a file and we don't know what to do with it (Should not happen)
throw new UnrecoverableDownloadContentException("Can't determine destination for kind: " + content.getKind());
}
protected boolean verify(File file, String expectedChecksum)
throws RecoverableDownloadContentException, UnrecoverableDownloadContentException {
InputStream inputStream = null;
try {
inputStream = new BufferedInputStream(new FileInputStream(file));
byte[] ctx = NativeCrypto.sha256init();
if (ctx == null) {
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.MEMORY,
"Could not create SHA-256 context");
}
byte[] buffer = new byte[4096];
int read;
while ((read = inputStream.read(buffer)) != -1) {
NativeCrypto.sha256update(ctx, buffer, read);
}
String actualChecksum = Utils.byte2Hex(NativeCrypto.sha256finalize(ctx));
if (!actualChecksum.equalsIgnoreCase(expectedChecksum)) {
Log.w(LOGTAG, "Checksums do not match. Expected=" + expectedChecksum + ", Actual=" + actualChecksum);
return false;
}
return true;
} catch (IOException e) {
// Recoverable: Just I/O discontinuation
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, e);
} finally {
IOUtils.safeStreamClose(inputStream);
}
}
protected HttpURLConnection buildHttpURLConnection(String url)
throws UnrecoverableDownloadContentException, IOException {
try {
System.setProperty("http.keepAlive", "true");
HttpURLConnection connection = (HttpURLConnection) ProxySelector.openConnectionWithProxy(new URI(url));
connection.setRequestProperty("User-Agent", HardwareUtils.isTablet() ?
AppConstants.USER_AGENT_FENNEC_TABLET :
AppConstants.USER_AGENT_FENNEC_MOBILE);
connection.setRequestMethod("GET");
connection.setInstanceFollowRedirects(true);
return connection;
} catch (MalformedURLException e) {
throw new UnrecoverableDownloadContentException(e);
} catch (URISyntaxException e) {
throw new UnrecoverableDownloadContentException(e);
}
}
}

49
mobile/android/base/java/org/mozilla/gecko/dlc/CleanupAction.java Normal file
View file

@ -0,0 +1,49 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import android.content.Context;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import java.io.File;
/**
* CleanupAction: Remove content that is no longer needed.
*/
public class CleanupAction extends BaseAction {
@Override
public void perform(Context context, DownloadContentCatalog catalog) {
for (DownloadContent content : catalog.getContentToDelete()) {
if (!content.isAssetArchive()) {
continue; // We do not know how to clean up this content. But this means we didn't
// download it anyways.
}
try {
File file = getDestinationFile(context, content);
if (!file.exists()) {
// File does not exist. As good as deleting.
catalog.remove(content);
return;
}
if (file.delete()) {
// File has been deleted. Now remove it from the catalog.
catalog.remove(content);
}
} catch (UnrecoverableDownloadContentException e) {
// We can't recover. Pretend the content is removed. It probably never existed in
// the first place.
catalog.remove(content);
} catch (RecoverableDownloadContentException e) {
// Try again next time.
}
}
}
}

325
mobile/android/base/java/org/mozilla/gecko/dlc/DownloadAction.java Normal file
View file

@ -0,0 +1,325 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import android.content.Context;
import android.net.ConnectivityManager;
import android.net.NetworkInfo;
import android.support.v4.net.ConnectivityManagerCompat;
import android.util.Log;
import org.mozilla.gecko.AppConstants;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import org.mozilla.gecko.util.HardwareUtils;
import org.mozilla.gecko.util.IOUtils;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.zip.GZIPInputStream;
/**
* Download content that has been scheduled during "study" or "verify".
*/
public class DownloadAction extends BaseAction {
private static final String LOGTAG = "DLCDownloadAction";
private static final String CACHE_DIRECTORY = "downloadContent";
private static final String CDN_BASE_URL = "https://fennec-catalog.cdn.mozilla.net/";
public interface Callback {
void onContentDownloaded(DownloadContent content);
}
private Callback callback;
public DownloadAction(Callback callback) {
this.callback = callback;
}
public void perform(Context context, DownloadContentCatalog catalog) {
Log.d(LOGTAG, "Downloading content..");
if (!isConnectedToNetwork(context)) {
Log.d(LOGTAG, "No connected network available. Postponing download.");
// TODO: Reschedule download (bug 1209498)
return;
}
if (isActiveNetworkMetered(context)) {
Log.d(LOGTAG, "Network is metered. Postponing download.");
// TODO: Reschedule download (bug 1209498)
return;
}
for (DownloadContent content : catalog.getScheduledDownloads()) {
Log.d(LOGTAG, "Downloading: " + content);
File temporaryFile = null;
try {
File destinationFile = getDestinationFile(context, content);
if (destinationFile.exists() && verify(destinationFile, content.getChecksum())) {
Log.d(LOGTAG, "Content already exists and is up-to-date.");
catalog.markAsDownloaded(content);
continue;
}
temporaryFile = createTemporaryFile(context, content);
if (!hasEnoughDiskSpace(content, destinationFile, temporaryFile)) {
Log.d(LOGTAG, "Not enough disk space to save content. Skipping download.");
continue;
}
// TODO: Check space on disk before downloading content (bug 1220145)
final String url = createDownloadURL(content);
if (!temporaryFile.exists() || temporaryFile.length() < content.getSize()) {
download(url, temporaryFile);
}
if (!verify(temporaryFile, content.getDownloadChecksum())) {
Log.w(LOGTAG, "Wrong checksum after download, content=" + content.getId());
temporaryFile.delete();
continue;
}
if (!content.isAssetArchive()) {
Log.e(LOGTAG, "Downloaded content is not of type 'asset-archive': " + content.getType());
temporaryFile.delete();
continue;
}
extract(temporaryFile, destinationFile, content.getChecksum());
catalog.markAsDownloaded(content);
Log.d(LOGTAG, "Successfully downloaded: " + content);
if (callback != null) {
callback.onContentDownloaded(content);
}
if (temporaryFile != null && temporaryFile.exists()) {
temporaryFile.delete();
}
} catch (RecoverableDownloadContentException e) {
Log.w(LOGTAG, "Downloading content failed (Recoverable): " + content, e);
if (e.shouldBeCountedAsFailure()) {
catalog.rememberFailure(content, e.getErrorType());
}
// TODO: Reschedule download (bug 1209498)
} catch (UnrecoverableDownloadContentException e) {
Log.w(LOGTAG, "Downloading content failed (Unrecoverable): " + content, e);
catalog.markAsPermanentlyFailed(content);
if (temporaryFile != null && temporaryFile.exists()) {
temporaryFile.delete();
}
}
}
Log.v(LOGTAG, "Done");
}
protected void download(String source, File temporaryFile)
throws RecoverableDownloadContentException, UnrecoverableDownloadContentException {
InputStream inputStream = null;
OutputStream outputStream = null;
HttpURLConnection connection = null;
try {
connection = buildHttpURLConnection(source);
final long offset = temporaryFile.exists() ? temporaryFile.length() : 0;
if (offset > 0) {
connection.setRequestProperty("Range", "bytes=" + offset + "-");
}
final int status = connection.getResponseCode();
if (status != HttpURLConnection.HTTP_OK && status != HttpURLConnection.HTTP_PARTIAL) {
// We are trying to be smart and only retry if this is an error that might resolve in the future.
// TODO: This is guesstimating at best. We want to implement failure counters (Bug 1215106).
if (status >= 500) {
// Recoverable: Server errors 5xx
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.SERVER,
"(Recoverable) Download failed. Status code: " + status);
} else if (status >= 400) {
// Unrecoverable: Client errors 4xx - Unlikely that this version of the client will ever succeed.
throw new UnrecoverableDownloadContentException("(Unrecoverable) Download failed. Status code: " + status);
} else {
// HttpsUrlConnection: -1 (No valid response code)
// Informational 1xx: They have no meaning to us.
// Successful 2xx: We don't know how to handle anything but 200.
// Redirection 3xx: HttpClient should have followed redirects if possible. We should not see those errors here.
throw new UnrecoverableDownloadContentException("(Unrecoverable) Download failed. Status code: " + status);
}
}
inputStream = new BufferedInputStream(connection.getInputStream());
outputStream = openFile(temporaryFile, status == HttpURLConnection.HTTP_PARTIAL);
IOUtils.copy(inputStream, outputStream);
inputStream.close();
outputStream.close();
} catch (IOException e) {
// Recoverable: Just I/O discontinuation
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.NETWORK, e);
} finally {
IOUtils.safeStreamClose(inputStream);
IOUtils.safeStreamClose(outputStream);
if (connection != null) {
connection.disconnect();
}
}
}
protected OutputStream openFile(File file, boolean append) throws FileNotFoundException {
return new BufferedOutputStream(new FileOutputStream(file, append));
}
protected void extract(File sourceFile, File destinationFile, String checksum)
throws UnrecoverableDownloadContentException, RecoverableDownloadContentException {
InputStream inputStream = null;
OutputStream outputStream = null;
File temporaryFile = null;
try {
File destinationDirectory = destinationFile.getParentFile();
if (!destinationDirectory.exists() && !destinationDirectory.mkdirs()) {
throw new IOException("Destination directory does not exist and cannot be created");
}
temporaryFile = new File(destinationDirectory, destinationFile.getName() + ".tmp");
inputStream = new GZIPInputStream(new BufferedInputStream(new FileInputStream(sourceFile)));
outputStream = new BufferedOutputStream(new FileOutputStream(temporaryFile));
IOUtils.copy(inputStream, outputStream);
inputStream.close();
outputStream.close();
if (!verify(temporaryFile, checksum)) {
Log.w(LOGTAG, "Checksum of extracted file does not match.");
return;
}
move(temporaryFile, destinationFile);
} catch (IOException e) {
// We could not extract to the destination: Keep temporary file and try again next time we run.
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, e);
} finally {
IOUtils.safeStreamClose(inputStream);
IOUtils.safeStreamClose(outputStream);
if (temporaryFile != null && temporaryFile.exists()) {
temporaryFile.delete();
}
}
}
protected boolean isConnectedToNetwork(Context context) {
ConnectivityManager manager = (ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE);
NetworkInfo networkInfo = manager.getActiveNetworkInfo();
return networkInfo != null && networkInfo.isConnected();
}
protected boolean isActiveNetworkMetered(Context context) {
return ConnectivityManagerCompat.isActiveNetworkMetered(
(ConnectivityManager) context.getSystemService(Context.CONNECTIVITY_SERVICE));
}
protected String createDownloadURL(DownloadContent content) {
final String location = content.getLocation();
return CDN_BASE_URL + content.getLocation();
}
protected File createTemporaryFile(Context context, DownloadContent content)
throws RecoverableDownloadContentException {
File cacheDirectory = new File(context.getCacheDir(), CACHE_DIRECTORY);
if (!cacheDirectory.exists() && !cacheDirectory.mkdirs()) {
// Recoverable: File system might not be mounted NOW and we didn't download anything yet anyways.
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO,
"Could not create cache directory: " + cacheDirectory);
}
return new File(cacheDirectory, content.getDownloadChecksum() + "-" + content.getId());
}
protected void move(File temporaryFile, File destinationFile)
throws RecoverableDownloadContentException, UnrecoverableDownloadContentException {
if (!temporaryFile.renameTo(destinationFile)) {
Log.d(LOGTAG, "Could not move temporary file to destination. Trying to copy..");
copy(temporaryFile, destinationFile);
temporaryFile.delete();
}
}
protected void copy(File temporaryFile, File destinationFile)
throws RecoverableDownloadContentException, UnrecoverableDownloadContentException {
InputStream inputStream = null;
OutputStream outputStream = null;
try {
File destinationDirectory = destinationFile.getParentFile();
if (!destinationDirectory.exists() && !destinationDirectory.mkdirs()) {
throw new IOException("Destination directory does not exist and cannot be created");
}
inputStream = new BufferedInputStream(new FileInputStream(temporaryFile));
outputStream = new BufferedOutputStream(new FileOutputStream(destinationFile));
IOUtils.copy(inputStream, outputStream);
inputStream.close();
outputStream.close();
} catch (IOException e) {
// We could not copy the temporary file to its destination: Keep the temporary file and
// try again the next time we run.
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.DISK_IO, e);
} finally {
IOUtils.safeStreamClose(inputStream);
IOUtils.safeStreamClose(outputStream);
}
}
protected boolean hasEnoughDiskSpace(DownloadContent content, File destinationFile, File temporaryFile) {
final File temporaryDirectory = temporaryFile.getParentFile();
if (temporaryDirectory.getUsableSpace() < content.getSize()) {
return false;
}
final File destinationDirectory = destinationFile.getParentFile();
// We need some more space to extract the file (getSize() returns the uncompressed size)
if (destinationDirectory.getUsableSpace() < content.getSize() * 2) {
return false;
}
return true;
}
}

144
mobile/android/base/java/org/mozilla/gecko/dlc/DownloadContentService.java Normal file
View file

@ -0,0 +1,144 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import org.mozilla.gecko.AppConstants;
import org.mozilla.gecko.GeckoAppShell;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import org.mozilla.gecko.util.HardwareUtils;
import android.app.IntentService;
import android.content.ComponentName;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
/**
* Service to handle downloadable content that did not ship with the APK.
*/
public class DownloadContentService extends IntentService {
private static final String LOGTAG = "GeckoDLCService";
/**
* Study: Scan the catalog for "new" content available for download.
*/
private static final String ACTION_STUDY_CATALOG = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.STUDY";
/**
* Verify: Validate downloaded content. Does it still exist and does it have the correct checksum?
*/
private static final String ACTION_VERIFY_CONTENT = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.VERIFY";
/**
* Download content that has been scheduled during "study" or "verify".
*/
private static final String ACTION_DOWNLOAD_CONTENT = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.DOWNLOAD";
/**
* Sync: Synchronize catalog from a Kinto instance.
*/
private static final String ACTION_SYNCHRONIZE_CATALOG = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.SYNC";
/**
* CleanupAction: Remove content that is no longer needed (e.g. Removed from the catalog after a sync).
*/
private static final String ACTION_CLEANUP_FILES = AppConstants.ANDROID_PACKAGE_NAME + ".DLC.CLEANUP";
public static void startStudy(Context context) {
Intent intent = new Intent(ACTION_STUDY_CATALOG);
intent.setComponent(new ComponentName(context, DownloadContentService.class));
context.startService(intent);
}
public static void startVerification(Context context) {
Intent intent = new Intent(ACTION_VERIFY_CONTENT);
intent.setComponent(new ComponentName(context, DownloadContentService.class));
context.startService(intent);
}
public static void startDownloads(Context context) {
Intent intent = new Intent(ACTION_DOWNLOAD_CONTENT);
intent.setComponent(new ComponentName(context, DownloadContentService.class));
context.startService(intent);
}
public static void startSync(Context context) {
Intent intent = new Intent(ACTION_SYNCHRONIZE_CATALOG);
intent.setComponent(new ComponentName(context, DownloadContentService.class));
context.startService(intent);
}
public static void startCleanup(Context context) {
Intent intent = new Intent(ACTION_CLEANUP_FILES);
intent.setComponent(new ComponentName(context, DownloadContentService.class));
context.startService(intent);
}
private DownloadContentCatalog catalog;
public DownloadContentService() {
super(LOGTAG);
}
@Override
public void onCreate() {
super.onCreate();
catalog = new DownloadContentCatalog(this);
}
protected void onHandleIntent(Intent intent) {
if (!AppConstants.MOZ_ANDROID_DOWNLOAD_CONTENT_SERVICE) {
Log.w(LOGTAG, "Download content is not enabled. Stop.");
return;
}
if (!HardwareUtils.isSupportedSystem()) {
// This service is running very early before checks in BrowserApp can prevent us from running.
Log.w(LOGTAG, "System is not supported. Stop.");
return;
}
if (intent == null) {
return;
}
final BaseAction action;
switch (intent.getAction()) {
case ACTION_STUDY_CATALOG:
action = new StudyAction();
break;
case ACTION_DOWNLOAD_CONTENT:
action = new DownloadAction(new DownloadAction.Callback() {
@Override
public void onContentDownloaded(DownloadContent content) {
if (content.isFont()) {
GeckoAppShell.notifyObservers("Fonts:Reload", "");
}
}
});
break;
case ACTION_VERIFY_CONTENT:
action = new VerifyAction();
break;
case ACTION_SYNCHRONIZE_CATALOG:
action = new SyncAction();
break;
default:
Log.e(LOGTAG, "Unknown action: " + intent.getAction());
return;
}
action.perform(this, catalog);
catalog.persistChanges();
}
}

81
mobile/android/base/java/org/mozilla/gecko/dlc/StudyAction.java Normal file
View file

@ -0,0 +1,81 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import android.content.Context;
import android.os.Build;
import android.text.TextUtils;
import android.util.Log;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import org.mozilla.gecko.util.ContextUtils;
/**
* Study: Scan the catalog for "new" content available for download.
*/
public class StudyAction extends BaseAction {
private static final String LOGTAG = "DLCStudyAction";
public void perform(Context context, DownloadContentCatalog catalog) {
Log.d(LOGTAG, "Studying catalog..");
for (DownloadContent content : catalog.getContentToStudy()) {
if (!isMatching(context, content)) {
// This content is not for this particular version of the application or system
continue;
}
if (content.isAssetArchive() && content.isFont()) {
catalog.scheduleDownload(content);
Log.d(LOGTAG, "Scheduled download: " + content);
}
}
if (catalog.hasScheduledDownloads()) {
startDownloads(context);
}
Log.v(LOGTAG, "Done");
}
protected boolean isMatching(Context context, DownloadContent content) {
final String androidApiPattern = content.getAndroidApiPattern();
if (!TextUtils.isEmpty(androidApiPattern)) {
final String apiVersion = String.valueOf(Build.VERSION.SDK_INT);
if (apiVersion.matches(androidApiPattern)) {
Log.d(LOGTAG, String.format("Android API (%s) does not match pattern: %s", apiVersion, androidApiPattern));
return false;
}
}
final String appIdPattern = content.getAppIdPattern();
if (!TextUtils.isEmpty(appIdPattern)) {
final String appId = context.getPackageName();
if (!appId.matches(appIdPattern)) {
Log.d(LOGTAG, String.format("App ID (%s) does not match pattern: %s", appId, appIdPattern));
return false;
}
}
final String appVersionPattern = content.getAppVersionPattern();
if (!TextUtils.isEmpty(appVersionPattern)) {
final String appVersion = ContextUtils.getCurrentPackageInfo(context).versionName;
if (!appVersion.matches(appVersionPattern)) {
Log.d(LOGTAG, String.format("App version (%s) does not match pattern: %s", appVersion, appVersionPattern));
return false;
}
}
// There are no patterns or all patterns have matched.
return true;
}
protected void startDownloads(Context context) {
DownloadContentService.startDownloads(context);
}
}

263
mobile/android/base/java/org/mozilla/gecko/dlc/SyncAction.java Normal file
View file

@ -0,0 +1,263 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import android.content.Context;
import android.net.Uri;
import android.util.Log;
import com.keepsafe.switchboard.SwitchBoard;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentBuilder;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import org.mozilla.gecko.Experiments;
import org.mozilla.gecko.util.IOUtils;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
/**
* Sync: Synchronize catalog from a Kinto instance.
*/
public class SyncAction extends BaseAction {
private static final String LOGTAG = "DLCSyncAction";
private static final String KINTO_KEY_ID = "id";
private static final String KINTO_KEY_DELETED = "deleted";
private static final String KINTO_KEY_DATA = "data";
private static final String KINTO_KEY_ATTACHMENT = "attachment";
private static final String KINTO_KEY_ORIGINAL = "original";
private static final String KINTO_PARAMETER_SINCE = "_since";
private static final String KINTO_PARAMETER_FIELDS = "_fields";
private static final String KINTO_PARAMETER_SORT = "_sort";
/**
* Kinto endpoint with online version of downloadable content catalog
*
* Dev instance:
* https://kinto-ota.dev.mozaws.net/v1/buckets/dlc/collections/catalog/records
*/
private static final String CATALOG_ENDPOINT = "https://firefox.settings.services.mozilla.com/v1/buckets/fennec/collections/catalog/records";
@Override
public void perform(Context context, DownloadContentCatalog catalog) {
Log.d(LOGTAG, "Synchronizing catalog.");
if (!isSyncEnabledForClient(context)) {
Log.d(LOGTAG, "Sync is not enabled for client. Skipping.");
return;
}
boolean cleanupRequired = false;
boolean studyRequired = false;
try {
long lastModified = catalog.getLastModified();
// TODO: Consider using ETag here (Bug 1257459)
JSONArray rawCatalog = fetchRawCatalog(lastModified);
Log.d(LOGTAG, "Server returned " + rawCatalog.length() + " records (since " + lastModified + ")");
for (int i = 0; i < rawCatalog.length(); i++) {
JSONObject object = rawCatalog.getJSONObject(i);
String id = object.getString(KINTO_KEY_ID);
final boolean isDeleted = object.optBoolean(KINTO_KEY_DELETED, false);
if (!isDeleted) {
JSONObject attachment = object.getJSONObject(KINTO_KEY_ATTACHMENT);
if (attachment.isNull(KINTO_KEY_ORIGINAL))
throw new JSONException(String.format("Old Attachment Format"));
}
DownloadContent existingContent = catalog.getContentById(id);
if (isDeleted) {
cleanupRequired |= deleteContent(catalog, id);
} else if (existingContent != null) {
studyRequired |= updateContent(catalog, object, existingContent);
} else {
studyRequired |= createContent(catalog, object);
}
}
} catch (UnrecoverableDownloadContentException e) {
Log.e(LOGTAG, "UnrecoverableDownloadContentException", e);
} catch (RecoverableDownloadContentException e) {
Log.e(LOGTAG, "RecoverableDownloadContentException");
} catch (JSONException e) {
Log.e(LOGTAG, "JSONException", e);
}
if (studyRequired) {
startStudyAction(context);
}
if (cleanupRequired) {
startCleanupAction(context);
}
Log.v(LOGTAG, "Done");
}
protected void startStudyAction(Context context) {
DownloadContentService.startStudy(context);
}
protected void startCleanupAction(Context context) {
DownloadContentService.startCleanup(context);
}
protected JSONArray fetchRawCatalog(long lastModified)
throws RecoverableDownloadContentException, UnrecoverableDownloadContentException {
HttpURLConnection connection = null;
try {
Uri.Builder builder = Uri.parse(CATALOG_ENDPOINT).buildUpon();
if (lastModified > 0) {
builder.appendQueryParameter(KINTO_PARAMETER_SINCE, String.valueOf(lastModified));
}
// Only select the fields we are actually going to read.
builder.appendQueryParameter(KINTO_PARAMETER_FIELDS,
"attachment.location,attachment.original.filename,attachment.original.hash,attachment.hash,type,kind,attachment.original.size,match");
// We want to process items in the order they have been modified. This is to ensure that
// our last_modified values are correct if we processing is interrupted and not all items
// have been processed.
builder.appendQueryParameter(KINTO_PARAMETER_SORT, "last_modified");
connection = buildHttpURLConnection(builder.build().toString());
// TODO: Read 'Alert' header and EOL message if existing (Bug 1249248)
// TODO: Read and use 'Backoff' header if available (Bug 1249251)
// TODO: Add support for Next-Page header (Bug 1257495)
final int responseCode = connection.getResponseCode();
if (responseCode != HttpURLConnection.HTTP_OK) {
if (responseCode >= 500) {
// A Retry-After header will be added to error responses (>=500), telling the
// client how many seconds it should wait before trying again.
// TODO: Read and obey value in "Retry-After" header (Bug 1249249)
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.SERVER, "Server error (" + responseCode + ")");
} else if (responseCode == 410) {
// A 410 Gone error response can be returned if the client version is too old,
// or the service had been replaced with a new and better service using a new
// protocol version.
// TODO: The server is gone. Stop synchronizing the catalog from this server (Bug 1249248).
throw new UnrecoverableDownloadContentException("Server is gone (410)");
} else if (responseCode >= 400) {
// If the HTTP status is >=400 the response contains a JSON response.
logErrorResponse(connection);
// Unrecoverable: Client errors 4xx - Unlikely that this version of the client will ever succeed.
throw new UnrecoverableDownloadContentException("(Unrecoverable) Catalog sync failed. Status code: " + responseCode);
} else if (responseCode < 200) {
// If the HTTP status is <200 the response contains a JSON response.
logErrorResponse(connection);
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.SERVER, "Response code: " + responseCode);
} else {
// HttpsUrlConnection: -1 (No valid response code)
// Successful 2xx: We don't know how to handle anything but 200.
// Redirection 3xx: We should have followed redirects if possible. We should not see those errors here.
throw new UnrecoverableDownloadContentException("(Unrecoverable) Download failed. Response code: " + responseCode);
}
}
return fetchJSONResponse(connection).getJSONArray(KINTO_KEY_DATA);
} catch (JSONException | IOException e) {
throw new RecoverableDownloadContentException(RecoverableDownloadContentException.NETWORK, e);
} finally {
if (connection != null) {
connection.disconnect();
}
}
}
private JSONObject fetchJSONResponse(HttpURLConnection connection) throws IOException, JSONException {
InputStream inputStream = null;
try {
inputStream = new BufferedInputStream(connection.getInputStream());
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
IOUtils.copy(inputStream, outputStream);
return new JSONObject(outputStream.toString("UTF-8"));
} finally {
IOUtils.safeStreamClose(inputStream);
}
}
protected boolean updateContent(DownloadContentCatalog catalog, JSONObject object, DownloadContent existingContent)
throws JSONException {
DownloadContent content = existingContent.buildUpon()
.updateFromKinto(object)
.build();
if (existingContent.getLastModified() >= content.getLastModified()) {
Log.d(LOGTAG, "Item has not changed: " + content);
return false;
}
catalog.update(content);
return true;
}
protected boolean createContent(DownloadContentCatalog catalog, JSONObject object) throws JSONException {
DownloadContent content = new DownloadContentBuilder()
.updateFromKinto(object)
.build();
catalog.add(content);
return true;
}
protected boolean deleteContent(DownloadContentCatalog catalog, String id) {
DownloadContent content = catalog.getContentById(id);
if (content == null) {
return false;
}
catalog.markAsDeleted(content);
return true;
}
protected boolean isSyncEnabledForClient(Context context) {
// Sync action is behind a switchboard flag for staged rollout.
return SwitchBoard.isInExperiment(context, Experiments.DOWNLOAD_CONTENT_CATALOG_SYNC);
}
private void logErrorResponse(HttpURLConnection connection) {
try {
JSONObject error = fetchJSONResponse(connection);
Log.w(LOGTAG, "Server returned error response:");
Log.w(LOGTAG, "- Code: " + error.getInt("code"));
Log.w(LOGTAG, "- Errno: " + error.getInt("errno"));
Log.w(LOGTAG, "- Error: " + error.optString("error", "-"));
Log.w(LOGTAG, "- Message: " + error.optString("message", "-"));
Log.w(LOGTAG, "- Info: " + error.optString("info", "-"));
} catch (JSONException | IOException e) {
Log.w(LOGTAG, "Could not fetch error response", e);
}
}
}

63
mobile/android/base/java/org/mozilla/gecko/dlc/VerifyAction.java Normal file
View file

@ -0,0 +1,63 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc;
import android.content.Context;
import android.util.Log;
import org.mozilla.gecko.dlc.catalog.DownloadContent;
import org.mozilla.gecko.dlc.catalog.DownloadContentCatalog;
import java.io.File;
/**
* Verify: Validate downloaded content. Does it still exist and does it have the correct checksum?
*/
public class VerifyAction extends BaseAction {
private static final String LOGTAG = "DLCVerifyAction";
@Override
public void perform(Context context, DownloadContentCatalog catalog) {
Log.d(LOGTAG, "Verifying catalog..");
for (DownloadContent content : catalog.getDownloadedContent()) {
try {
File destinationFile = getDestinationFile(context, content);
if (!destinationFile.exists()) {
Log.d(LOGTAG, "Downloaded content does not exist anymore: " + content);
// This file does not exist even though it is marked as downloaded in the catalog. Scheduling a
// download to fetch it again.
catalog.scheduleDownload(content);
continue;
}
if (!verify(destinationFile, content.getChecksum())) {
catalog.scheduleDownload(content);
Log.d(LOGTAG, "Wrong checksum. Scheduling download: " + content);
continue;
}
Log.v(LOGTAG, "Content okay: " + content);
} catch (UnrecoverableDownloadContentException e) {
Log.w(LOGTAG, "Unrecoverable exception while verifying downloaded file", e);
} catch (RecoverableDownloadContentException e) {
// That's okay, we are just verifying already existing content. No log.
}
}
if (catalog.hasScheduledDownloads()) {
startDownloads(context);
}
Log.v(LOGTAG, "Done");
}
protected void startDownloads(Context context) {
DownloadContentService.startDownloads(context);
}
}

189
mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContent.java Normal file
View file

@ -0,0 +1,189 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc.catalog;
import android.support.annotation.IntDef;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.annotation.StringDef;
public class DownloadContent {
@IntDef({STATE_NONE, STATE_SCHEDULED, STATE_DOWNLOADED, STATE_FAILED, STATE_UPDATED, STATE_DELETED})
public @interface State {}
public static final int STATE_NONE = 0;
public static final int STATE_SCHEDULED = 1;
public static final int STATE_DOWNLOADED = 2;
public static final int STATE_FAILED = 3; // Permanently failed for this version of the content
public static final int STATE_UPDATED = 4;
public static final int STATE_DELETED = 5;
@StringDef({TYPE_ASSET_ARCHIVE})
public @interface Type {}
public static final String TYPE_ASSET_ARCHIVE = "asset-archive";
@StringDef({KIND_FONT, KIND_HYPHENATION_DICTIONARY})
public @interface Kind {}
public static final String KIND_FONT = "font";
public static final String KIND_HYPHENATION_DICTIONARY = "hyphenation";
private final String id;
private final String location;
private final String filename;
private final String checksum;
private final String downloadChecksum;
private final long lastModified;
private final String type;
private final String kind;
private final long size;
private final String appVersionPattern;
private final String androidApiPattern;
private final String appIdPattern;
private int state;
private int failures;
private int lastFailureType;
/* package-private */ DownloadContent(@NonNull String id, @NonNull String location, @NonNull String filename,
@NonNull String checksum, @NonNull String downloadChecksum, @NonNull long lastModified,
@NonNull String type, @NonNull String kind, long size, int failures, int lastFailureType,
@Nullable String appVersionPattern, @Nullable String androidApiPattern, @Nullable String appIdPattern) {
this.id = id;
this.location = location;
this.filename = filename;
this.checksum = checksum;
this.downloadChecksum = downloadChecksum;
this.lastModified = lastModified;
this.type = type;
this.kind = kind;
this.size = size;
this.state = STATE_NONE;
this.failures = failures;
this.lastFailureType = lastFailureType;
this.appVersionPattern = appVersionPattern;
this.androidApiPattern = androidApiPattern;
this.appIdPattern = appIdPattern;
}
public String getId() {
return id;
}
/* package-private */ void setState(@State int state) {
this.state = state;
}
@State
public int getState() {
return state;
}
public boolean isStateIn(@State int... states) {
for (int state : states) {
if (this.state == state) {
return true;
}
}
return false;
}
@Kind
public String getKind() {
return kind;
}
@Type
public String getType() {
return type;
}
public String getLocation() {
return location;
}
public long getLastModified() {
return lastModified;
}
public String getFilename() {
return filename;
}
public String getChecksum() {
return checksum;
}
public String getDownloadChecksum() {
return downloadChecksum;
}
public long getSize() {
return size;
}
public boolean isFont() {
return KIND_FONT.equals(kind);
}
public boolean isHyphenationDictionary() {
return KIND_HYPHENATION_DICTIONARY.equals(kind);
}
/**
*Checks whether the content to be downloaded is a known content.
*Currently it checks whether the type is "Asset Archive" and is of kind
*"Font" or "Hyphenation Dictionary".
*/
public boolean isKnownContent() {
return ((isFont() || isHyphenationDictionary()) && isAssetArchive());
}
public boolean isAssetArchive() {
return TYPE_ASSET_ARCHIVE.equals(type);
}
/* package-private */ int getFailures() {
return failures;
}
/* package-private */ int getLastFailureType() {
return lastFailureType;
}
/* package-private */ void rememberFailure(int failureType) {
if (lastFailureType != failureType) {
lastFailureType = failureType;
failures = 1;
} else {
failures++;
}
}
/* package-private */ void resetFailures() {
failures = 0;
lastFailureType = 0;
}
public String getAppVersionPattern() {
return appVersionPattern;
}
public String getAndroidApiPattern() {
return androidApiPattern;
}
public String getAppIdPattern() {
return appIdPattern;
}
public DownloadContentBuilder buildUpon() {
return DownloadContentBuilder.buildUpon(this);
}
public String toString() {
return String.format("[%s,%s] %s (%d bytes) %s", getType(), getKind(), getId(), getSize(), getChecksum());
}
}

161
mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBootstrap.java Normal file
View file

@ -0,0 +1,161 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc.catalog;
import android.support.v4.util.ArrayMap;
import org.mozilla.gecko.AppConstants;
import java.util.Arrays;
import java.util.List;
/* package-private */ class DownloadContentBootstrap {
public static ArrayMap<String, DownloadContent> createInitialDownloadContentList() {
if (!AppConstants.MOZ_ANDROID_EXCLUDE_FONTS) {
// We are packaging fonts. There's nothing we want to download;
return new ArrayMap<>();
}
List<DownloadContent> initialList = Arrays.asList(
new DownloadContentBuilder()
.setId("c40929cf-7f4c-fa72-3dc9-12cadf56905d")
.setLocation("fennec/catalog/f63e5f92-793c-4574-a2d7-fbc50242b8cb.gz")
.setFilename("CharisSILCompact-B.ttf")
.setChecksum("699d958b492eda0cc2823535f8567d0393090e3842f6df3c36dbe7239cb80b6d")
.setDownloadChecksum("a9f9b34fed353169a88cc159b8f298cb285cce0b8b0f979c22a7d85de46f0532")
.setSize(1676072)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("6d265876-85ed-0917-fdc8-baf583ca2cba")
.setLocation("fennec/catalog/19af6c88-09d9-4d6c-805e-cfebb8699a6c.gz")
.setFilename("CharisSILCompact-BI.ttf")
.setChecksum("82465e747b4f41471dbfd942842b2ee810749217d44b55dbc43623b89f9c7d9b")
.setDownloadChecksum("2be26671039a5e2e4d0360a948b4fa42048171133076a3bb6173d93d4b9cd55b")
.setSize(1667812)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("8460dc6d-d129-fd1a-24b6-343dbf6531dd")
.setLocation("fennec/catalog/f35a384a-90ea-41c6-a957-bb1845de97eb.gz")
.setFilename("CharisSILCompact-I.ttf")
.setChecksum("ab3ed6f2a4d4c2095b78227bd33155d7ccd05a879c107a291912640d4d64f767")
.setDownloadChecksum("38a6469041c02624d43dfd41d2dd745e3e3211655e616188f65789a90952a1e9")
.setSize(1693988)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("c906275c-3747-fe27-426f-6187526a6f06")
.setLocation("fennec/catalog/8c3bec92-d2df-4789-8c4a-0f523f026d96.gz")
.setFilename("CharisSILCompact-R.ttf")
.setChecksum("4ed509317f1bb441b185ea13bf1c9d19d1a0b396962efa3b5dc3190ad88f2067")
.setDownloadChecksum("7c2ec1f550c2005b75383b878f737266b5f0b1c82679dd886c8bbe30c82e340e")
.setSize(1727656)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("ff5deecc-6ecc-d816-bb51-65face460119")
.setLocation("fennec/catalog/ea115d71-e2ac-4609-853e-c978780776b1.gz")
.setFilename("ClearSans-Bold.ttf")
.setChecksum("385d0a293c1714770e198f7c762ab32f7905a0ed9d2993f69d640bd7232b4b70")
.setDownloadChecksum("0d3c22bef90e7096f75b331bb7391de3aa43017e10d61041cd3085816db4919a")
.setSize(140136)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("a173d1db-373b-ce42-1335-6b3285cfdebd")
.setLocation("fennec/catalog/0838e513-2d99-4e53-b58f-6b970f6548c6.gz")
.setFilename("ClearSans-BoldItalic.ttf")
.setChecksum("7bce66864e38eecd7c94b6657b9b38c35ebfacf8046bfb1247e08f07fe933198")
.setDownloadChecksum("de0903164dde1ad3768d0bd6dec949871d6ab7be08f573d9d70f38c138a22e37")
.setSize(156124)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("e65c66df-0088-940d-ca5c-207c22118c0e")
.setLocation("fennec/catalog/7550fa42-0947-478c-a5f0-5ea1bbb6ba27.gz")
.setFilename("ClearSans-Italic.ttf")
.setChecksum("87c13c5fbae832e4f85c3bd46fcbc175978234f39be5fe51c4937be4cbff3b68")
.setDownloadChecksum("6e323db3115005dd0e96d2422db87a520f9ae426de28a342cd6cc87b55601d87")
.setSize(155672)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("25610abb-5dc8-fd75-40e7-990507f010c4")
.setLocation("fennec/catalog/dd9bee7d-d784-476b-a3dd-69af8e516487.gz")
.setFilename("ClearSans-Light.ttf")
.setChecksum("e4885f6188e7a8587f5621c077c6c1f5e8d3739dffc8f4d055c2ba87568c750a")
.setDownloadChecksum("19d4f7c67176e9e254c61420da9c7363d9fe5e6b4bb9d61afa4b3b574280714f")
.setSize(145976)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("ffe40339-a096-2262-c3f8-54af75c81fe6")
.setLocation("fennec/catalog/bc5ada8c-8cfc-443d-93d7-dc5f98138a07.gz")
.setFilename("ClearSans-Medium.ttf")
.setChecksum("5d0e0115f3a3ed4be3eda6d7eabb899bb9a361292802e763d53c72e00f629da1")
.setDownloadChecksum("edec86dab3ad2a97561cb41b584670262a48bed008c57bb587ee05ca47fb067f")
.setSize(148892)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("139a94be-ac69-0264-c9cc-8f2d071fd29d")
.setLocation("fennec/catalog/0490c768-6178-49c2-af88-9f8769ff3167.gz")
.setFilename("ClearSans-MediumItalic.ttf")
.setChecksum("937dda88b26469306258527d38e42c95e27e7ebb9f05bd1d7c5d706a3c9541d7")
.setDownloadChecksum("34edbd1b325dbffe7791fba8dd2d19852eb3c2fe00cff517ea2161ddc424ee22")
.setSize(155228)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("b887012a-01e1-7c94-fdcb-ca44d5b974a2")
.setLocation("fennec/catalog/78205bf8-c668-41b1-b68f-afd54f98713b.gz")
.setFilename("ClearSans-Regular.ttf")
.setChecksum("9b91bbdb95ffa6663da24fdaa8ee06060cd0a4d2dceaf1ffbdda00e04915ee5b")
.setDownloadChecksum("a72f1420b4da1ba9e6797adac34f08e72f94128a85e56542d5e6a8080af5f08a")
.setSize(142572)
.setKind("font")
.setType("asset-archive")
.build(),
new DownloadContentBuilder()
.setId("c8703652-d317-0356-0bf8-95441a5b2c9b")
.setLocation("fennec/catalog/3570f44f-9440-4aa0-abd0-642eaf2a1aa0.gz")
.setFilename("ClearSans-Thin.ttf")
.setChecksum("07b0db85a3ad99afeb803f0f35631436a7b4c67ac66d0c7f77d26a47357c592a")
.setDownloadChecksum("d9f23fd8687d6743f5c281c33539fb16f163304795039959b8caf159e6d62822")
.setSize(147004)
.setKind("font")
.setType("asset-archive")
.build());
ArrayMap<String, DownloadContent> content = new ArrayMap<>();
for (DownloadContent currentContent : initialList) {
content.put(currentContent.getId(), currentContent);
}
return content;
}
}

238
mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentBuilder.java Normal file
View file

@ -0,0 +1,238 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc.catalog;
import android.text.TextUtils;
import org.json.JSONException;
import org.json.JSONObject;
public class DownloadContentBuilder {
private static final String LOCAL_KEY_ID = "id";
private static final String LOCAL_KEY_LOCATION = "location";
private static final String LOCAL_KEY_FILENAME = "filename";
private static final String LOCAL_KEY_CHECKSUM = "checksum";
private static final String LOCAL_KEY_DOWNLOAD_CHECKSUM = "download_checksum";
private static final String LOCAL_KEY_LAST_MODIFIED = "last_modified";
private static final String LOCAL_KEY_TYPE = "type";
private static final String LOCAL_KEY_KIND = "kind";
private static final String LOCAL_KEY_SIZE = "size";
private static final String LOCAL_KEY_STATE = "state";
private static final String LOCAL_KEY_FAILURES = "failures";
private static final String LOCAL_KEY_LAST_FAILURE_TYPE = "last_failure_type";
private static final String LOCAL_KEY_PATTERN_APP_ID = "pattern_app_id";
private static final String LOCAL_KEY_PATTERN_ANDROID_API = "pattern_android_api";
private static final String LOCAL_KEY_PATTERN_APP_VERSION = "pattern_app_version";
private static final String KINTO_KEY_ID = "id";
private static final String KINTO_KEY_ATTACHMENT = "attachment";
private static final String KINTO_KEY_ORIGINAL = "original";
private static final String KINTO_KEY_LOCATION = "location";
private static final String KINTO_KEY_FILENAME = "filename";
private static final String KINTO_KEY_HASH = "hash";
private static final String KINTO_KEY_LAST_MODIFIED = "last_modified";
private static final String KINTO_KEY_TYPE = "type";
private static final String KINTO_KEY_KIND = "kind";
private static final String KINTO_KEY_SIZE = "size";
private static final String KINTO_KEY_MATCH = "match";
private static final String KINTO_KEY_APP_ID = "appId";
private static final String KINTO_KEY_ANDROID_API = "androidApi";
private static final String KINTO_KEY_APP_VERSION = "appVersion";
private String id;
private String location;
private String filename;
private String checksum;
private String downloadChecksum;
private long lastModified;
private String type;
private String kind;
private long size;
private int state;
private int failures;
private int lastFailureType;
private String appVersionPattern;
private String androidApiPattern;
private String appIdPattern;
public static DownloadContentBuilder buildUpon(DownloadContent content) {
DownloadContentBuilder builder = new DownloadContentBuilder();
builder.id = content.getId();
builder.location = content.getLocation();
builder.filename = content.getFilename();
builder.checksum = content.getChecksum();
builder.downloadChecksum = content.getDownloadChecksum();
builder.lastModified = content.getLastModified();
builder.type = content.getType();
builder.kind = content.getKind();
builder.size = content.getSize();
builder.state = content.getState();
builder.failures = content.getFailures();
builder.lastFailureType = content.getLastFailureType();
return builder;
}
public static DownloadContent fromJSON(JSONObject object) throws JSONException {
return new DownloadContentBuilder()
.setId(object.getString(LOCAL_KEY_ID))
.setLocation(object.getString(LOCAL_KEY_LOCATION))
.setFilename(object.getString(LOCAL_KEY_FILENAME))
.setChecksum(object.getString(LOCAL_KEY_CHECKSUM))
.setDownloadChecksum(object.getString(LOCAL_KEY_DOWNLOAD_CHECKSUM))
.setLastModified(object.getLong(LOCAL_KEY_LAST_MODIFIED))
.setType(object.getString(LOCAL_KEY_TYPE))
.setKind(object.getString(LOCAL_KEY_KIND))
.setSize(object.getLong(LOCAL_KEY_SIZE))
.setState(object.getInt(LOCAL_KEY_STATE))
.setFailures(object.optInt(LOCAL_KEY_FAILURES), object.optInt(LOCAL_KEY_LAST_FAILURE_TYPE))
.setAppVersionPattern(object.optString(LOCAL_KEY_PATTERN_APP_VERSION))
.setAppIdPattern(object.optString(LOCAL_KEY_PATTERN_APP_ID))
.setAndroidApiPattern(object.optString(LOCAL_KEY_PATTERN_ANDROID_API))
.build();
}
public static JSONObject toJSON(DownloadContent content) throws JSONException {
final JSONObject object = new JSONObject();
object.put(LOCAL_KEY_ID, content.getId());
object.put(LOCAL_KEY_LOCATION, content.getLocation());
object.put(LOCAL_KEY_FILENAME, content.getFilename());
object.put(LOCAL_KEY_CHECKSUM, content.getChecksum());
object.put(LOCAL_KEY_DOWNLOAD_CHECKSUM, content.getDownloadChecksum());
object.put(LOCAL_KEY_LAST_MODIFIED, content.getLastModified());
object.put(LOCAL_KEY_TYPE, content.getType());
object.put(LOCAL_KEY_KIND, content.getKind());
object.put(LOCAL_KEY_SIZE, content.getSize());
object.put(LOCAL_KEY_STATE, content.getState());
object.put(LOCAL_KEY_PATTERN_APP_VERSION, content.getAppVersionPattern());
object.put(LOCAL_KEY_PATTERN_APP_ID, content.getAppIdPattern());
object.put(LOCAL_KEY_PATTERN_ANDROID_API, content.getAndroidApiPattern());
final int failures = content.getFailures();
if (failures > 0) {
object.put(LOCAL_KEY_FAILURES, failures);
object.put(LOCAL_KEY_LAST_FAILURE_TYPE, content.getLastFailureType());
}
return object;
}
public DownloadContent build() {
DownloadContent content = new DownloadContent(id, location, filename, checksum,
downloadChecksum, lastModified, type, kind, size, failures, lastFailureType,
appVersionPattern, androidApiPattern, appIdPattern);
content.setState(state);
return content;
}
public DownloadContentBuilder setId(String id) {
this.id = id;
return this;
}
public DownloadContentBuilder setLocation(String location) {
this.location = location;
return this;
}
public DownloadContentBuilder setFilename(String filename) {
this.filename = filename;
return this;
}
public DownloadContentBuilder setChecksum(String checksum) {
this.checksum = checksum;
return this;
}
public DownloadContentBuilder setDownloadChecksum(String downloadChecksum) {
this.downloadChecksum = downloadChecksum;
return this;
}
public DownloadContentBuilder setLastModified(long lastModified) {
this.lastModified = lastModified;
return this;
}
public DownloadContentBuilder setType(String type) {
this.type = type;
return this;
}
public DownloadContentBuilder setKind(String kind) {
this.kind = kind;
return this;
}
public DownloadContentBuilder setSize(long size) {
this.size = size;
return this;
}
public DownloadContentBuilder setState(int state) {
this.state = state;
return this;
}
/* package-private */ DownloadContentBuilder setFailures(int failures, int lastFailureType) {
this.failures = failures;
this.lastFailureType = lastFailureType;
return this;
}
public DownloadContentBuilder setAppVersionPattern(String appVersionPattern) {
this.appVersionPattern = appVersionPattern;
return this;
}
public DownloadContentBuilder setAndroidApiPattern(String androidApiPattern) {
this.androidApiPattern = androidApiPattern;
return this;
}
public DownloadContentBuilder setAppIdPattern(String appIdPattern) {
this.appIdPattern = appIdPattern;
return this;
}
public DownloadContentBuilder updateFromKinto(JSONObject object) throws JSONException {
final String objectId = object.getString(KINTO_KEY_ID);
if (TextUtils.isEmpty(id)) {
// New object without an id yet
id = objectId;
} else if (!id.equals(objectId)) {
throw new JSONException(String.format("Record ids do not match: Expected=%s, Actual=%s", id, objectId));
}
setType(object.getString(KINTO_KEY_TYPE));
setKind(object.getString(KINTO_KEY_KIND));
setLastModified(object.getLong(KINTO_KEY_LAST_MODIFIED));
JSONObject attachment = object.getJSONObject(KINTO_KEY_ATTACHMENT);
JSONObject original = attachment.getJSONObject(KINTO_KEY_ORIGINAL);
setFilename(original.getString(KINTO_KEY_FILENAME));
setChecksum(original.getString(KINTO_KEY_HASH));
setSize(original.getLong(KINTO_KEY_SIZE));
setLocation(attachment.getString(KINTO_KEY_LOCATION));
setDownloadChecksum(attachment.getString(KINTO_KEY_HASH));
JSONObject match = object.optJSONObject(KINTO_KEY_MATCH);
if (match != null) {
setAndroidApiPattern(match.optString(KINTO_KEY_ANDROID_API));
setAppIdPattern(match.optString(KINTO_KEY_APP_ID));
setAppVersionPattern(match.optString(KINTO_KEY_APP_VERSION));
}
return this;
}
}

303
mobile/android/base/java/org/mozilla/gecko/dlc/catalog/DownloadContentCatalog.java Normal file
View file

@ -0,0 +1,303 @@
/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.dlc.catalog;
import android.content.Context;
import android.support.annotation.Nullable;
import android.support.v4.util.ArrayMap;
import android.support.v4.util.AtomicFile;
import android.util.Log;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;
/**
* Catalog of downloadable content (DLC).
*
* Changing elements returned by the catalog should be guarded by the catalog instance to guarantee visibility when
* persisting changes.
*/
public class DownloadContentCatalog {
private static final String LOGTAG = "GeckoDLCCatalog";
private static final String FILE_NAME = "download_content_catalog";
private static final String JSON_KEY_CONTENT = "content";
private static final int MAX_FAILURES_UNTIL_PERMANENTLY_FAILED = 10;
private final AtomicFile file; // Guarded by 'file'
private ArrayMap<String, DownloadContent> content; // Guarded by 'this'
private boolean hasLoadedCatalog; // Guarded by 'this
private boolean hasCatalogChanged; // Guarded by 'this'
public DownloadContentCatalog(Context context) {
this(new AtomicFile(new File(context.getApplicationInfo().dataDir, FILE_NAME)));
startLoadFromDisk();
}
// For injecting mocked AtomicFile objects during test
protected DownloadContentCatalog(AtomicFile file) {
this.content = new ArrayMap<>();
this.file = file;
}
public List<DownloadContent> getContentToStudy() {
return filterByState(DownloadContent.STATE_NONE, DownloadContent.STATE_UPDATED);
}
public List<DownloadContent> getContentToDelete() {
return filterByState(DownloadContent.STATE_DELETED);
}
public List<DownloadContent> getDownloadedContent() {
return filterByState(DownloadContent.STATE_DOWNLOADED);
}
public List<DownloadContent> getScheduledDownloads() {
return filterByState(DownloadContent.STATE_SCHEDULED);
}
private synchronized List<DownloadContent> filterByState(@DownloadContent.State int... filterStates) {
awaitLoadingCatalogLocked();
List<DownloadContent> filteredContent = new ArrayList<>();
for (DownloadContent currentContent : content.values()) {
if (currentContent.isStateIn(filterStates)) {
filteredContent.add(currentContent);
}
}
return filteredContent;
}
public boolean hasScheduledDownloads() {
return !filterByState(DownloadContent.STATE_SCHEDULED).isEmpty();
}
public synchronized void add(DownloadContent newContent) {
awaitLoadingCatalogLocked();
content.put(newContent.getId(), newContent);
hasCatalogChanged = true;
}
public synchronized void update(DownloadContent changedContent) {
awaitLoadingCatalogLocked();
if (!content.containsKey(changedContent.getId())) {
Log.w(LOGTAG, "Did not find content with matching id (" + changedContent.getId() + ") to update");
return;
}
changedContent.setState(DownloadContent.STATE_UPDATED);
changedContent.resetFailures();
content.put(changedContent.getId(), changedContent);
hasCatalogChanged = true;
}
public synchronized void remove(DownloadContent removedContent) {
awaitLoadingCatalogLocked();
if (!content.containsKey(removedContent.getId())) {
Log.w(LOGTAG, "Did not find content with matching id (" + removedContent.getId() + ") to remove");
return;
}
content.remove(removedContent.getId());
}
@Nullable
public synchronized DownloadContent getContentById(String id) {
return content.get(id);
}
public synchronized long getLastModified() {
awaitLoadingCatalogLocked();
long lastModified = 0;
for (DownloadContent currentContent : content.values()) {
if (currentContent.getLastModified() > lastModified) {
lastModified = currentContent.getLastModified();
}
}
return lastModified;
}
public synchronized void scheduleDownload(DownloadContent content) {
content.setState(DownloadContent.STATE_SCHEDULED);
hasCatalogChanged = true;
}
public synchronized void markAsDownloaded(DownloadContent content) {
content.setState(DownloadContent.STATE_DOWNLOADED);
content.resetFailures();
hasCatalogChanged = true;
}
public synchronized void markAsPermanentlyFailed(DownloadContent content) {
content.setState(DownloadContent.STATE_FAILED);
hasCatalogChanged = true;
}
public synchronized void markAsDeleted(DownloadContent content) {
content.setState(DownloadContent.STATE_DELETED);
hasCatalogChanged = true;
}
public synchronized void rememberFailure(DownloadContent content, int failureType) {
if (content.getFailures() >= MAX_FAILURES_UNTIL_PERMANENTLY_FAILED) {
Log.d(LOGTAG, "Maximum number of failures reached. Marking content has permanently failed.");
markAsPermanentlyFailed(content);
} else {
content.rememberFailure(failureType);
hasCatalogChanged = true;
}
}
public void persistChanges() {
new Thread(LOGTAG + "-Persist") {
public void run() {
writeToDisk();
}
}.start();
}
private void startLoadFromDisk() {
new Thread(LOGTAG + "-Load") {
public void run() {
loadFromDisk();
}
}.start();
}
private void awaitLoadingCatalogLocked() {
while (!hasLoadedCatalog) {
try {
Log.v(LOGTAG, "Waiting for catalog to be loaded");
wait();
} catch (InterruptedException e) {
// Ignore
}
}
}
protected synchronized boolean hasCatalogChanged() {
return hasCatalogChanged;
}
protected synchronized void loadFromDisk() {
Log.d(LOGTAG, "Loading from disk");
if (hasLoadedCatalog) {
return;
}
ArrayMap<String, DownloadContent> loadedContent = new ArrayMap<>();
try {
JSONObject catalog;
synchronized (file) {
catalog = new JSONObject(new String(file.readFully(), "UTF-8"));
}
JSONArray array = catalog.getJSONArray(JSON_KEY_CONTENT);
for (int i = 0; i < array.length(); i++) {
DownloadContent currentContent = DownloadContentBuilder.fromJSON(array.getJSONObject(i));
loadedContent.put(currentContent.getId(), currentContent);
}
} catch (FileNotFoundException e) {
Log.d(LOGTAG, "Catalog file does not exist: Bootstrapping initial catalog");
loadedContent = DownloadContentBootstrap.createInitialDownloadContentList();
} catch (JSONException e) {
Log.w(LOGTAG, "Unable to parse catalog JSON. Re-creating catalog.", e);
// Catalog seems to be broken. Re-create catalog:
loadedContent = DownloadContentBootstrap.createInitialDownloadContentList();
hasCatalogChanged = true; // Indicate that we want to persist the new catalog
} catch (NullPointerException e) {
// Bad content can produce an NPE in JSON code -- bug 1300139
Log.w(LOGTAG, "Unable to parse catalog JSON. Re-creating catalog.", e);
// Catalog seems to be broken. Re-create catalog:
loadedContent = DownloadContentBootstrap.createInitialDownloadContentList();
hasCatalogChanged = true; // Indicate that we want to persist the new catalog
} catch (UnsupportedEncodingException e) {
AssertionError error = new AssertionError("Should not happen: This device does not speak UTF-8");
error.initCause(e);
throw error;
} catch (IOException e) {
Log.d(LOGTAG, "Can't read catalog due to IOException", e);
}
onCatalogLoaded(loadedContent);
notifyAll();
Log.d(LOGTAG, "Loaded " + content.size() + " elements");
}
protected void onCatalogLoaded(ArrayMap<String, DownloadContent> content) {
this.content = content;
this.hasLoadedCatalog = true;
}
protected synchronized void writeToDisk() {
if (!hasCatalogChanged) {
Log.v(LOGTAG, "Not persisting: Catalog has not changed");
return;
}
Log.d(LOGTAG, "Writing to disk");
FileOutputStream outputStream = null;
synchronized (file) {
try {
outputStream = file.startWrite();
JSONArray array = new JSONArray();
for (DownloadContent currentContent : content.values()) {
array.put(DownloadContentBuilder.toJSON(currentContent));
}
JSONObject catalog = new JSONObject();
catalog.put(JSON_KEY_CONTENT, array);
outputStream.write(catalog.toString().getBytes("UTF-8"));
file.finishWrite(outputStream);
hasCatalogChanged = false;
} catch (UnsupportedEncodingException e) {
AssertionError error = new AssertionError("Should not happen: This device does not speak UTF-8");
error.initCause(e);
throw error;
} catch (IOException | JSONException e) {
Log.e(LOGTAG, "IOException during writing catalog", e);
if (outputStream != null) {
file.failWrite(outputStream);
}
}
}
}
}