import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

toolkit/components/places/BookmarkJSONUtils.jsm

@@ -0,0 +1,589 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = [ "BookmarkJSONUtils" ];
+
+const Ci = Components.interfaces;
+const Cc = Components.classes;
+const Cu = Components.utils;
+const Cr = Components.results;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/NetUtil.jsm");
+Cu.import("resource://gre/modules/osfile.jsm");
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource://gre/modules/PromiseUtils.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PlacesBackups",
+  "resource://gre/modules/PlacesBackups.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Deprecated",
+  "resource://gre/modules/Deprecated.jsm");
+
+XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => new TextDecoder());
+XPCOMUtils.defineLazyGetter(this, "gTextEncoder", () => new TextEncoder());
+
+/**
+ * Generates an hash for the given string.
+ *
+ * @note The generated hash is returned in base64 form.  Mind the fact base64
+ * is case-sensitive if you are going to reuse this code.
+ */
+function generateHash(aString) {
+  let cryptoHash = Cc["@mozilla.org/security/hash;1"]
+                     .createInstance(Ci.nsICryptoHash);
+  cryptoHash.init(Ci.nsICryptoHash.MD5);
+  let stringStream = Cc["@mozilla.org/io/string-input-stream;1"]
+                       .createInstance(Ci.nsIStringInputStream);
+  stringStream.data = aString;
+  cryptoHash.updateFromStream(stringStream, -1);
+  // base64 allows the '/' char, but we can't use it for filenames.
+  return cryptoHash.finish(true).replace(/\//g, "-");
+}
+
+this.BookmarkJSONUtils = Object.freeze({
+  /**
+   * Import bookmarks from a url.
+   *
+   * @param aSpec
+   *        url of the bookmark data.
+   * @param aReplace
+   *        Boolean if true, replace existing bookmarks, else merge.
+   *
+   * @return {Promise}
+   * @resolves When the new bookmarks have been created.
+   * @rejects JavaScript exception.
+   */
+  importFromURL: function BJU_importFromURL(aSpec, aReplace) {
+    return Task.spawn(function* () {
+      notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN);
+      try {
+        let importer = new BookmarkImporter(aReplace);
+        yield importer.importFromURL(aSpec);
+
+        notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS);
+      } catch (ex) {
+        Cu.reportError("Failed to restore bookmarks from " + aSpec + ": " + ex);
+        notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED);
+      }
+    });
+  },
+
+  /**
+   * Restores bookmarks and tags from a JSON file.
+   * @note any item annotated with "places/excludeFromBackup" won't be removed
+   *       before executing the restore.
+   *
+   * @param aFilePath
+   *        OS.File path string of bookmarks in JSON or JSONlz4 format to be restored.
+   * @param aReplace
+   *        Boolean if true, replace existing bookmarks, else merge.
+   *
+   * @return {Promise}
+   * @resolves When the new bookmarks have been created.
+   * @rejects JavaScript exception.
+   * @deprecated passing an nsIFile is deprecated
+   */
+  importFromFile: function BJU_importFromFile(aFilePath, aReplace) {
+    if (aFilePath instanceof Ci.nsIFile) {
+      Deprecated.warning("Passing an nsIFile to BookmarksJSONUtils.importFromFile " +
+                         "is deprecated. Please use an OS.File path string instead.",
+                         "https://developer.mozilla.org/docs/JavaScript_OS.File");
+      aFilePath = aFilePath.path;
+    }
+
+    return Task.spawn(function* () {
+      notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_BEGIN);
+      try {
+        if (!(yield OS.File.exists(aFilePath)))
+          throw new Error("Cannot restore from nonexisting json file");
+
+        let importer = new BookmarkImporter(aReplace);
+        if (aFilePath.endsWith("jsonlz4")) {
+          yield importer.importFromCompressedFile(aFilePath);
+        } else {
+          yield importer.importFromURL(OS.Path.toFileURI(aFilePath));
+        }
+        notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_SUCCESS);
+      } catch (ex) {
+        Cu.reportError("Failed to restore bookmarks from " + aFilePath + ": " + ex);
+        notifyObservers(PlacesUtils.TOPIC_BOOKMARKS_RESTORE_FAILED);
+        throw ex;
+      }
+    });
+  },
+
+  /**
+   * Serializes bookmarks using JSON, and writes to the supplied file path.
+   *
+   * @param aFilePath
+   *        OS.File path string for the bookmarks file to be created.
+   * @param [optional] aOptions
+   *        Object containing options for the export:
+   *         - failIfHashIs: if the generated file would have the same hash
+   *                         defined here, will reject with ex.becauseSameHash
+   *         - compress: if true, writes file using lz4 compression
+   * @return {Promise}
+   * @resolves once the file has been created, to an object with the
+   *           following properties:
+   *            - count: number of exported bookmarks
+   *            - hash: file hash for contents comparison
+   * @rejects JavaScript exception.
+   * @deprecated passing an nsIFile is deprecated
+   */
+  exportToFile: function BJU_exportToFile(aFilePath, aOptions={}) {
+    if (aFilePath instanceof Ci.nsIFile) {
+      Deprecated.warning("Passing an nsIFile to BookmarksJSONUtils.exportToFile " +
+                         "is deprecated. Please use an OS.File path string instead.",
+                         "https://developer.mozilla.org/docs/JavaScript_OS.File");
+      aFilePath = aFilePath.path;
+    }
+    return Task.spawn(function* () {
+      let [bookmarks, count] = yield PlacesBackups.getBookmarksTree();
+      let startTime = Date.now();
+      let jsonString = JSON.stringify(bookmarks);
+      // Report the time taken to convert the tree to JSON.
+      try {
+        Services.telemetry
+                .getHistogramById("PLACES_BACKUPS_TOJSON_MS")
+                .add(Date.now() - startTime);
+      } catch (ex) {
+        Components.utils.reportError("Unable to report telemetry.");
+      }
+
+      let hash = generateHash(jsonString);
+
+      if (hash === aOptions.failIfHashIs) {
+        let e = new Error("Hash conflict");
+        e.becauseSameHash = true;
+        throw e;
+      }
+
+      // Do not write to the tmp folder, otherwise if it has a different
+      // filesystem writeAtomic will fail.  Eventual dangling .tmp files should
+      // be cleaned up by the caller.
+      let writeOptions = { tmpPath: OS.Path.join(aFilePath + ".tmp") };
+      if (aOptions.compress)
+        writeOptions.compression = "lz4";
+
+      yield OS.File.writeAtomic(aFilePath, jsonString, writeOptions);
+      return { count: count, hash: hash };
+    });
+  }
+});
+
+function BookmarkImporter(aReplace) {
+  this._replace = aReplace;
+  // The bookmark change source, used to determine the sync status and change
+  // counter.
+  this._source = aReplace ? PlacesUtils.bookmarks.SOURCE_IMPORT_REPLACE :
+                            PlacesUtils.bookmarks.SOURCE_IMPORT;
+}
+BookmarkImporter.prototype = {
+  /**
+   * Import bookmarks from a url.
+   *
+   * @param aSpec
+   *        url of the bookmark data.
+   *
+   * @return {Promise}
+   * @resolves When the new bookmarks have been created.
+   * @rejects JavaScript exception.
+   */
+  importFromURL(spec) {
+    return new Promise((resolve, reject) => {
+      let streamObserver = {
+        onStreamComplete: (aLoader, aContext, aStatus, aLength, aResult) => {
+          let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+                          createInstance(Ci.nsIScriptableUnicodeConverter);
+          converter.charset = "UTF-8";
+          try {
+            let jsonString = converter.convertFromByteArray(aResult,
+                                                            aResult.length);
+            resolve(this.importFromJSON(jsonString));
+          } catch (ex) {
+            Cu.reportError("Failed to import from URL: " + ex);
+            reject(ex);
+          }
+        }
+      };
+
+      let uri = NetUtil.newURI(spec);
+      let channel = NetUtil.newChannel({
+        uri: uri,
+        loadUsingSystemPrincipal: true
+      });
+      let streamLoader = Cc["@mozilla.org/network/stream-loader;1"]
+                           .createInstance(Ci.nsIStreamLoader);
+      streamLoader.init(streamObserver);
+      channel.asyncOpen2(streamLoader);
+    });
+  },
+
+  /**
+   * Import bookmarks from a compressed file.
+   *
+   * @param aFilePath
+   *        OS.File path string of the bookmark data.
+   *
+   * @return {Promise}
+   * @resolves When the new bookmarks have been created.
+   * @rejects JavaScript exception.
+   */
+  importFromCompressedFile: function* BI_importFromCompressedFile(aFilePath) {
+      let aResult = yield OS.File.read(aFilePath, { compression: "lz4" });
+      let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+                        createInstance(Ci.nsIScriptableUnicodeConverter);
+      converter.charset = "UTF-8";
+      let jsonString = converter.convertFromByteArray(aResult, aResult.length);
+      yield this.importFromJSON(jsonString);
+  },
+
+  /**
+   * Import bookmarks from a JSON string.
+   *
+   * @param aString
+   *        JSON string of serialized bookmark data.
+   */
+  importFromJSON: Task.async(function* (aString) {
+    this._importPromises = [];
+    let deferred = PromiseUtils.defer();
+    let nodes =
+      PlacesUtils.unwrapNodes(aString, PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER);
+
+    if (nodes.length == 0 || !nodes[0].children ||
+        nodes[0].children.length == 0) {
+      deferred.resolve(); // Nothing to restore
+    } else {
+      // Ensure tag folder gets processed last
+      nodes[0].children.sort(function sortRoots(aNode, bNode) {
+        if (aNode.root && aNode.root == "tagsFolder")
+          return 1;
+        if (bNode.root && bNode.root == "tagsFolder")
+          return -1;
+        return 0;
+      });
+
+      let batch = {
+        nodes: nodes[0].children,
+        runBatched: function runBatched() {
+          if (this._replace) {
+            // Get roots excluded from the backup, we will not remove them
+            // before restoring.
+            let excludeItems = PlacesUtils.annotations.getItemsWithAnnotation(
+                                 PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO);
+            // Delete existing children of the root node, excepting:
+            // 1. special folders: delete the child nodes
+            // 2. tags folder: untag via the tagging api
+            let root = PlacesUtils.getFolderContents(PlacesUtils.placesRootId,
+                                                   false, false).root;
+            let childIds = [];
+            for (let i = 0; i < root.childCount; i++) {
+              let childId = root.getChild(i).itemId;
+              if (!excludeItems.includes(childId) &&
+                  childId != PlacesUtils.tagsFolderId) {
+                childIds.push(childId);
+              }
+            }
+            root.containerOpen = false;
+
+            for (let i = 0; i < childIds.length; i++) {
+              let rootItemId = childIds[i];
+              if (PlacesUtils.isRootItem(rootItemId)) {
+                PlacesUtils.bookmarks.removeFolderChildren(rootItemId,
+                                                           this._source);
+              } else {
+                PlacesUtils.bookmarks.removeItem(rootItemId, this._source);
+              }
+            }
+          }
+
+          let searchIds = [];
+          let folderIdMap = [];
+
+          for (let node of batch.nodes) {
+            if (!node.children || node.children.length == 0)
+              continue; // Nothing to restore for this root
+
+            if (node.root) {
+              let container = PlacesUtils.placesRootId; // Default to places root
+              switch (node.root) {
+                case "bookmarksMenuFolder":
+                  container = PlacesUtils.bookmarksMenuFolderId;
+                  break;
+                case "tagsFolder":
+                  container = PlacesUtils.tagsFolderId;
+                  break;
+                case "unfiledBookmarksFolder":
+                  container = PlacesUtils.unfiledBookmarksFolderId;
+                  break;
+                case "toolbarFolder":
+                  container = PlacesUtils.toolbarFolderId;
+                  break;
+                case "mobileFolder":
+                  container = PlacesUtils.mobileFolderId;
+                  break;
+              }
+
+              // Insert the data into the db
+              for (let child of node.children) {
+                let index = child.index;
+                let [folders, searches] =
+                  this.importJSONNode(child, container, index, 0);
+                for (let i = 0; i < folders.length; i++) {
+                  if (folders[i])
+                    folderIdMap[i] = folders[i];
+                }
+                searchIds = searchIds.concat(searches);
+              }
+            } else {
+              let [folders, searches] = this.importJSONNode(
+                node, PlacesUtils.placesRootId, node.index, 0);
+              for (let i = 0; i < folders.length; i++) {
+                if (folders[i])
+                  folderIdMap[i] = folders[i];
+              }
+              searchIds = searchIds.concat(searches);
+            }
+          }
+
+          // Fixup imported place: uris that contain folders
+          for (let id of searchIds) {
+            let oldURI = PlacesUtils.bookmarks.getBookmarkURI(id);
+            let uri = fixupQuery(oldURI, folderIdMap);
+            if (!uri.equals(oldURI)) {
+              PlacesUtils.bookmarks.changeBookmarkURI(id, uri, this._source);
+            }
+          }
+
+          deferred.resolve();
+        }.bind(this)
+      };
+
+      PlacesUtils.bookmarks.runInBatchMode(batch, null);
+    }
+    yield deferred.promise;
+    // TODO (bug 1095426) once converted to the new bookmarks API, methods will
+    // yield, so this hack should not be needed anymore.
+    try {
+      yield Promise.all(this._importPromises);
+    } finally {
+      delete this._importPromises;
+    }
+  }),
+
+  /**
+   * Takes a JSON-serialized node and inserts it into the db.
+   *
+   * @param aData
+   *        The unwrapped data blob of dropped or pasted data.
+   * @param aContainer
+   *        The container the data was dropped or pasted into
+   * @param aIndex
+   *        The index within the container the item was dropped or pasted at
+   * @return an array containing of maps of old folder ids to new folder ids,
+   *         and an array of saved search ids that need to be fixed up.
+   *         eg: [[[oldFolder1, newFolder1]], [search1]]
+   */
+  importJSONNode: function BI_importJSONNode(aData, aContainer, aIndex,
+                                             aGrandParentId) {
+    let folderIdMap = [];
+    let searchIds = [];
+    let id = -1;
+    switch (aData.type) {
+      case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
+        if (aContainer == PlacesUtils.tagsFolderId) {
+          // Node is a tag
+          if (aData.children) {
+            for (let child of aData.children) {
+              try {
+                PlacesUtils.tagging.tagURI(
+                  NetUtil.newURI(child.uri), [aData.title], this._source);
+              } catch (ex) {
+                // Invalid tag child, skip it
+              }
+            }
+            return [folderIdMap, searchIds];
+          }
+        } else if (aData.annos &&
+                   aData.annos.some(anno => anno.name == PlacesUtils.LMANNO_FEEDURI)) {
+          // Node is a livemark
+          let feedURI = null;
+          let siteURI = null;
+          aData.annos = aData.annos.filter(function(aAnno) {
+            switch (aAnno.name) {
+              case PlacesUtils.LMANNO_FEEDURI:
+                feedURI = NetUtil.newURI(aAnno.value);
+                return false;
+              case PlacesUtils.LMANNO_SITEURI:
+                siteURI = NetUtil.newURI(aAnno.value);
+                return false;
+              default:
+                return true;
+            }
+          });
+
+          if (feedURI) {
+            let lmPromise = PlacesUtils.livemarks.addLivemark({
+              title: aData.title,
+              feedURI: feedURI,
+              parentId: aContainer,
+              index: aIndex,
+              lastModified: aData.lastModified,
+              siteURI: siteURI,
+              guid: aData.guid,
+              source: this._source
+            }).then(aLivemark => {
+              let id = aLivemark.id;
+              if (aData.dateAdded)
+                PlacesUtils.bookmarks.setItemDateAdded(id, aData.dateAdded,
+                                                       this._source);
+              if (aData.annos && aData.annos.length)
+                PlacesUtils.setAnnotationsForItem(id, aData.annos,
+                                                  this._source);
+            });
+            this._importPromises.push(lmPromise);
+          }
+        } else {
+          let isMobileFolder = aData.annos &&
+                               aData.annos.some(anno => anno.name == PlacesUtils.MOBILE_ROOT_ANNO);
+          if (isMobileFolder) {
+            // Mobile bookmark folders are special: we move their children to
+            // the mobile root instead of importing them. We also rewrite
+            // queries to use the special folder ID, and ignore generic
+            // properties like timestamps and annotations set on the folder.
+            id = PlacesUtils.mobileFolderId;
+          } else {
+            // For other folders, set `id` so that we can import timestamps
+            // and annotations at the end of this function.
+            id = PlacesUtils.bookmarks.createFolder(
+                   aContainer, aData.title, aIndex, aData.guid, this._source);
+          }
+          folderIdMap[aData.id] = id;
+          // Process children
+          if (aData.children) {
+            for (let i = 0; i < aData.children.length; i++) {
+              let child = aData.children[i];
+              let [folders, searches] =
+                this.importJSONNode(child, id, i, aContainer);
+              for (let j = 0; j < folders.length; j++) {
+                if (folders[j])
+                  folderIdMap[j] = folders[j];
+              }
+              searchIds = searchIds.concat(searches);
+            }
+          }
+        }
+        break;
+      case PlacesUtils.TYPE_X_MOZ_PLACE:
+        id = PlacesUtils.bookmarks.insertBookmark(
+               aContainer, NetUtil.newURI(aData.uri), aIndex, aData.title, aData.guid, this._source);
+        if (aData.keyword) {
+          // POST data could be set in 2 ways:
+          // 1. new backups have a postData property
+          // 2. old backups have an item annotation
+          let postDataAnno = aData.annos &&
+                             aData.annos.find(anno => anno.name == PlacesUtils.POST_DATA_ANNO);
+          let postData = aData.postData || (postDataAnno && postDataAnno.value);
+          let kwPromise = PlacesUtils.keywords.insert({ keyword: aData.keyword,
+                                                        url: aData.uri,
+                                                        postData,
+                                                        source: this._source });
+          this._importPromises.push(kwPromise);
+        }
+        if (aData.tags) {
+          let tags = aData.tags.split(",").filter(aTag =>
+            aTag.length <= Ci.nsITaggingService.MAX_TAG_LENGTH);
+          if (tags.length) {
+            try {
+              PlacesUtils.tagging.tagURI(NetUtil.newURI(aData.uri), tags, this._source);
+            } catch (ex) {
+              // Invalid tag child, skip it.
+              Cu.reportError(`Unable to set tags "${tags.join(", ")}" for ${aData.uri}: ${ex}`);
+            }
+          }
+        }
+        if (aData.charset) {
+          PlacesUtils.annotations.setPageAnnotation(
+            NetUtil.newURI(aData.uri), PlacesUtils.CHARSET_ANNO, aData.charset,
+            0, Ci.nsIAnnotationService.EXPIRE_NEVER);
+        }
+        if (aData.uri.substr(0, 6) == "place:")
+          searchIds.push(id);
+        if (aData.icon) {
+          try {
+            // Create a fake faviconURI to use (FIXME: bug 523932)
+            let faviconURI = NetUtil.newURI("fake-favicon-uri:" + aData.uri);
+            PlacesUtils.favicons.replaceFaviconDataFromDataURL(
+              faviconURI, aData.icon, 0,
+              Services.scriptSecurityManager.getSystemPrincipal());
+            PlacesUtils.favicons.setAndFetchFaviconForPage(
+              NetUtil.newURI(aData.uri), faviconURI, false,
+              PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, null,
+              Services.scriptSecurityManager.getSystemPrincipal());
+          } catch (ex) {
+            Components.utils.reportError("Failed to import favicon data:" + ex);
+          }
+        }
+        if (aData.iconUri) {
+          try {
+            PlacesUtils.favicons.setAndFetchFaviconForPage(
+              NetUtil.newURI(aData.uri), NetUtil.newURI(aData.iconUri), false,
+              PlacesUtils.favicons.FAVICON_LOAD_NON_PRIVATE, null,
+              Services.scriptSecurityManager.getSystemPrincipal());
+          } catch (ex) {
+            Components.utils.reportError("Failed to import favicon URI:" + ex);
+          }
+        }
+        break;
+      case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
+        id = PlacesUtils.bookmarks.insertSeparator(aContainer, aIndex, aData.guid, this._source);
+        break;
+      default:
+        // Unknown node type
+    }
+
+    // Set generic properties, valid for all nodes except tags and the mobile
+    // root.
+    if (id != -1 && id != PlacesUtils.mobileFolderId &&
+        aContainer != PlacesUtils.tagsFolderId &&
+        aGrandParentId != PlacesUtils.tagsFolderId) {
+      if (aData.dateAdded)
+        PlacesUtils.bookmarks.setItemDateAdded(id, aData.dateAdded,
+                                               this._source);
+      if (aData.lastModified)
+        PlacesUtils.bookmarks.setItemLastModified(id, aData.lastModified,
+                                                  this._source);
+      if (aData.annos && aData.annos.length)
+        PlacesUtils.setAnnotationsForItem(id, aData.annos, this._source);
+    }
+
+    return [folderIdMap, searchIds];
+  }
+}
+
+function notifyObservers(topic) {
+  Services.obs.notifyObservers(null, topic, "json");
+}
+
+/**
+ * Replaces imported folder ids with their local counterparts in a place: URI.
+ *
+ * @param   aURI
+ *          A place: URI with folder ids.
+ * @param   aFolderIdMap
+ *          An array mapping old folder id to new folder ids.
+ * @returns the fixed up URI if all matched. If some matched, it returns
+ *          the URI with only the matching folders included. If none matched
+ *          it returns the input URI unchanged.
+ */
+function fixupQuery(aQueryURI, aFolderIdMap) {
+  let convert = function(str, p1, offset, s) {
+    return "folder=" + aFolderIdMap[p1];
+  }
+  let stringURI = aQueryURI.spec.replace(/folder=([0-9]+)/g, convert);
+
+  return NetUtil.newURI(stringURI);
+}