Dactyloidae iOS initial commit

mobile/ios/Storage/SQL/BrowserDB.swift

@@ -0,0 +1,226 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import XCGLogger
+import Deferred
+import Shared
+
+public let NotificationDatabaseWasRecreated = Notification.Name("NotificationDatabaseWasRecreated")
+
+private let log = Logger.syncLogger
+
+public typealias Args = [Any?]
+
+open class BrowserDB {
+    fileprivate let db: SwiftData
+
+    // SQLITE_MAX_VARIABLE_NUMBER = 999 by default. This controls how many ?s can
+    // appear in a query string.
+    open static let MaxVariableNumber = 999
+
+    public init(filename: String, secretKey: String? = nil, schema: Schema, files: FileAccessor) {
+        log.debug("Initializing BrowserDB: \(filename).")
+
+        let file = URL(fileURLWithPath: (try! files.getAndEnsureDirectory())).appendingPathComponent(filename).path
+
+        if AppConstants.BuildChannel == .developer && secretKey != nil {
+            log.debug("Will attempt to use encrypted DB: \(file) with secret = \(secretKey ?? "nil")")
+        }
+
+        self.db = SwiftData(filename: file, key: secretKey, prevKey: nil, schema: schema, files: files)
+    }
+
+    // Remove the DB op from the queue (by marking it cancelled), and if it is already running tell sqlite to cancel it.
+    // At any point the operation could complete on another thread, so it is held weakly.
+    // Swift compiler bug: failing to compile WeakRef<Cancellable> here.
+    public func cancel(databaseOperation: WeakRef<AnyObject>) {
+        weak var databaseOperation = databaseOperation.value as? Cancellable
+
+        db.suspendQueue()
+        defer {
+            db.resumeQueue()
+        }
+
+        databaseOperation?.cancel()
+        if databaseOperation?.running ?? false {
+            db.cancel()
+        }
+    }
+
+    // For testing purposes or other cases where we want to ensure that this `BrowserDB`
+    // instance has been initialized (schema is created/updated).
+    public func touch() -> Success {
+        return withConnection { connection -> Void in
+            guard let _ = connection as? ConcreteSQLiteDBConnection else {
+                throw DatabaseError(description: "Could not establish a database connection")
+            }
+        }
+    }
+
+    /*
+     * Opening a WAL-using database with a hot journal cannot complete in read-only mode.
+     * The supported mechanism for a read-only query against a WAL-using SQLite database is to use PRAGMA query_only,
+     * but this isn't all that useful for us, because we have a mixed read/write workload.
+     */
+    @discardableResult func withConnection<T>(flags: SwiftData.Flags = .readWriteCreate, _ callback: @escaping (_ connection: SQLiteDBConnection) throws -> T) -> Deferred<Maybe<T>> {
+        return db.withConnection(flags, callback)
+    }
+
+    func transaction<T>(_ callback: @escaping (_ connection: SQLiteDBConnection) throws -> T) -> Deferred<Maybe<T>> {
+        return db.transaction(callback)
+    }
+
+    @discardableResult func vacuum() -> Success {
+        log.debug("Vacuuming a BrowserDB.")
+
+        return withConnection({ connection -> Void in
+            try connection.vacuum()
+        })
+    }
+
+    @discardableResult func checkpoint() -> Success {
+        log.debug("Checkpointing a BrowserDB.")
+        
+        return transaction { connection in
+            connection.checkpoint()
+        }
+    }
+
+    public class func varlist(_ count: Int) -> String {
+        return "(" + Array(repeating: "?", count: count).joined(separator: ", ") + ")"
+    }
+
+    enum InsertOperation: String {
+        case Insert = "INSERT"
+        case Replace = "REPLACE"
+        case InsertOrIgnore = "INSERT OR IGNORE"
+        case InsertOrReplace = "INSERT OR REPLACE"
+        case InsertOrRollback = "INSERT OR ROLLBACK"
+        case InsertOrAbort = "INSERT OR ABORT"
+        case InsertOrFail = "INSERT OR FAIL"
+    }
+
+    /**
+     * Insert multiple sets of values into the given table.
+     *
+     * Assumptions:
+     * 1. The table exists and contains the provided columns.
+     * 2. Every item in `values` is the same length.
+     * 3. That length is the same as the length of `columns`.
+     * 4. Every value in each element of `values` is non-nil.
+     *
+     * If there are too many items to insert, multiple individual queries will run
+     * in sequence.
+     *
+     * A failure anywhere in the sequence will cause immediate return of failure, but
+     * will not roll back — use a transaction if you need one.
+     */
+    func bulkInsert(_ table: String, op: InsertOperation, columns: [String], values: [Args]) -> Success {
+        // Note that there's a limit to how many ?s can be in a single query!
+        // So here we execute 999 / (columns * rows) insertions per query.
+        // Note that we can't use variables for the column names, so those don't affect the count.
+        if values.isEmpty {
+            log.debug("No values to insert.")
+            return succeed()
+        }
+
+        let variablesPerRow = columns.count
+
+        // Sanity check.
+        assert(values[0].count == variablesPerRow)
+
+        let cols = columns.joined(separator: ", ")
+        let queryStart = "\(op.rawValue) INTO \(table) (\(cols)) VALUES "
+
+        let varString = BrowserDB.varlist(variablesPerRow)
+
+        let insertChunk: ([Args]) -> Success = { vals -> Success in
+            let valuesString = Array(repeating: varString, count: vals.count).joined(separator: ", ")
+            let args: Args = vals.flatMap { $0 }
+            return self.run(queryStart + valuesString, withArgs: args)
+        }
+
+        let rowCount = values.count
+        if (variablesPerRow * rowCount) < BrowserDB.MaxVariableNumber {
+            return insertChunk(values)
+        }
+
+        log.debug("Splitting bulk insert across multiple runs. I hope you started a transaction!")
+        let rowsPerInsert = (999 / variablesPerRow)
+        let chunks = chunk(values, by: rowsPerInsert)
+        log.debug("Inserting in \(chunks.count) chunks.")
+
+        // There's no real reason why we can't pass the ArraySlice here, except that I don't
+        // want to keep fighting Swift.
+        return walk(chunks, f: { insertChunk(Array($0)) })
+    }
+
+    func write(_ sql: String, withArgs args: Args? = nil) -> Deferred<Maybe<Int>> {
+        return withConnection { connection -> Int in
+            try connection.executeChange(sql, withArgs: args)
+            
+            let modified = connection.numberOfRowsModified
+            log.debug("Modified rows: \(modified).")
+            return modified
+        }
+    }
+
+    public func forceClose() {
+        db.forceClose()
+    }
+
+    public func reopenIfClosed() {
+        db.reopenIfClosed()
+    }
+
+    func run(_ sql: String, withArgs args: Args? = nil) -> Success {
+        return run([(sql, args)])
+    }
+
+    func run(_ commands: [String]) -> Success {
+        return run(commands.map { (sql: $0, args: nil) })
+    }
+
+    /**
+     * Runs an array of SQL commands. Note: These will all run in order in a transaction and will block
+     * the caller's thread until they've finished. If any of them fail the operation will abort (no more
+     * commands will be run) and the transaction will roll back, returning a DatabaseError.
+     */
+    func run(_ commands: [(sql: String, args: Args?)]) -> Success {
+        if commands.isEmpty {
+            return succeed()
+        }
+
+        return transaction { connection -> Void in
+            for (sql, args) in commands {
+                try connection.executeChange(sql, withArgs: args)
+            }
+        }
+    }
+
+    func runQuery<T>(_ sql: String, args: Args?, factory: @escaping (SDRow) -> T) -> Deferred<Maybe<Cursor<T>>> {
+        return withConnection { connection -> Cursor<T> in
+            connection.executeQuery(sql, factory: factory, withArgs: args)
+        }
+    }
+
+    func runQueryUnsafe<T, U>(_ sql: String, args: Args?, factory: @escaping (SDRow) -> T, block: @escaping (Cursor<T>) throws -> U) -> Deferred<Maybe<U>> {
+        return withConnection { connection -> U in
+            let cursor = connection.executeQueryUnsafe(sql, factory: factory, withArgs: args)
+            defer { cursor.close() }
+            return try block(cursor)
+        }
+    }
+
+    func queryReturnsResults(_ sql: String, args: Args? = nil) -> Deferred<Maybe<Bool>> {
+        return runQuery(sql, args: args, factory: { _ in true })
+         >>== { deferMaybe($0[0] ?? false) }
+    }
+
+    func queryReturnsNoResults(_ sql: String, args: Args? = nil) -> Deferred<Maybe<Bool>> {
+        return runQuery(sql, args: nil, factory: { _ in false })
+          >>== { deferMaybe($0[0] ?? true) }
+    }
+}

mobile/ios/Storage/SQL/LoginsSchema.swift

@@ -0,0 +1,123 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+
+let TableLoginsMirror = "loginsM"
+let TableLoginsLocal = "loginsL"
+let IndexLoginsOverrideHostname = "idx_loginsM_is_overridden_hostname"
+let IndexLoginsDeletedHostname = "idx_loginsL_is_deleted_hostname"
+
+private let AllTables: [String] = [
+    TableLoginsMirror,
+    TableLoginsLocal
+]
+
+private let log = Logger.syncLogger
+
+open class LoginsSchema: Schema {
+    static let DefaultVersion = 3
+    
+    public var name: String { return "LOGINS" }
+    public var version: Int { return LoginsSchema.DefaultVersion }
+
+    public init() {}
+
+    func run(_ db: SQLiteDBConnection, sql: String, args: Args? = nil) -> Bool {
+        do {
+            try db.executeChange(sql, withArgs: args)
+        } catch let err as NSError {
+            log.error("Error running SQL in LoginsSchema: \(err.localizedDescription)")
+            log.error("SQL was \(sql)")
+            return false
+        }
+
+        return true
+    }
+    
+    // TODO: transaction.
+    func run(_ db: SQLiteDBConnection, queries: [String]) -> Bool {
+        for sql in queries {
+            if !run(db, sql: sql, args: nil) {
+                return false
+            }
+        }
+        return true
+    }
+    
+    let indexIsOverriddenHostname =
+    "CREATE INDEX IF NOT EXISTS \(IndexLoginsOverrideHostname) ON \(TableLoginsMirror) (is_overridden, hostname)"
+    
+    let indexIsDeletedHostname =
+    "CREATE INDEX IF NOT EXISTS \(IndexLoginsDeletedHostname) ON \(TableLoginsLocal) (is_deleted, hostname)"
+    
+    public func create(_ db: SQLiteDBConnection) -> Bool {
+        let common =
+            "id INTEGER PRIMARY KEY AUTOINCREMENT" +
+                ", hostname TEXT NOT NULL" +
+                ", httpRealm TEXT" +
+                ", formSubmitURL TEXT" +
+                ", usernameField TEXT" +
+                ", passwordField TEXT" +
+                ", timesUsed INTEGER NOT NULL DEFAULT 0" +
+                ", timeCreated INTEGER NOT NULL" +
+                ", timeLastUsed INTEGER" +
+                ", timePasswordChanged INTEGER NOT NULL" +
+                ", username TEXT" +
+        ", password TEXT NOT NULL"
+        
+        let mirror = "CREATE TABLE IF NOT EXISTS \(TableLoginsMirror) (" +
+            common +
+            ", guid TEXT NOT NULL UNIQUE" +
+            ", server_modified INTEGER NOT NULL" +              // Integer milliseconds.
+            ", is_overridden TINYINT NOT NULL DEFAULT 0" +
+        ")"
+        
+        let local = "CREATE TABLE IF NOT EXISTS \(TableLoginsLocal) (" +
+            common +
+            ", guid TEXT NOT NULL UNIQUE " +                  // Typically overlaps one in the mirror unless locally new.
+            ", local_modified INTEGER" +                      // Can be null. Client clock. In extremis only.
+            ", is_deleted TINYINT NOT NULL DEFAULT 0" +       // Boolean. Locally deleted.
+            ", sync_status TINYINT " +                        // SyncStatus enum. Set when changed or created.
+            "NOT NULL DEFAULT \(SyncStatus.synced.rawValue)" +
+        ")"
+        return self.run(db, queries: [mirror, local, indexIsOverriddenHostname, indexIsDeletedHostname])
+    }
+    
+    public func update(_ db: SQLiteDBConnection, from: Int) -> Bool {
+        let to = self.version
+        if from == to {
+            log.debug("Skipping update from \(from) to \(to).")
+            return true
+        }
+        
+        if from == 0 {
+            // This is likely an upgrade from before Bug 1160399.
+            log.debug("Updating logins tables from zero. Assuming drop and recreate.")
+            return drop(db) && create(db)
+        }
+        
+        if from < 3 && to >= 3 {
+            log.debug("Updating logins tables to include version 3 indices")
+            return self.run(db, queries: [indexIsOverriddenHostname, indexIsDeletedHostname])
+        }
+        
+        // TODO: real update!
+        log.debug("Updating logins table from \(from) to \(to).")
+        return drop(db) && create(db)
+    }
+    
+    public func drop(_ db: SQLiteDBConnection) -> Bool {
+        log.debug("Dropping logins table.")
+        do {
+            try db.executeChange("DROP TABLE IF EXISTS \(name)")
+        } catch {
+            return false
+        }
+
+        return true
+    }
+}

mobile/ios/Storage/SQL/SQLiteBookmarksBase.swift

@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Deferred
+import Foundation
+import Shared
+
+private let log = Logger.syncLogger
+
+class NoSuchSearchKeywordError: MaybeErrorType {
+    let keyword: String
+    init(keyword: String) {
+        self.keyword = keyword
+    }
+    var description: String {
+        return "No such search keyword: \(keyword)."
+    }
+}
+
+open class SQLiteBookmarks: BookmarksModelFactorySource, KeywordSearchSource {
+    let db: BrowserDB
+    let favicons: SQLiteFavicons
+
+    static let defaultFolderTitle: String = NSLocalizedString("Untitled", tableName: "Storage", comment: "The default name for bookmark folders without titles.")
+    static let defaultItemTitle: String = NSLocalizedString("Untitled", tableName: "Storage", comment: "The default name for bookmark nodes without titles.")
+
+    open lazy var modelFactory: Deferred<Maybe<BookmarksModelFactory>> =
+        deferMaybe(SQLiteBookmarksModelFactory(bookmarks: self, direction: .local))
+
+    public init(db: BrowserDB) {
+        self.db = db
+        self.favicons = SQLiteFavicons(db: self.db)
+    }
+
+    open func isBookmarked(_ url: String, direction: Direction) -> Deferred<Maybe<Bool>> {
+        let sql = "SELECT id FROM " +
+            "(SELECT id FROM \(direction.valueTable) WHERE " +
+            " bmkUri = ? AND is_deleted IS NOT 1" +
+            " UNION ALL " +
+            " SELECT id FROM \(TableBookmarksMirror) WHERE " +
+            " bmkUri = ? AND is_deleted IS NOT 1 AND is_overridden IS NOT 1" +
+            " LIMIT 1)"
+        let args: Args = [url, url]
+
+        return self.db.queryReturnsResults(sql, args: args)
+    }
+
+    open func getURLForKeywordSearch(_ keyword: String) -> Deferred<Maybe<String>> {
+        let sql = "SELECT bmkUri FROM \(ViewBookmarksBufferOnMirror) WHERE " +
+        " keyword = ?"
+        let args: Args = [keyword]
+
+        return self.db.runQuery(sql, args: args, factory: { $0["bmkUri"] as! String })
+            >>== { cursor in
+                if cursor.status == .success {
+                    if let str = cursor[0] {
+                        return deferMaybe(str)
+                    }
+                }
+
+                return deferMaybe(NoSuchSearchKeywordError(keyword: keyword))
+        }
+    }
+}

mobile/ios/Storage/SQL/SQLiteBookmarksHelpers.swift

@@ -0,0 +1,75 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+
+public func titleForSpecialGUID(_ guid: GUID) -> String? {
+    switch guid {
+    case BookmarkRoots.RootGUID:
+        return "<Root>"
+    case BookmarkRoots.MobileFolderGUID:
+        return BookmarksFolderTitleMobile
+    case BookmarkRoots.ToolbarFolderGUID:
+        return BookmarksFolderTitleToolbar
+    case BookmarkRoots.MenuFolderGUID:
+        return BookmarksFolderTitleMenu
+    case BookmarkRoots.UnfiledFolderGUID:
+        return BookmarksFolderTitleUnsorted
+    default:
+        return nil
+    }
+}
+
+class BookmarkURLTooLargeError: MaybeErrorType {
+    init() {
+    }
+    var description: String {
+        return "URL too long to bookmark."
+    }
+}
+
+extension String {
+    public func truncateToUTF8ByteCount(_ keep: Int) -> String {
+        let byteCount = self.lengthOfBytes(using: String.Encoding.utf8)
+        if byteCount <= keep {
+            return self
+        }
+        let toDrop = keep - byteCount
+
+        // If we drop this many characters from the string, we will drop at least this many bytes.
+        // That's aggressive, but that's OK for our purposes.
+        guard let endpoint = self.index(self.endIndex, offsetBy: toDrop, limitedBy: self.startIndex) else {
+            return ""
+        }
+        return self.substring(to: endpoint)
+    }
+}
+
+extension SQLiteBookmarks: ShareToDestination {
+    public func addToMobileBookmarks(_ url: URL, title: String, favicon: Favicon?) -> Success {
+        if url.absoluteString.lengthOfBytes(using: String.Encoding.utf8) > AppConstants.DB_URL_LENGTH_MAX {
+            return deferMaybe(BookmarkURLTooLargeError())
+        }
+
+        let title = title.truncateToUTF8ByteCount(AppConstants.DB_TITLE_LENGTH_MAX)
+
+        return isBookmarked(String(describing: url), direction: Direction.local)
+            >>== { yes in
+                guard !yes else { return succeed() }
+                return self.insertBookmark(url, title: title, favicon: favicon,
+                                           intoFolder: BookmarkRoots.MobileFolderGUID,
+                                           withTitle: BookmarksFolderTitleMobile)
+        }
+    }
+
+    public func shareItem(_ item: ShareItem) -> Success {
+        // We parse here in anticipation of getting real URLs at some point.
+        if let url = item.url.asURL {
+            let title = item.title ?? url.absoluteString
+            return self.addToMobileBookmarks(url, title: title, favicon: item.favicon)
+        }
+        return succeed()
+    }
+}

mobile/ios/Storage/SQL/SQLiteBookmarksModel.swift

@@ -0,0 +1,948 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Deferred
+import Foundation
+import Shared
+
+private let log = Logger.syncLogger
+private let desktopBookmarksLabel: String = NSLocalizedString("Desktop Bookmarks", tableName: "BookmarkPanel", comment: "The folder name for the virtual folder that contains all desktop bookmarks.")
+
+public enum Direction {
+    case buffer
+    case local
+
+    var structureTable: String {
+        switch self {
+        case .local:
+            return TableBookmarksLocalStructure
+        case .buffer:
+            return TableBookmarksBufferStructure
+        }
+    }
+
+    var valueTable: String {
+        switch self {
+        case .local:
+            return TableBookmarksLocal
+        case .buffer:
+            return TableBookmarksBuffer
+        }
+    }
+
+    var valueView: String {
+        switch self {
+        case .local:
+            return ViewBookmarksLocalOnMirror
+        case .buffer:
+            return ViewBookmarksBufferWithDeletionsOnMirror
+        }
+    }
+
+    var structureView: String {
+        switch self {
+        case .local:
+            return ViewBookmarksLocalStructureOnMirror
+        case .buffer:
+            return ViewBookmarksBufferStructureOnMirror
+        }
+    }
+}
+
+public protocol KeywordSearchSource {
+    func getURLForKeywordSearch(_ keyword: String) -> Deferred<Maybe<String>>
+}
+
+open class SQLiteBookmarksModelFactory: BookmarksModelFactory {
+    fileprivate let bookmarks: SQLiteBookmarks
+    fileprivate let direction: Direction
+
+    public init(bookmarks: SQLiteBookmarks, direction: Direction) {
+        self.bookmarks = bookmarks
+        self.direction = direction
+    }
+
+    public func factoryForIndex(_ index: Int, inFolder folder: BookmarkFolder) -> BookmarksModelFactory {
+        return self
+    }
+
+    fileprivate func withDifferentDirection(_ direction: Direction) -> SQLiteBookmarksModelFactory {
+        if self.direction == direction {
+            return self
+        }
+        return SQLiteBookmarksModelFactory(bookmarks: self.bookmarks, direction: direction)
+    }
+
+    fileprivate func getChildrenWithParent(_ parentGUID: GUID, excludingGUIDs: [GUID]?=nil, includeIcon: Bool) -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+        return self.bookmarks.getChildrenWithParent(parentGUID, direction: self.direction, excludingGUIDs: excludingGUIDs, includeIcon: includeIcon)
+    }
+
+    fileprivate func getRootChildren() -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+        return self.getChildrenWithParent(BookmarkRoots.RootGUID, excludingGUIDs: [BookmarkRoots.RootGUID], includeIcon: true)
+    }
+
+    fileprivate func getChildren(_ guid: String) -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+        return self.getChildrenWithParent(guid, includeIcon: true)
+    }
+
+    func folderForGUID(_ guid: GUID, title: String) -> Deferred<Maybe<BookmarkFolder>> {
+        return self.getChildren(guid)
+            >>== { cursor in
+
+            if cursor.status == .failure {
+                return deferMaybe(DatabaseError(description: "Couldn't get children: \(cursor.statusMessage)."))
+            }
+
+            return deferMaybe(SQLiteBookmarkFolder(guid: guid, title: title, children: cursor))
+        }
+    }
+
+    fileprivate func modelWithRoot(_ root: BookmarkFolder) -> Deferred<Maybe<BookmarksModel>> {
+        return deferMaybe(BookmarksModel(modelFactory: self, root: root))
+    }
+
+    open func modelForFolder(_ guid: String, title: String) -> Deferred<Maybe<BookmarksModel>> {
+        if guid == BookmarkRoots.MobileFolderGUID {
+            return self.modelForRoot()
+        }
+
+        if guid == BookmarkRoots.FakeDesktopFolderGUID {
+            return self.modelForDesktopBookmarks()
+        }
+
+        let outputTitle = titleForSpecialGUID(guid) ?? title
+        return self.folderForGUID(guid, title: outputTitle)
+          >>== self.modelWithRoot
+    }
+
+    open func modelForFolder(_ folder: BookmarkFolder) -> Deferred<Maybe<BookmarksModel>> {
+        return self.modelForFolder(folder.guid, title: folder.title)
+    }
+
+    open func modelForFolder(_ guid: String) -> Deferred<Maybe<BookmarksModel>> {
+        return self.modelForFolder(guid, title: "")
+    }
+
+    open func modelForRoot() -> Deferred<Maybe<BookmarksModel>> {
+        log.debug("Getting model for root.")
+        let getFolder = self.folderForGUID(BookmarkRoots.MobileFolderGUID, title: BookmarksFolderTitleMobile)
+        if self.direction == .buffer {
+            return getFolder >>== self.modelWithRoot
+        }
+
+        // Return a virtual model containing "Desktop bookmarks" prepended to the local mobile bookmarks.
+        return getFolder >>== { folder in
+            self.extendWithDesktopBookmarksFolder(folder, factory: self)
+        }
+    }
+
+    open var nullModel: BookmarksModel {
+        let children = Cursor<BookmarkNode>(status: .failure, msg: "Null model")
+        let folder = SQLiteBookmarkFolder(guid: "Null", title: "Null", children: children)
+        return BookmarksModel(modelFactory: self, root: folder)
+    }
+
+    open func isBookmarked(_ url: String) -> Deferred<Maybe<Bool>> {
+        return self.bookmarks.isBookmarked(url, direction: self.direction)
+    }
+
+    open func removeByURL(_ url: String) -> Success {
+        if self.direction == Direction.buffer {
+            return deferMaybe(DatabaseError(description: "Refusing to remove URL from buffer in model."))
+        }
+
+        // Find all of the records for the provided URL. Don't bother with
+        // any that are already deleted!
+        return self.bookmarks.nonDeletedGUIDsForURL(url)
+          >>== self.bookmarks.removeGUIDs
+    }
+
+    open func removeByGUID(_ guid: GUID) -> Success {
+        if self.direction == Direction.buffer {
+            return deferMaybe(DatabaseError(description: "Refusing to remove GUID from buffer in model."))
+        }
+
+        log.debug("removeByGUID: \(guid)")
+        return self.bookmarks.removeGUIDs([guid])
+    }
+
+    func hasDesktopBookmarks() -> Deferred<Maybe<Bool>> {
+        // This is very lazy, but it has the nice property of keeping Desktop Bookmarks visible
+        // for a while after you mark the last desktop child as deleted.
+        let parents: Args = [
+            // Local.
+            BookmarkRoots.MenuFolderGUID,
+            BookmarkRoots.ToolbarFolderGUID,
+            BookmarkRoots.UnfiledFolderGUID,
+
+            // Mirror.
+            BookmarkRoots.MenuFolderGUID,
+            BookmarkRoots.ToolbarFolderGUID,
+            BookmarkRoots.UnfiledFolderGUID,
+        ]
+
+        let sql =
+        "SELECT 1 FROM \(self.direction.structureTable) WHERE parent IN (?, ?, ?)" +
+        " UNION ALL " +
+        "SELECT 1 FROM \(TableBookmarksMirrorStructure) WHERE parent IN (?, ?, ?)" +
+        " LIMIT 1"
+
+        return self.bookmarks.db.queryReturnsResults(sql, args: parents)
+    }
+
+    func getDesktopRoots() -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+        if self.direction == .buffer {
+            // DesktopRoots excludes the Mobile folder, local and non-local mobile are aggregated
+            return self.bookmarks.getRecordsWithGUIDs(BookmarkRoots.DesktopRoots, direction: self.direction, includeIcon: false)
+        }
+
+        // We deliberately exclude the mobile folder, because we're inverting the containment
+        // relationship here.
+        let exclude = [BookmarkRoots.MobileFolderGUID, BookmarkRoots.RootGUID]
+        return self.getChildrenWithParent(BookmarkRoots.RootGUID, excludingGUIDs: exclude, includeIcon: false)
+    }
+
+    /**
+     * Prepend the provided mobile bookmarks folder with a single folder.
+     * The prepended folder is "Desktop Bookmarks". It contains mirrored folders.
+     */
+    open func extendWithDesktopBookmarksFolder(_ mobile: BookmarkFolder, factory: BookmarksModelFactory) -> Deferred<Maybe<BookmarksModel>> {
+
+        func onlyMobile() -> Deferred<Maybe<BookmarksModel>> {
+            // No desktop bookmarks.
+            log.debug("No desktop bookmarks. Only showing mobile.")
+            return deferMaybe(BookmarksModel(modelFactory: factory, root: mobile))
+        }
+
+        return self.hasDesktopBookmarks() >>== { yes in
+            if !yes {
+                return onlyMobile()
+            }
+
+            return self.getDesktopRoots() >>== { cursor in
+                if cursor.count == 0 {
+                    // This shouldn't occur.
+                    return onlyMobile()
+                }
+
+                let desktop = self.folderForDesktopBookmarksCursor(cursor)
+                let prepended = PrependedBookmarkFolder(main: mobile, prepend: desktop)
+                return deferMaybe(BookmarksModel(modelFactory: factory, root: prepended))
+            }
+        }
+    }
+
+    fileprivate func modelForDesktopBookmarks() -> Deferred<Maybe<BookmarksModel>> {
+        return self.getDesktopRoots() >>== { cursor in
+            let desktop = self.folderForDesktopBookmarksCursor(cursor)
+            return deferMaybe(BookmarksModel(modelFactory: self, root: desktop))
+        }
+    }
+
+    fileprivate func folderForDesktopBookmarksCursor(_ cursor: Cursor<BookmarkNode>) -> SQLiteBookmarkFolder {
+        return SQLiteBookmarkFolder(guid: BookmarkRoots.FakeDesktopFolderGUID, title: desktopBookmarksLabel, children: cursor)
+    }
+}
+
+class EditableBufferBookmarksSQLiteBookmarksModelFactory: SQLiteBookmarksModelFactory {
+    override func getChildrenWithParent(_ parentGUID: GUID, excludingGUIDs: [GUID]?, includeIcon: Bool) -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+        if parentGUID == BookmarkRoots.MobileFolderGUID {
+            return self.bookmarks.getChildrenWithParent(parentGUID, direction: self.direction, excludingGUIDs: excludingGUIDs, includeIcon: includeIcon, factory: BookmarkFactory.editableItemsFactory)
+        }
+        return super.getChildrenWithParent(parentGUID, excludingGUIDs: excludingGUIDs, includeIcon: includeIcon)
+    }
+
+    override func removeByGUID(_ guid: GUID) -> Success {
+        log.debug("Removing \(guid) from buffer.")
+        return self.bookmarks.markBufferBookmarkAsDeleted(guid)
+    }
+}
+
+private func isEditableExpression(_ direction: Direction) -> String {
+    if direction == .buffer {
+        return "0"
+    }
+
+    return "SELECT exists( " +
+           "   SELECT exists(SELECT 1 FROM \(TableBookmarksBuffer)) AS hasBuffer, exists(SELECT 1 FROM \(TableBookmarksMirror)) AS hasMirror " +
+           "   WHERE hasBuffer IS 0 OR hasMirror IS 0" +
+           ")"
+}
+
+extension SQLiteBookmarks {
+
+    fileprivate func getRecordsWithGUIDs(_ guids: [GUID], direction: Direction, includeIcon: Bool) -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+
+        let args: Args = guids
+        let varlist = BrowserDB.varlist(args.count)
+        let values =
+        "SELECT -1 AS id, guid, type, date_added, is_deleted, parentid, parentName, feedUri, pos, title, bmkUri, siteUri, folderName, faviconID, (\(isEditableExpression(direction))) AS isEditable " +
+        "FROM \(direction.valueView) WHERE guid IN \(varlist) AND NOT is_deleted"
+
+        let withIcon = [
+            "SELECT bookmarks.id AS id, bookmarks.guid AS guid, bookmarks.type AS type,",
+            "       bookmarks.date_added AS date_added,",
+            "       bookmarks.is_deleted AS is_deleted,",
+            "       bookmarks.parentid AS parentid, bookmarks.parentName AS parentName,",
+            "       bookmarks.feedUri AS feedUri, bookmarks.pos AS pos, title AS title,",
+            "       bookmarks.bmkUri AS bmkUri, bookmarks.siteUri AS siteUri,",
+            "       bookmarks.folderName AS folderName,",
+            "       bookmarks.isEditable AS isEditable,",
+            "       favicons.url AS iconURL, favicons.date AS iconDate, favicons.type AS iconType",
+            "FROM (", values, ") AS bookmarks",
+            "LEFT OUTER JOIN favicons ON bookmarks.faviconID = favicons.id",
+            "ORDER BY title ASC",
+            ].joined(separator: " ")
+
+        let sql = (includeIcon ? withIcon : values) + " ORDER BY title ASC"
+        return self.db.runQuery(sql, args: args, factory: BookmarkFactory.factory)
+    }
+
+    /**
+     * Return the children of the provided parent.
+     * Rows are ordered by positional index.
+     * This method is aware of is_overridden and deletion, using local override structure by preference.
+     * Note that a folder can be empty locally; we thus use the flag rather than looking at the structure itself.
+     */
+    func getChildrenWithParent(_ parentGUID: GUID, direction: Direction, excludingGUIDs: [GUID]?=nil, includeIcon: Bool, factory: @escaping (SDRow) -> BookmarkNode = BookmarkFactory.factory) -> Deferred<Maybe<Cursor<BookmarkNode>>> {
+
+        precondition((excludingGUIDs ?? []).count < 100, "Sanity bound for the number of GUIDs we can exclude.")
+
+        let valueView = direction.valueView
+        let structureView = direction.structureView
+
+        let structure =
+        "SELECT parent, child AS guid, idx FROM \(structureView) " +
+        "WHERE parent = ?"
+
+        let values =
+        "SELECT -1 AS id, guid, type, date_added, is_deleted, parentid, parentName, feedUri, pos, title, bmkUri, siteUri, folderName, faviconID, (\(isEditableExpression(direction))) AS isEditable " +
+        "FROM \(valueView)"
+
+        // We exclude queries and dynamic containers, because we can't
+        // usefully display them.
+        let typeQuery = BookmarkNodeType.query.rawValue
+        let typeDynamic = BookmarkNodeType.dynamicContainer.rawValue
+        let typeFilter = " vals.type NOT IN (\(typeQuery), \(typeDynamic))"
+
+        let args: Args
+        let exclusion: String
+        if let excludingGUIDs = excludingGUIDs {
+            args = ([parentGUID] + excludingGUIDs).map { $0 }
+            exclusion = "\(typeFilter) AND vals.guid NOT IN " + BrowserDB.varlist(excludingGUIDs.count)
+        } else {
+            args = [parentGUID]
+            exclusion = typeFilter
+        }
+
+        let fleshed =
+        "SELECT vals.id AS id, vals.guid AS guid, vals.type AS type, vals.date_added AS date_added, vals.is_deleted AS is_deleted, " +
+        "       vals.parentid AS parentid, vals.parentName AS parentName, vals.feedUri AS feedUri, " +
+        "       vals.siteUri AS siteUri," +
+        "       vals.pos AS pos, vals.title AS title, vals.bmkUri AS bmkUri, vals.folderName AS folderName, " +
+        "       vals.faviconID AS faviconID, " +
+        "       vals.isEditable AS isEditable, " +
+        "       structure.idx AS idx, " +
+        "       structure.parent AS _parent " +
+        "FROM (\(structure)) AS structure JOIN (\(values)) AS vals " +
+        "ON vals.guid = structure.guid " +
+        "WHERE " + exclusion
+
+        let withIcon =
+        "SELECT bookmarks.id AS id, bookmarks.guid AS guid, bookmarks.type AS type, " +
+        "       bookmarks.date_added AS date_added, " +
+        "       bookmarks.is_deleted AS is_deleted, " +
+        "       bookmarks.parentid AS parentid, bookmarks.parentName AS parentName, " +
+        "       bookmarks.feedUri AS feedUri, bookmarks.siteUri AS siteUri, " +
+        "       bookmarks.pos AS pos, title AS title, " +
+        "       bookmarks.bmkUri AS bmkUri, bookmarks.folderName AS folderName, " +
+        "       bookmarks.idx AS idx, bookmarks._parent AS _parent, " +
+        "       bookmarks.isEditable AS isEditable, " +
+        "       favicons.url AS iconURL, favicons.date AS iconDate, favicons.type AS iconType " +
+        "FROM (\(fleshed)) AS bookmarks " +
+        "LEFT OUTER JOIN favicons ON bookmarks.faviconID = favicons.id"
+
+        let sql = (includeIcon ? withIcon : fleshed) + " ORDER BY idx ASC"
+        return self.db.runQuery(sql, args: args, factory: factory)
+    }
+
+    // This is only used from tests.
+    func clearBookmarks() -> Success {
+        log.warning("CALLING clearBookmarks -- this should only be used from tests.")
+        return self.db.run([
+            ("DELETE FROM \(TableBookmarksLocal) WHERE parentid IS NOT ?", [BookmarkRoots.RootGUID]),
+            self.favicons.getCleanupFaviconsQuery()
+        ])
+    }
+
+    public func removeGUIDs(_ guids: [GUID]) -> Success {
+        log.debug("removeGUIDs: \(guids)")
+
+        // Override any parents that aren't already overridden. We're about to remove some
+        // of their children.
+        return self.overrideParentsOfGUIDs(guids)
+
+        // Find, recursively, any children of the provided GUIDs. This will only be the case
+        // if you specify folders. This is a special case because we're removing *all*
+        // children, so we don't need to do the reindexing dance.
+           >>> { self.deleteChildrenOfGUIDs(guids) }
+
+        // Override any records that aren't already overridden.
+           >>> { self.overrideGUIDs(guids) }
+
+        // Then delete the already-overridden records. We do this one at a time in order
+        // to get indices correct in edge cases. (We do bulk-delete their children
+        // one layer at a time, at least.)
+           >>> { walk(guids, f: self.removeLocalByGUID) }
+    }
+
+    fileprivate func nonDeletedGUIDsForURL(_ url: String) -> Deferred<Maybe<([GUID])>> {
+        let sql = "SELECT DISTINCT guid FROM \(ViewBookmarksLocalOnMirror) WHERE bmkUri = ? AND is_deleted = 0"
+        let args: Args = [url]
+
+        return self.db.runQuery(sql, args: args, factory: { $0[0] as! GUID }) >>== { guids in
+            return deferMaybe(guids.asArray())
+        }
+    }
+
+    fileprivate func overrideParentsOfGUIDs(_ guids: [GUID]) -> Success {
+        log.debug("Overriding parents of \(guids).")
+
+        // TODO: Yes, this can be done in one go.
+        let getParentsSQL =
+        "SELECT DISTINCT parent FROM \(ViewBookmarksLocalStructureOnMirror) " +
+        "WHERE child IN \(BrowserDB.varlist(guids.count)) AND is_overridden = 0"
+        let getParentsArgs: Args = guids
+
+        return self.db.runQuery(getParentsSQL, args: getParentsArgs, factory: { $0[0] as! GUID })
+            >>== { parentsCursor in
+                let parents = parentsCursor.asArray()
+                log.debug("Overriding parents: \(parents).")
+                let (sql, args) = self.getSQLToOverrideFolders(parents, atModifiedTime: Date.now())
+                return self.db.run(sql.map { ($0, args) })
+        }
+    }
+
+    fileprivate func overrideGUIDs(_ guids: [GUID]) -> Success {
+        log.debug("Overriding GUIDs: \(guids).")
+        let (sql, args) = self.getSQLToOverrideNonFolders(guids, atModifiedTime: Date.now())
+        return self.db.run(sql.map { ($0, args) })
+    }
+
+    // Recursive.
+    fileprivate func deleteChildrenOfGUIDs(_ guids: [GUID]) -> Success {
+        if guids.isEmpty {
+            return succeed()
+        }
+
+        precondition(BookmarkRoots.All.intersection(guids).isEmpty, "You can't even touch the roots for removal.")
+
+        log.debug("Deleting children of \(guids).")
+
+        let topArgs: Args = guids
+        let topVarlist = BrowserDB.varlist(topArgs.count)
+        let query =
+        "SELECT child FROM \(ViewBookmarksLocalStructureOnMirror) " +
+        "WHERE parent IN \(topVarlist)"
+
+        // We're deleting whole folders, so we don't need to worry about indices.
+        return self.db.runQuery(query, args: topArgs, factory: { $0[0] as! GUID })
+            >>== { children in
+                let childGUIDs = children.asArray()
+                log.debug("… children of \(guids) are \(childGUIDs).")
+
+                if childGUIDs.isEmpty {
+                    log.debug("No children; nothing more to do.")
+                    return succeed()
+                }
+
+                let childArgs: Args = childGUIDs
+                let childVarlist = BrowserDB.varlist(childArgs.count)
+
+                // Mirror the children if they're not already.
+                // We use the non-folder version of this query because we're recursively
+                // destroying structure right after this, so there's no point cloning the
+                // mirror structure.
+                // Then delete the children's children, so we don't leave orphans. This is
+                // recursive, so by the time this succeeds we know that all of these records
+                // have no remaining children.
+                let (overrideSQL, overrideArgs) = self.getSQLToOverrideNonFolders(childGUIDs, atModifiedTime: Date.now())
+
+                return self.deleteChildrenOfGUIDs(childGUIDs)
+                    >>> { self.db.run(overrideSQL.map { ($0, overrideArgs) }) }
+                    >>> {
+                        // Delete the children themselves.
+
+                        // Remove each child from structure. We use the top list to save effort.
+                        let deleteStructure =
+                        "DELETE FROM \(TableBookmarksLocalStructure) WHERE parent IN \(topVarlist)"
+
+                        // If a bookmark is New, delete it outright.
+                        let deleteNew =
+                        "DELETE FROM \(TableBookmarksLocal) WHERE guid IN \(childVarlist) AND sync_status = \(SyncStatus.new.rawValue)"
+
+                        // If a bookmark is Changed, mark it as deleted and bump its modified time.
+                        let markChanged = self.getMarkDeletedSQLWithWhereFragment("guid IN \(childVarlist)")
+
+                        return self.db.run([
+                            (deleteStructure, topArgs),
+                            (deleteNew, childArgs),
+                            (markChanged, childArgs),
+                        ])
+                }
+        }
+    }
+
+    fileprivate func getMarkDeletedSQLWithWhereFragment(_ whereFragment: String) -> String {
+        let sql =
+        "UPDATE \(TableBookmarksLocal) SET" +
+        "  date_added = NULL" +
+        ", is_deleted = 1" +
+        ", local_modified = \(Date.now())" +
+        ", bmkUri = NULL" +
+        ", feedUri = NULL" +
+        ", siteUri = NULL" +
+        ", pos = NULL" +
+        ", title = NULL" +
+        ", tags = NULL" +
+        ", keyword = NULL" +
+        ", description = NULL" +
+        ", parentid = NULL" +
+        ", parentName = NULL" +
+        ", folderName = NULL" +
+        ", queryId = NULL" +
+        " WHERE \(whereFragment) AND sync_status = \(SyncStatus.changed.rawValue)"
+
+        return sql
+    }
+    /**
+     * This depends on the record's parent already being overridden if necessary.
+     */
+    fileprivate func removeLocalByGUID(_ guid: GUID) -> Success {
+        let args: Args = [guid]
+
+        // Find the index we're currently occupying.
+        let previousIndexSubquery = "SELECT idx FROM \(TableBookmarksLocalStructure) WHERE child = ?"
+
+        // Fix up the indices of subsequent siblings.
+        let updateIndices =
+        "UPDATE \(TableBookmarksLocalStructure) SET idx = (idx - 1) WHERE idx > (\(previousIndexSubquery))"
+
+        // If the bookmark is New, delete it outright.
+        let deleteNew =
+        "DELETE FROM \(TableBookmarksLocal) WHERE guid = ? AND sync_status = \(SyncStatus.new.rawValue)"
+
+        // If the bookmark is Changed, mark it as deleted and bump its modified time.
+        let markChanged = self.getMarkDeletedSQLWithWhereFragment("guid = ?")
+
+        // Its parent must be either New or Changed, so we don't need to re-mirror it.
+        // TODO: bump the parent's modified time, because the child list changed?
+
+        // Now delete from structure.
+        let deleteStructure =
+        "DELETE FROM \(TableBookmarksLocalStructure) WHERE child = ?"
+
+        return self.db.run([
+            (updateIndices, args),
+            (deleteNew, args),
+            (markChanged, args),
+            (deleteStructure, args),
+        ])
+    }
+
+    fileprivate func markBufferBookmarkAsDeleted(_ guid: GUID) -> Success {
+        let insertInPendingDeletions =
+        "INSERT OR IGNORE INTO \(TablePendingBookmarksDeletions) " +
+        "(id) " +
+        "VALUES (?)"
+        let args: Args = [guid]
+        return self.db.run(insertInPendingDeletions, withArgs: args)
+    }
+}
+
+class SQLiteBookmarkFolder: BookmarkFolder {
+    fileprivate let cursor: Cursor<BookmarkNode>
+    override var count: Int {
+        return cursor.count
+    }
+
+    override subscript(index: Int) -> BookmarkNode {
+        let bookmark = cursor[index]
+        return bookmark! as BookmarkNode
+    }
+
+    init(guid: String, title: String, children: Cursor<BookmarkNode>) {
+        self.cursor = children
+        super.init(guid: guid, title: title)
+    }
+
+    override func removeItemWithGUID(_ guid: GUID) -> BookmarkFolder? {
+        let without = cursor.asArray().filter { $0.guid != guid }
+        return MemoryBookmarkFolder(guid: self.guid, title: self.title, children: without)
+    }
+}
+
+class BookmarkFactory {
+    fileprivate class func addIcon(_ bookmark: BookmarkNode, row: SDRow) {
+        // TODO: share this logic with SQLiteHistory.
+        if let faviconURL = row["iconURL"] as? String,
+           let date = row["iconDate"] as? Double,
+           let faviconType = row["iconType"] as? Int,
+           let type = IconType(rawValue: faviconType) {
+                bookmark.favicon = Favicon(url: faviconURL,
+                                           date: Date(timeIntervalSince1970: date),
+                                           type: type)
+        }
+    }
+
+    fileprivate class func livemarkFactory(_ row: SDRow) -> BookmarkItem {
+        let id = row["id"] as? Int
+        let guid = row["guid"] as! String
+        let url = row["siteUri"] as! String
+        let title = row["title"] as? String ?? "Livemark"       // TODO
+        let isEditable = row.getBoolean("isEditable")           // Defaults to false.
+        let bookmark = BookmarkItem(guid: guid, title: title, url: url, isEditable: isEditable)
+        bookmark.id = id
+        BookmarkFactory.addIcon(bookmark, row: row)
+        return bookmark
+    }
+
+    // We ignore queries altogether inside the model factory.
+    fileprivate class func queryFactory(_ row: SDRow) -> BookmarkItem {
+        log.warning("Creating a BookmarkItem from a query. This is almost certainly unexpected.")
+        let id = row["id"] as? Int
+        let guid = row["guid"] as! String
+        let title = row["title"] as? String ?? SQLiteBookmarks.defaultItemTitle
+        let isEditable = row.getBoolean("isEditable")           // Defaults to false.
+        let bookmark = BookmarkItem(guid: guid, title: title, url: "about:blank", isEditable: isEditable)
+        bookmark.id = id
+        BookmarkFactory.addIcon(bookmark, row: row)
+        return bookmark
+    }
+
+    fileprivate class func separatorFactory(_ row: SDRow) -> BookmarkSeparator {
+        let id = row["id"] as? Int
+        let guid = row["guid"] as! String
+        let separator = BookmarkSeparator(guid: guid)
+        separator.id = id
+        return separator
+    }
+
+    fileprivate class func itemRowFactory(_ row: SDRow, forceEditable: Bool = false) -> BookmarkItem {
+        let id = row["id"] as? Int
+        let guid = row["guid"] as! String
+        let url = row["bmkUri"] as! String
+        let title = row["title"] as? String ?? url
+        let isEditable = forceEditable || row.getBoolean("isEditable")           // Defaults to false.
+        let bookmark = BookmarkItem(guid: guid, title: title, url: url, isEditable: isEditable)
+        bookmark.id = id
+        BookmarkFactory.addIcon(bookmark, row: row)
+        return bookmark
+    }
+
+    fileprivate class func itemFactory(_ row: SDRow) -> BookmarkItem {
+        return BookmarkFactory.itemRowFactory(row, forceEditable: false)
+    }
+
+    fileprivate class func folderFactory(_ row: SDRow) -> BookmarkFolder {
+        let id = row["id"] as? Int
+        let guid = row["guid"] as! String
+        let isEditable = row.getBoolean("isEditable")           // Defaults to false.
+        let title = titleForSpecialGUID(guid) ??
+                    row["title"] as? String ??
+                    SQLiteBookmarks.defaultFolderTitle
+
+        let folder = BookmarkFolder(guid: guid, title: title, isEditable: isEditable)
+        folder.id = id
+        BookmarkFactory.addIcon(folder, row: row)
+        return folder
+    }
+
+    class func factory(_ row: SDRow) -> BookmarkNode {
+        return BookmarkFactory.rowFactory(row, forceEditable: false)
+    }
+
+    class func editableItemsFactory(_ row: SDRow) -> BookmarkNode {
+        return BookmarkFactory.rowFactory(row, forceEditable: true)
+    }
+
+    class func rowFactory(_ row: SDRow, forceEditable: Bool = false) -> BookmarkNode {
+        if let typeCode = row["type"] as? Int, let type = BookmarkNodeType(rawValue: typeCode) {
+            switch type {
+            case .bookmark:
+                return itemRowFactory(row, forceEditable: forceEditable)
+            case .dynamicContainer:
+                // This should never be hit: we exclude dynamic containers from our models.
+                fallthrough
+            case .folder:
+                return folderFactory(row)
+            case .separator:
+                return separatorFactory(row)
+            case .livemark:
+                return livemarkFactory(row)
+            case .query:
+                // This should never be hit: we exclude queries from our models.
+                return queryFactory(row)
+            }
+        }
+        assert(false, "Invalid bookmark data.")
+        return itemFactory(row)     // This will fail, but it keeps the compiler happy.
+    }
+
+    // N.B., doesn't include children!
+    class func mirrorItemFactory(_ row: SDRow) -> BookmarkMirrorItem {
+        // TODO
+        // let id = row["id"] as! Int
+
+        let guid = row["guid"] as! GUID
+        let typeCode = row["type"] as! Int
+        let is_deleted = row.getBoolean("is_deleted")
+        let parentid = row["parentid"] as? GUID
+        let parentName = row["parentName"] as? String
+        let feedUri = row["feedUri"] as? String
+        let siteUri = row["siteUri"] as? String
+        let pos = row["pos"] as? Int
+        let title = row["title"] as? String
+        let description = row["description"] as? String
+        let bmkUri = row["bmkUri"] as? String
+        let tags = row["tags"] as? String
+        let keyword = row["keyword"] as? String
+        let folderName = row["folderName"] as? String
+        let queryId = row["queryId"] as? String
+        let date_added = row.getTimestamp("date_added")
+
+        // Local and mirror only.
+        let faviconID = row["faviconID"] as? Int
+
+        // Local only.
+        let local_modified = row.getTimestamp("local_modified")
+
+        // Mirror and buffer.
+        let server_modified = row.getTimestamp("server_modified")
+        let hasDupe = row.getBoolean("hasDupe")
+
+        // Mirror only. TODO
+        //let is_overridden = row.getBoolean("is_overridden")
+
+        // Use the struct initializer directly. Yes, this doesn't validate as strongly as
+        // using the static constructors, but it'll be as valid as the contents of the DB.
+        let type = BookmarkNodeType(rawValue: typeCode)!
+
+        // This one might really be missing (it's local-only), so do this the hard way.
+        let syncStatus: SyncStatus?
+        if let s = row["sync_status"] as? Int {
+            syncStatus = SyncStatus(rawValue: s)
+        } else {
+            syncStatus = nil
+        }
+        let item = BookmarkMirrorItem(guid: guid, type: type, dateAdded: date_added, serverModified: server_modified ?? 0,
+                                      isDeleted: is_deleted, hasDupe: hasDupe, parentID: parentid, parentName: parentName,
+                                      feedURI: feedUri, siteURI: siteUri,
+                                      pos: pos,
+                                      title: title, description: description,
+                                      bookmarkURI: bmkUri, tags: tags, keyword: keyword,
+                                      folderName: folderName, queryID: queryId,
+                                      children: nil,
+                                      faviconID: faviconID, localModified: local_modified,
+                                      syncStatus: syncStatus)
+        return item
+    }
+}
+
+extension SQLiteBookmarks: SearchableBookmarks {
+    public func bookmarksByURL(_ url: URL) -> Deferred<Maybe<Cursor<BookmarkItem>>> {
+        let inner =
+        "SELECT id, type, date_added, guid, bmkUri, title, faviconID FROM \(TableBookmarksLocal) " +
+        "WHERE " +
+        "type = \(BookmarkNodeType.bookmark.rawValue) AND is_deleted IS NOT 1 AND bmkUri = ? " +
+        "UNION ALL " +
+        "SELECT id, type, date_added, guid, bmkUri, title, faviconID FROM \(TableBookmarksMirror) " +
+        "WHERE " +
+        "type = \(BookmarkNodeType.bookmark.rawValue) AND is_overridden IS NOT 1 AND is_deleted IS NOT 1 AND bmkUri = ? "
+
+        let sql =
+        "SELECT bookmarks.id AS id, bookmarks.type AS type, bookmarks.date_added AS date_added, guid, bookmarks.bmkUri AS bmkUri, title, " +
+        "favicons.url AS iconURL, favicons.date AS iconDate, favicons.type AS iconType " +
+        "FROM (\(inner)) AS bookmarks " +
+        "LEFT OUTER JOIN favicons ON bookmarks.faviconID = favicons.id"
+
+        let u = url.absoluteString
+        let args: Args = [u, u]
+        return db.runQuery(sql, args: args, factory: BookmarkFactory.itemFactory)
+    }
+}
+
+extension SQLiteBookmarks {
+    // We're in sync (even partially) if the mirror is non-empty.
+    // We can show fallback desktop bookmarks if the mirror is empty and the buffer contains
+    // children of the roots.
+    func hasOnlyUnmergedRemoteBookmarks() -> Deferred<Maybe<Bool>> {
+        let parents: Args = [
+            BookmarkRoots.MenuFolderGUID,
+            BookmarkRoots.ToolbarFolderGUID,
+            BookmarkRoots.UnfiledFolderGUID,
+            BookmarkRoots.MobileFolderGUID,
+        ]
+        let sql = [
+            "SELECT",
+            "not exists(SELECT 1 FROM \(TableBookmarksMirror))",
+            "AND",
+            "exists(SELECT 1 FROM \(TableBookmarksBufferStructure) WHERE parent IN (?, ?, ?, ?))",
+            ].joined(separator: " ")
+        return self.db.runQuery(sql, args: parents, factory: { $0[0] as! Int == 1 })
+            >>== { row in
+                guard row.status == .success,
+                    let result = row[0] else {
+                        // if the query did not succeed, we should return false so that we can use local bookmarks
+                        return deferMaybe(false)
+                }
+                return deferMaybe(result)
+        }
+    }
+}
+
+// It's a factory where the root contains Desktop Bookmarks from the buffer, and
+// mobile bookmarks from local.
+open class UnsyncedBookmarksFallbackModelFactory: BookmarksModelFactory {
+    let localFactory: SQLiteBookmarksModelFactory
+    let bufferFactory: SQLiteBookmarksModelFactory
+
+    init(bookmarks: SQLiteBookmarks) {
+        // This relies on SQLiteBookmarks being the storage for both directions.
+        self.localFactory = SQLiteBookmarksModelFactory(bookmarks: bookmarks, direction: .local)
+        if AppConstants.MOZ_SIMPLE_BOOKMARKS_SYNCING {
+            self.bufferFactory = EditableBufferBookmarksSQLiteBookmarksModelFactory(bookmarks: bookmarks, direction: .buffer)
+        } else {
+            self.bufferFactory = SQLiteBookmarksModelFactory(bookmarks: bookmarks, direction: .buffer)
+        }
+    }
+
+    // This is a special-case class, so here's the special-case behavior to
+    // know how to handle a folder that contains items drawn from different
+    // parts of the database. We look for the special kinds of folders we
+    // nest at the top level, and then we pick a folder to match.
+    public func factoryForIndex(_ index: Int, inFolder folder: BookmarkFolder) -> BookmarksModelFactory {
+        let concatenated: ConcatenatedBookmarkFolder
+        let i: Int
+
+        // We have either just remote and local mobile bookmarks, or we have Desktop Bookmarks
+        // followed by remote and local mobile bookmarks. Handle either.
+        if let prepended = folder as? PrependedBookmarkFolder {
+            if index == 0 {
+                return self
+            }
+
+            guard let c = prepended.main as? ConcatenatedBookmarkFolder else {
+                return self
+            }
+            i = index - 1        // Drop the prepend.
+            concatenated = c
+        } else {
+            guard let c = folder as? ConcatenatedBookmarkFolder else {
+                return self
+            }
+            i = index
+            concatenated = c
+        }
+
+        if i < concatenated.pivot {
+            return self.bufferFactory   // This comes first in our concatenation.
+        }
+        return self.localFactory
+    }
+
+    open func modelForFolder(_ folder: BookmarkFolder) -> Deferred<Maybe<BookmarksModel>> {
+        return self.modelForFolder(folder.guid, title: folder.title)
+    }
+
+    open func modelForFolder(_ guid: GUID) -> Deferred<Maybe<BookmarksModel>> {
+        return self.modelForFolder(guid, title: "")
+    }
+
+    open func modelForFolder(_ guid: GUID, title: String) -> Deferred<Maybe<BookmarksModel>> {
+        if guid == BookmarkRoots.MobileFolderGUID {
+            return self.modelForRoot()
+        }
+
+        if guid == BookmarkRoots.FakeDesktopFolderGUID {
+            return self.bufferFactory.modelForFolder(guid, title: title)
+        }
+
+        return self.localFactory.modelForFolder(guid, title: title)
+    }
+
+    open func modelForRoot() -> Deferred<Maybe<BookmarksModel>> {
+        log.debug("Getting model for fallback root.")
+        // Return a virtual model containing "Desktop bookmarks" prepended to the local mobile bookmarks.
+        return self.localFactory.folderForGUID(BookmarkRoots.MobileFolderGUID, title: BookmarksFolderTitleMobile)
+            >>== {
+                localMobileFolder in
+                
+                self.bufferFactory.folderForGUID(BookmarkRoots.MobileFolderGUID, title: BookmarksFolderTitleMobile) >>== {
+                    bufferMobileFolder in
+
+                    let bufferAndLocalMobile = ConcatenatedBookmarkFolder(main: bufferMobileFolder, append: localMobileFolder)
+                    return self.bufferFactory.hasDesktopBookmarks() >>== { yes in
+                        guard yes else {
+                            return deferMaybe(BookmarksModel(modelFactory: self, root: bufferAndLocalMobile))
+                        }
+                        return self.bufferFactory.getDesktopRoots() >>== { cursor in
+                            let desktop = self.bufferFactory.folderForDesktopBookmarksCursor(cursor)
+                            let withDesktopPrepended = PrependedBookmarkFolder(main: bufferAndLocalMobile, prepend: desktop)
+                            return deferMaybe(BookmarksModel(modelFactory: self, root: withDesktopPrepended))
+                        }
+                    }
+                }
+        }
+    }
+
+    // Whenever async construction is necessary, we fall into a pattern of needing
+    // a placeholder that behaves correctly for the period between kickoff and set.
+    open var nullModel: BookmarksModel {
+        let children = Cursor<BookmarkNode>(status: .failure, msg: "Null model")
+        let folder = SQLiteBookmarkFolder(guid: "Null", title: "Null", children: children)
+        return BookmarksModel(modelFactory: self, root: folder)
+    }
+
+    open func isBookmarked(_ url: String) -> Deferred<Maybe<Bool>> {
+        // We don't include buffer items in this check, because we can't un-star them!
+        return self.localFactory.isBookmarked(url)
+    }
+
+    open func removeByGUID(_ guid: GUID) -> Success {
+        return self.localFactory.removeByGUID(guid)
+    }
+
+    open func removeByURL(_ url: String) -> Success {
+        return self.localFactory.removeByURL(url)
+    }
+}
+
+open class MergedSQLiteBookmarks: BookmarksModelFactorySource, KeywordSearchSource {
+    let local: SQLiteBookmarks
+    let buffer: SQLiteBookmarkBufferStorage
+
+    // Figuring out our factory can require hitting the DB, so this is async.
+    // Note that we check *every time* -- we don't want to get stuck in a dead
+    // end when you might sync soon.
+    open var modelFactory: Deferred<Maybe<BookmarksModelFactory>> {
+        return self.local.hasOnlyUnmergedRemoteBookmarks() >>== { yes in
+            if yes {
+                log.debug("Only unmerged remote bookmarks; using fallback factory.")
+                return deferMaybe(UnsyncedBookmarksFallbackModelFactory(bookmarks: self.local))
+            }
+            log.debug("Using local+mirror bookmark factory.")
+            return self.local.modelFactory
+        }
+    }
+
+    public init(db: BrowserDB) {
+        self.local = SQLiteBookmarks(db: db)
+        self.buffer = SQLiteBookmarkBufferStorage(db: db)
+    }
+
+    open func getURLForKeywordSearch(_ keyword: String) -> Deferred<Maybe<String>> {
+        return self.local.getURLForKeywordSearch(keyword)
+    }
+}

mobile/ios/Storage/SQL/SQLiteBookmarksResetting.swift

@@ -0,0 +1,146 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+
+private let log = Logger.syncLogger
+
+extension MergedSQLiteBookmarks: AccountRemovalDelegate {
+    public func onRemovedAccount() -> Success {
+        return self.local.onRemovedAccount() >>> self.buffer.onRemovedAccount
+    }
+}
+
+extension MergedSQLiteBookmarks: ResettableSyncStorage {
+    public func resetClient() -> Success {
+        return self.local.resetClient() >>> self.buffer.resetClient
+    }
+}
+
+extension SQLiteBookmarkBufferStorage: AccountRemovalDelegate {
+    public func onRemovedAccount() -> Success {
+        return self.resetClient()
+    }
+}
+
+extension SQLiteBookmarkBufferStorage: ResettableSyncStorage {
+    /**
+     * Our buffer is simply a copy of server contents. That means we should
+     * be very willing to drop it and re-populate it from the server whenever we might
+     * be out of sync. See Bug 1212431 Comment 2.
+     */
+    public func resetClient() -> Success {
+        return self.wipeBookmarks()
+    }
+
+    public func wipeBookmarks() -> Success {
+        return self.db.run([
+            "DELETE FROM \(TableBookmarksBufferStructure)",
+            "DELETE FROM \(TableBookmarksBuffer)",
+        ])
+    }
+}
+
+extension SQLiteBookmarks {
+    /**
+     * If a synced record is deleted locally, but hasn't been synced to the server,
+     * then `preserveDeletions=true` will result in that deletion being kept.
+     *
+     * During a reset, we'll redownload all server records. If we don't keep the
+     * local deletion, then when we re-process the (non-deleted) server counterpart
+     * to the now-missing local record, it'll be reinserted: the user's deletion will
+     * be undone.
+     *
+     * Right now we don't preserve deletions when removing the Firefox Account, but
+     * we could do so if we were willing to trade local database space to handle this
+     * possible situation.
+     */
+    fileprivate func collapseMirrorIntoLocalPreservingDeletions(_ preserveDeletions: Bool) -> Success {
+        // As implemented, this won't work correctly without ON DELETE CASCADE.
+        assert(SwiftData.EnableForeignKeys)
+
+        // 1. Wait until we commit to complain about constraint violations.
+        let deferForeignKeys =
+        "PRAGMA defer_foreign_keys = ON"
+
+        // 2. Drop anything from local that's deleted. We don't need to track the
+        //    deletion now. Optional: keep them around if they're non-uploaded changes.
+        let removeLocalDeletions =
+        "DELETE FROM \(TableBookmarksLocal) WHERE is_deleted IS 1 " +
+            (preserveDeletions ? "AND sync_status IS NOT \(SyncStatus.changed.rawValue)" : "")
+
+        // 3. Mark everything in local as New.
+        let markLocalAsNew =
+        "UPDATE \(TableBookmarksLocal) SET sync_status = \(SyncStatus.new.rawValue)"
+
+        // 4. Insert into local anything not overridden left in mirror.
+        //    Note that we use the server modified time as our substitute local modified time.
+        //    This will provide an ounce of conflict avoidance if the user re-links the same
+        //    account at a later date.
+        let copyMirrorContents =
+        "INSERT OR IGNORE INTO \(TableBookmarksLocal) " +
+        "(sync_status, local_modified, " +
+        " guid, date_added, type, bmkUri, title, parentid, parentName, feedUri, siteUri, pos," +
+        " description, tags, keyword, folderName, queryId, faviconID) " +
+        "SELECT " +
+        "\(SyncStatus.new.rawValue) AS sync_status, " +
+        "server_modified AS local_modified, " +
+        "guid, date_added, type, bmkUri, title, parentid, parentName, " +
+        "feedUri, siteUri, pos, description, tags, keyword, folderName, queryId, faviconID " +
+        "FROM \(TableBookmarksMirror) WHERE is_overridden IS 0"
+
+        // 5.(pre) I have a database right in front of me that violates an assumption: a full
+        // bookmarksMirrorStructure and an empty bookmarksMirror. Clean up, just in case.
+        let removeOverriddenStructure =
+        "DELETE FROM \(TableBookmarksMirrorStructure) WHERE parent IN (SELECT guid FROM \(TableBookmarksMirror) WHERE is_overridden IS 1)"
+
+        // 5. Insert into localStructure anything left in mirrorStructure.
+        //    This won't copy the structure of any folders that were already overridden --
+        //    we already deleted those, and the deletions cascaded.
+        let copyMirrorStructure =
+        "INSERT INTO \(TableBookmarksLocalStructure) SELECT * FROM \(TableBookmarksMirrorStructure)"
+
+        // 6. Blank the mirror.
+        let removeMirrorStructure =
+        "DELETE FROM \(TableBookmarksMirrorStructure)"
+
+        let removeMirrorContents =
+        "DELETE FROM \(TableBookmarksMirror)"
+
+        return db.run([
+            deferForeignKeys,
+            removeLocalDeletions,
+            markLocalAsNew,
+            copyMirrorContents,
+            removeOverriddenStructure,
+            copyMirrorStructure,
+            removeMirrorStructure,
+            removeMirrorContents,
+        ])
+    }
+}
+extension SQLiteBookmarks: AccountRemovalDelegate {
+    public func onRemovedAccount() -> Success {
+        return self.collapseMirrorIntoLocalPreservingDeletions(false)
+    }
+}
+
+extension SQLiteBookmarks: ResettableSyncStorage {
+    public func resetClient() -> Success {
+        // Flip flags to prompt a re-sync.
+        //
+        // We copy the mirror to local, preserving local changes, apart from
+        // deletions of records that were never synced.
+        //
+        // Records that match the server record that we'll redownload will be
+        // marked as Synced and won't be reuploaded.
+        //
+        // Records that are present locally but aren't on the server will be
+        // uploaded.
+        //
+        return self.collapseMirrorIntoLocalPreservingDeletions(true)
+    }
+}

mobile/ios/Storage/SQL/SQLiteFavicons.swift

@@ -0,0 +1,87 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Deferred
+import Shared
+
+open class SQLiteFavicons {
+    let db: BrowserDB
+    
+    required public init(db: BrowserDB) {
+        self.db = db
+    }
+    
+    public func getFaviconIDQuery(url: String) -> (sql: String, args: Args?) {
+        var args: Args = []
+        args.append(url)
+        return (sql: "SELECT id FROM \(TableFavicons) WHERE url = ? LIMIT 1", args: args)
+    }
+    
+    public func getInsertFaviconQuery(favicon: Favicon) -> (sql: String, args: Args?) {
+        var args: Args = []
+        args.append(favicon.url)
+        args.append(favicon.width)
+        args.append(favicon.height)
+        args.append(favicon.date)
+        args.append(favicon.type.rawValue)
+        return (sql: "INSERT INTO \(TableFavicons) (url, width, height, date, type) VALUES (?,?,?,?,?)", args: args)
+    }
+    
+    public func getUpdateFaviconQuery(favicon: Favicon) -> (sql: String, args: Args?) {
+        var args = Args()
+        args.append(favicon.width)
+        args.append(favicon.height)
+        args.append(favicon.date)
+        args.append(favicon.type.rawValue)
+        args.append(favicon.url)
+        return (sql: "UPDATE \(TableFavicons) SET width = ?, height = ?, date = ?, type = ? WHERE url = ?", args: args)
+    }
+    
+    public func getCleanupFaviconsQuery() -> (sql: String, args: Args?) {
+        return (sql: "DELETE FROM \(TableFavicons) " +
+            "WHERE \(TableFavicons).id NOT IN (" +
+            "SELECT faviconID FROM \(TableFaviconSites) " +
+            "UNION ALL " +
+            "SELECT faviconID FROM \(TableBookmarksLocal) WHERE faviconID IS NOT NULL " +
+            "UNION ALL " +
+            "SELECT faviconID FROM \(TableBookmarksMirror) WHERE faviconID IS NOT NULL" +
+            ")", args: nil)
+    }
+    
+    public func insertOrUpdateFavicon(_ favicon: Favicon) -> Deferred<Maybe<Int>> {
+        return db.withConnection { conn -> Int in
+            self.insertOrUpdateFaviconInTransaction(favicon, conn: conn) ?? 0
+        }
+    }
+
+    func insertOrUpdateFaviconInTransaction(_ favicon: Favicon, conn: SQLiteDBConnection) -> Int? {
+        let query = self.getFaviconIDQuery(url: favicon.url)
+        let cursor = conn.executeQuery(query.sql, factory: IntFactory, withArgs: query.args)
+        
+        if let id = cursor[0] {
+            let updateQuery = self.getUpdateFaviconQuery(favicon: favicon)
+            do {
+                try conn.executeChange(updateQuery.sql, withArgs: updateQuery.args)
+            } catch {
+                return nil
+            }
+            
+            return id
+        }
+        
+        let insertQuery = self.getInsertFaviconQuery(favicon: favicon)
+        do {
+            try conn.executeChange(insertQuery.sql, withArgs: insertQuery.args)
+        } catch {
+            return nil
+        }
+        
+        return conn.lastInsertedRowID
+    }
+    
+    public func cleanupFavicons() -> Success {
+        return self.db.run([getCleanupFaviconsQuery()])
+    }
+}

mobile/ios/Storage/SQL/SQLiteHistoryFactories.swift

@@ -0,0 +1,75 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+
+/*
+ * Factory methods for converting rows from SQLite into model objects
+ */
+extension SQLiteHistory {
+    class func basicHistoryColumnFactory(_ row: SDRow) -> Site {
+        let id = row["historyID"] as? Int
+        let url = row["url"] as! String
+        let title = row["title"] as! String
+        let guid = row["guid"] as! String
+
+        // Extract a boolean from the row if it's present.
+        let iB = row["is_bookmarked"] as? Int
+        let isBookmarked: Bool? = (iB == nil) ? nil : (iB! != 0)
+
+        let site = Site(url: url, title: title, bookmarked: isBookmarked)
+        site.guid = guid
+        site.id = id
+
+        // Find the most recent visit, regardless of which column it might be in.
+        let local = row.getTimestamp("localVisitDate") ?? 0
+        let remote = row.getTimestamp("remoteVisitDate") ?? 0
+        let either = row.getTimestamp("visitDate") ?? 0
+
+        let latest = max(local, remote, either)
+        if latest > 0 {
+            site.latestVisit = Visit(date: latest, type: VisitType.unknown)
+        }
+
+        return site
+    }
+
+    class func iconColumnFactory(_ row: SDRow) -> Favicon? {
+        if let iconType = row["iconType"] as? Int,
+            let iconURL = row["iconURL"] as? String,
+            let iconDate = row["iconDate"] as? Double,
+            let _ = row["iconID"] as? Int {
+                let date = Date(timeIntervalSince1970: iconDate)
+                return Favicon(url: iconURL, date: date, type: IconType(rawValue: iconType)!)
+        }
+        return nil
+    }
+
+    class func pageMetadataColumnFactory(_ row: SDRow) -> PageMetadata? {
+        guard let siteURL = row["url"] as? String else {
+            return nil
+        }
+
+        return PageMetadata(id: row["metadata_id"] as? Int, siteURL: siteURL, mediaURL: row["media_url"] as? String, title: row["metadata_title"] as? String, description: row["description"] as? String, type: row["type"] as? String, providerName: row["provider_name"] as? String, mediaDataURI: nil)
+    }
+
+    class func iconHistoryColumnFactory(_ row: SDRow) -> Site {
+        let site = basicHistoryColumnFactory(row)
+        site.icon = iconColumnFactory(row)
+        return site
+    }
+
+    class func iconHistoryMetadataColumnFactory(_ row: SDRow) -> Site {
+        let site = iconHistoryColumnFactory(row)
+        site.metadata = pageMetadataColumnFactory(row)
+        return site
+    }
+
+    class func basicHistoryMetadataColumnFactory(_ row: SDRow) -> Site {
+        let site = basicHistoryColumnFactory(row)
+        site.metadata = pageMetadataColumnFactory(row)
+        return site
+    }
+}

mobile/ios/Storage/SQL/SQLiteHistoryRecommendations.swift

@@ -0,0 +1,179 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+import Deferred
+
+fileprivate let log = Logger.syncLogger
+
+extension SQLiteHistory: HistoryRecommendations {
+    // Bookmarks Query
+    static let removeMultipleDomainsSubquery =
+        " INNER JOIN (SELECT \(ViewHistoryVisits).domain_id AS domain_id" +
+        " FROM \(ViewHistoryVisits)" +
+        " GROUP BY \(ViewHistoryVisits).domain_id) AS domains ON domains.domain_id = \(TableHistory).domain_id"
+
+    static let urisForSimpleSyncedBookmarks =
+        "SELECT bmkUri FROM \(TableBookmarksBuffer) WHERE server_modified > ? AND is_deleted = 0 " +
+        "UNION ALL " +
+        "SELECT bmkUri FROM \(TableBookmarksLocal) WHERE local_modified > ? AND is_deleted = 0"
+
+    static let urisForLocalBookmarks =
+        "SELECT bmkUri" +
+        "FROM \(ViewBookmarksLocalOnMirror) " +
+        "WHERE \(ViewBookmarksLocalOnMirror).server_modified > ? OR " +
+            "\(ViewBookmarksLocalOnMirror).local_modified > ?"
+
+    static let bookmarkHighlights =
+        "SELECT historyID, url, siteTitle, guid, is_bookmarked FROM (" +
+            "   SELECT \(TableHistory).id AS historyID, \(TableHistory).url AS url, \(TableHistory).title AS siteTitle, guid, \(TableHistory).domain_id, NULL AS visitDate, 1 AS is_bookmarked" +
+            "   FROM (" +
+                    (AppConstants.MOZ_SIMPLE_BOOKMARKS_SYNCING ? urisForSimpleSyncedBookmarks : urisForLocalBookmarks) +
+            "   )" +
+            "   LEFT JOIN \(TableHistory) ON \(TableHistory).url = bmkUri" + removeMultipleDomainsSubquery +
+            "   WHERE \(TableHistory).title NOT NULL and \(TableHistory).title != '' AND url NOT IN" +
+            "       (SELECT \(TableActivityStreamBlocklist).url FROM \(TableActivityStreamBlocklist))" +
+            "   LIMIT ?" +
+        ")"
+
+    static let bookmarksQuery =
+        "SELECT historyID, url, siteTitle AS title, guid, is_bookmarked, iconID, iconURL, iconType, iconDate, iconWidth, \(TablePageMetadata).title AS metadata_title, media_url, type, description, provider_name " +
+            "FROM (\(bookmarkHighlights) ) " +
+            "LEFT JOIN \(ViewHistoryIDsWithWidestFavicons) ON \(ViewHistoryIDsWithWidestFavicons).id = historyID " +
+            "LEFT OUTER JOIN \(TablePageMetadata) ON \(TablePageMetadata).cache_key = url " +
+    "GROUP BY url"
+
+    // Highlights Query
+    static let highlightsLimit = 8
+    static let blacklistedHosts: Args = [
+        "google.com",
+        "google.ca",
+        "calendar.google.com",
+        "mail.google.com",
+        "mail.yahoo.com",
+        "search.yahoo.com",
+        "localhost",
+        "t.co"
+    ]
+
+    static let blacklistSubquery = "SELECT \(TableDomains).id FROM \(TableDomains) WHERE \(TableDomains).domain IN " + BrowserDB.varlist(blacklistedHosts.count)
+    static let removeMultipleDomainsSubqueryFromHighlights =
+        "   INNER JOIN (SELECT \(ViewHistoryVisits).domain_id AS domain_id, MAX(\(ViewHistoryVisits).visitDate) AS visit_date" +
+            "   FROM \(ViewHistoryVisits)" +
+    "   GROUP BY \(ViewHistoryVisits).domain_id) AS domains ON domains.domain_id = \(TableHistory).domain_id AND visitDate = domains.visit_date"
+
+    static let nonRecentHistory =
+        "SELECT historyID, url, siteTitle, guid, visitCount, visitDate, is_bookmarked, visitCount * icon_url_score * media_url_score AS score FROM (" +
+            "   SELECT \(TableHistory).id as historyID, url, \(TableHistory).title AS siteTitle, guid, visitDate, \(TableHistory).domain_id," +
+            "       (SELECT COUNT(1) FROM \(TableVisits) WHERE s = \(TableVisits).siteID) AS visitCount," +
+            "       (SELECT COUNT(1) FROM \(ViewBookmarksLocalOnMirror) WHERE \(ViewBookmarksLocalOnMirror).bmkUri == url) AS is_bookmarked," +
+            "     CASE WHEN iconURL IS NULL THEN 1 ELSE 2 END AS icon_url_score," +
+            "     CASE WHEN media_url IS NULL THEN 1 ELSE 4 END AS media_url_score" +
+            "   FROM (" +
+            "       SELECT siteID AS s, MAX(date) AS visitDate" +
+            "       FROM \(TableVisits)" +
+            "       WHERE date < ?" +
+            "       GROUP BY siteID" +
+            "       ORDER BY visitDate DESC" +
+            "   )" +
+            "   LEFT JOIN \(TableHistory) ON \(TableHistory).id = s" +
+            removeMultipleDomainsSubqueryFromHighlights +
+            "   LEFT OUTER JOIN \(ViewHistoryIDsWithWidestFavicons) ON" +
+            "       \(ViewHistoryIDsWithWidestFavicons).id = \(TableHistory).id" +
+            "   LEFT OUTER JOIN \(TablePageMetadata) ON" +
+            "       \(TablePageMetadata).site_url = \(TableHistory).url" +
+            "   WHERE visitCount <= 3 AND \(TableHistory).title NOT NULL AND \(TableHistory).title != '' AND is_bookmarked == 0 AND url NOT IN" +
+            "       (SELECT url FROM \(TableActivityStreamBlocklist))" +
+            "        AND \(TableHistory).domain_id NOT IN ("
+            + blacklistSubquery + ")" +
+    ")"
+
+    public func getHighlights() -> Deferred<Maybe<Cursor<Site>>> {
+        let highlightsProjection = [
+            "historyID",
+            "\(TableHighlights).cache_key AS cache_key",
+            "url",
+            "\(TableHighlights).title AS title",
+            "guid",
+            "visitCount",
+            "visitDate",
+            "is_bookmarked"
+        ]
+        let faviconsProjection = ["iconID", "iconURL", "iconType", "iconDate", "iconWidth"]
+        let metadataProjections = [
+            "\(TablePageMetadata).title AS metadata_title",
+            "media_url",
+            "type",
+            "description",
+            "provider_name"
+        ]
+
+        let allProjection = highlightsProjection + faviconsProjection + metadataProjections
+
+        let highlightsHistoryIDs =
+        "SELECT historyID FROM \(TableHighlights)"
+
+        // Search the history/favicon view with our limited set of highlight IDs
+        // to avoid doing a full table scan on history
+        let faviconSearch =
+        "SELECT * FROM \(ViewHistoryIDsWithWidestFavicons) WHERE id IN (\(highlightsHistoryIDs))"
+
+        let sql =
+        "SELECT \(allProjection.joined(separator: ",")) " +
+        "FROM \(TableHighlights) " +
+        "LEFT JOIN (\(faviconSearch)) AS f1 ON f1.id = historyID " +
+        "LEFT OUTER JOIN \(TablePageMetadata) ON " +
+        "\(TablePageMetadata).cache_key = \(TableHighlights).cache_key"
+
+        return self.db.runQuery(sql, args: nil, factory: SQLiteHistory.iconHistoryMetadataColumnFactory)
+    }
+
+    public func removeHighlightForURL(_ url: String) -> Success {
+        return self.db.run([("INSERT INTO \(TableActivityStreamBlocklist) (url) VALUES (?)", [url])])
+    }
+
+    private func repopulateHighlightsQuery() -> [(String, Args?)] {
+        let (query, args) = computeHighlightsQuery()
+        let clearHighlightsQuery = "DELETE FROM \(TableHighlights)"
+
+        let sql = "INSERT INTO \(TableHighlights) " +
+            "SELECT historyID, url as cache_key, url, title, guid, visitCount, visitDate, is_bookmarked " +
+            "FROM (\(query))"
+        return [(clearHighlightsQuery, nil), (sql, args)]
+    }
+
+    public func repopulate(invalidateTopSites shouldInvalidateTopSites: Bool, invalidateHighlights shouldInvalidateHighlights: Bool) -> Success {
+        var queries: [(String, Args?)] = []
+        if shouldInvalidateTopSites {
+            queries.append(contentsOf: self.refreshTopSitesQuery())
+        }
+        if shouldInvalidateHighlights {
+            queries.append(contentsOf: self.repopulateHighlightsQuery())
+        }
+        return self.db.run(queries)
+    }
+
+    public func getRecentBookmarks(_ limit: Int = 3) -> Deferred<Maybe<Cursor<Site>>> {
+        let fiveDaysAgo: UInt64 = Date.now() - (OneDayInMilliseconds * 5) // The data is joined with a millisecond not a microsecond one. (History)
+        let args = [fiveDaysAgo, fiveDaysAgo, limit] as Args
+        return self.db.runQuery(SQLiteHistory.bookmarksQuery, args: args, factory: SQLiteHistory.iconHistoryMetadataColumnFactory)
+    }
+
+    private func computeHighlightsQuery() -> (String, Args) {
+        let microsecondsPerMinute: UInt64 = 60_000_000 // 1000 * 1000 * 60
+        let now = Date.nowMicroseconds()
+        let thirtyMinutesAgo: UInt64 = now - 30 * microsecondsPerMinute
+
+        let highlightsQuery =
+            "SELECT historyID, url, siteTitle AS title, guid, visitCount, visitDate, is_bookmarked, score " +
+            "FROM ( \(SQLiteHistory.nonRecentHistory) ) " +
+            "GROUP BY url " +
+            "ORDER BY score DESC " +
+            "LIMIT \(SQLiteHistory.highlightsLimit)"
+        let args: Args = [thirtyMinutesAgo] + SQLiteHistory.blacklistedHosts
+        return (highlightsQuery, args)
+    }
+}

mobile/ios/Storage/SQL/SQLiteLogins.swift

@@ -0,0 +1,894 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+open class SQLiteLogins: BrowserLogins {
+
+    fileprivate let db: BrowserDB
+    fileprivate static let MainColumns: String = "guid, username, password, hostname, httpRealm, formSubmitURL, usernameField, passwordField"
+    fileprivate static let MainWithLastUsedColumns: String = MainColumns + ", timeLastUsed, timesUsed"
+    fileprivate static let LoginColumns: String = MainColumns + ", timeCreated, timeLastUsed, timePasswordChanged, timesUsed"
+
+    public init(db: BrowserDB) {
+        self.db = db
+    }
+
+    fileprivate class func populateLogin(_ login: Login, row: SDRow) {
+        login.formSubmitURL = row["formSubmitURL"] as? String
+        login.usernameField = row["usernameField"] as? String
+        login.passwordField = row["passwordField"] as? String
+        login.guid = row["guid"] as! String
+
+        if let timeCreated = row.getTimestamp("timeCreated"),
+            let timeLastUsed = row.getTimestamp("timeLastUsed"),
+            let timePasswordChanged = row.getTimestamp("timePasswordChanged"),
+            let timesUsed = row["timesUsed"] as? Int {
+                login.timeCreated = timeCreated
+                login.timeLastUsed = timeLastUsed
+                login.timePasswordChanged = timePasswordChanged
+                login.timesUsed = timesUsed
+        }
+    }
+
+    fileprivate class func constructLogin<T: Login>(_ row: SDRow, c: T.Type) -> T {
+        let credential = URLCredential(user: row["username"] as? String ?? "",
+            password: row["password"] as! String,
+            persistence: URLCredential.Persistence.none)
+
+        // There was a bug in previous versions of the app where we saved only the hostname and not the
+        // scheme and port in the DB. To work with these scheme-less hostnames, we try to extract the scheme and
+        // hostname by converting to a URL first. If there is no valid hostname or scheme for the URL,
+        // fallback to returning the raw hostname value from the DB as the host and allow NSURLProtectionSpace
+        // to use the default (http) scheme. See https://bugzilla.mozilla.org/show_bug.cgi?id=1238103.
+
+        let hostnameString = (row["hostname"] as? String) ?? ""
+        let hostnameURL = hostnameString.asURL
+
+        let scheme = hostnameURL?.scheme
+        let port = hostnameURL?.port ?? 0
+
+        // Check for malformed hostname urls in the DB
+        let host: String
+        var malformedHostname = false
+        if let h = hostnameURL?.host {
+            host = h
+        } else {
+            host = hostnameString
+            malformedHostname = true
+        }
+
+        let protectionSpace = URLProtectionSpace(host: host,
+            port: port,
+            protocol: scheme,
+            realm: row["httpRealm"] as? String,
+            authenticationMethod: nil)
+
+        let login = T(credential: credential, protectionSpace: protectionSpace)
+        self.populateLogin(login, row: row)
+        login.hasMalformedHostname = malformedHostname
+        return login
+    }
+
+    class func LocalLoginFactory(_ row: SDRow) -> LocalLogin {
+        let login = self.constructLogin(row, c: LocalLogin.self)
+
+        login.localModified = row.getTimestamp("local_modified") ?? 0
+        login.isDeleted = row.getBoolean("is_deleted")
+        login.syncStatus = SyncStatus(rawValue: row["sync_status"] as! Int)!
+
+        return login
+    }
+
+    class func MirrorLoginFactory(_ row: SDRow) -> MirrorLogin {
+        let login = self.constructLogin(row, c: MirrorLogin.self)
+
+        login.serverModified = row.getTimestamp("server_modified")!
+        login.isOverridden = row.getBoolean("is_overridden")
+
+        return login
+    }
+
+    fileprivate class func LoginFactory(_ row: SDRow) -> Login {
+        return self.constructLogin(row, c: Login.self)
+    }
+
+    fileprivate class func LoginDataFactory(_ row: SDRow) -> LoginData {
+        return LoginFactory(row) as LoginData
+    }
+
+    fileprivate class func LoginUsageDataFactory(_ row: SDRow) -> LoginUsageData {
+        return LoginFactory(row) as LoginUsageData
+    }
+
+    func notifyLoginDidChange() {
+        log.debug("Notifying login did change.")
+
+        // For now we don't care about the contents.
+        // This posts immediately to the shared notification center.
+        NotificationCenter.default.post(name: NotificationDataLoginDidChange, object: nil)
+    }
+
+    open func getUsageDataForLoginByGUID(_ guid: GUID) -> Deferred<Maybe<LoginUsageData>> {
+        let projection = SQLiteLogins.LoginColumns
+        let sql =
+        "SELECT \(projection) FROM " +
+        "\(TableLoginsLocal) WHERE is_deleted = 0 AND guid = ? " +
+        "UNION ALL " +
+        "SELECT \(projection) FROM " +
+        "\(TableLoginsMirror) WHERE is_overridden = 0 AND guid = ? " +
+        "LIMIT 1"
+
+        let args: Args = [guid, guid]
+        return db.runQuery(sql, args: args, factory: SQLiteLogins.LoginUsageDataFactory)
+            >>== { value in
+            deferMaybe(value[0]!)
+        }
+    }
+
+    open func getLoginDataForGUID(_ guid: GUID) -> Deferred<Maybe<Login>> {
+        let projection = SQLiteLogins.LoginColumns
+        let sql =
+        "SELECT \(projection) FROM " +
+            "\(TableLoginsLocal) WHERE is_deleted = 0 AND guid = ? " +
+            "UNION ALL " +
+            "SELECT \(projection) FROM " +
+            "\(TableLoginsMirror) WHERE is_overriden IS NOT 1 AND guid = ? " +
+        "ORDER BY hostname ASC " +
+        "LIMIT 1"
+
+        let args: Args = [guid, guid]
+        return db.runQuery(sql, args: args, factory: SQLiteLogins.LoginFactory)
+            >>== { value in
+            if let login = value[0] {
+                return deferMaybe(login)
+            } else {
+                return deferMaybe(LoginDataError(description: "Login not found for GUID \(guid)"))
+            }
+        }
+    }
+
+    open func getLoginsForProtectionSpace(_ protectionSpace: URLProtectionSpace) -> Deferred<Maybe<Cursor<LoginData>>> {
+        let projection = SQLiteLogins.MainWithLastUsedColumns
+
+        let sql =
+        "SELECT \(projection) FROM " +
+        "\(TableLoginsLocal) WHERE is_deleted = 0 AND hostname IS ? OR hostname IS ?" +
+        "UNION ALL " +
+        "SELECT \(projection) FROM " +
+        "\(TableLoginsMirror) WHERE is_overridden = 0 AND hostname IS ? OR hostname IS ?" +
+        "ORDER BY timeLastUsed DESC"
+
+        // Since we store hostnames as the full scheme/protocol + host, combine the two to look up in our DB.
+        // In the case of https://bugzilla.mozilla.org/show_bug.cgi?id=1238103, there may be hostnames without
+        // a scheme. Check for these as well.
+        let args: Args = [
+            protectionSpace.urlString(),
+            protectionSpace.host,
+            protectionSpace.urlString(),
+            protectionSpace.host,
+        ]
+        if Logger.logPII {
+            log.debug("Looking for login: \(protectionSpace.urlString()) && \(protectionSpace.host)")
+        }
+        return db.runQuery(sql, args: args, factory: SQLiteLogins.LoginDataFactory)
+    }
+
+    // username is really Either<String, NULL>; we explicitly match no username.
+    open func getLoginsForProtectionSpace(_ protectionSpace: URLProtectionSpace, withUsername username: String?) -> Deferred<Maybe<Cursor<LoginData>>> {
+        let projection = SQLiteLogins.MainWithLastUsedColumns
+
+        let args: Args
+        let usernameMatch: String
+        if let username = username {
+            args = [
+                protectionSpace.urlString(), username, protectionSpace.host,
+                protectionSpace.urlString(), username, protectionSpace.host
+            ]
+            usernameMatch = "username = ?"
+        } else {
+            args = [
+                protectionSpace.urlString(), protectionSpace.host,
+                protectionSpace.urlString(), protectionSpace.host
+            ]
+            usernameMatch = "username IS NULL"
+        }
+
+        if Logger.logPII {
+            log.debug("Looking for login with username: \(username ?? "nil"), first arg: \(args[0] ?? "nil")")
+        }
+
+        let sql =
+        "SELECT \(projection) FROM " +
+        "\(TableLoginsLocal) WHERE is_deleted = 0 AND hostname IS ? AND \(usernameMatch) OR hostname IS ?" +
+        "UNION ALL " +
+        "SELECT \(projection) FROM " +
+        "\(TableLoginsMirror) WHERE is_overridden = 0 AND hostname IS ? AND \(usernameMatch) OR hostname IS ?" +
+        "ORDER BY timeLastUsed DESC"
+
+        return db.runQuery(sql, args: args, factory: SQLiteLogins.LoginDataFactory)
+    }
+
+    open func getAllLogins() -> Deferred<Maybe<Cursor<Login>>> {
+        return searchLoginsWithQuery(nil)
+    }
+
+    open func searchLoginsWithQuery(_ query: String?) -> Deferred<Maybe<Cursor<Login>>> {
+        let projection = SQLiteLogins.LoginColumns
+        var searchClauses = [String]()
+        var args: Args? = nil
+        if let query = query, !query.isEmpty {
+            // Add wildcards to change query to 'contains in' and add them to args. We need 6 args because
+            // we include the where clause twice: Once for the local table and another for the remote.
+            args = (0..<6).map { _ in
+                return "%\(query)%" as String?
+            }
+
+            searchClauses.append("username LIKE ? ")
+            searchClauses.append(" password LIKE ? ")
+            searchClauses.append(" hostname LIKE ?")
+        }
+
+        let whereSearchClause = searchClauses.count > 0 ? "AND (" + searchClauses.joined(separator: "OR") + ") " : ""
+        let sql =
+        "SELECT \(projection) FROM " +
+            "\(TableLoginsLocal) WHERE is_deleted = 0 " + whereSearchClause +
+            "UNION ALL " +
+            "SELECT \(projection) FROM " +
+            "\(TableLoginsMirror) WHERE is_overridden = 0 " + whereSearchClause +
+        "ORDER BY hostname ASC"
+
+        return db.runQuery(sql, args: args, factory: SQLiteLogins.LoginFactory)
+    }
+
+    open func addLogin(_ login: LoginData) -> Success {
+        if let error = login.isValid.failureValue {
+            return deferMaybe(error)
+        }
+
+        let nowMicro = Date.nowMicroseconds()
+        let nowMilli = nowMicro / 1000
+        let dateMicro = nowMicro
+        let dateMilli = nowMilli
+
+        let args: Args = [
+            login.hostname,
+            login.httpRealm,
+            login.formSubmitURL,
+            login.usernameField,
+            login.passwordField,
+            login.username,
+            login.password,
+            login.guid,
+            dateMicro,            // timeCreated
+            dateMicro,            // timeLastUsed
+            dateMicro,            // timePasswordChanged
+            dateMilli,            // localModified
+        ]
+
+        let sql =
+        "INSERT OR IGNORE INTO \(TableLoginsLocal) " +
+        // Shared fields.
+        "( hostname" +
+        ", httpRealm" +
+        ", formSubmitURL" +
+        ", usernameField" +
+        ", passwordField" +
+        ", timesUsed" +
+        ", username" +
+        ", password " +
+        // Local metadata.
+        ", guid " +
+        ", timeCreated" +
+        ", timeLastUsed" +
+        ", timePasswordChanged" +
+        ", local_modified " +
+        ", is_deleted " +
+        ", sync_status " +
+        ") " +
+        "VALUES (?,?,?,?,?,1,?,?,?,?,?, " +
+        "?, ?, 0, \(SyncStatus.new.rawValue)" +         // Metadata.
+        ")"
+
+        return db.run(sql, withArgs: args)
+                >>> effect(self.notifyLoginDidChange)
+    }
+
+    fileprivate func cloneMirrorToOverlay(whereClause: String?, args: Args?) -> Deferred<Maybe<Int>> {
+        let shared = "guid, hostname, httpRealm, formSubmitURL, usernameField, passwordField, timeCreated, timeLastUsed, timePasswordChanged, timesUsed, username, password "
+        let local = ", local_modified, is_deleted, sync_status "
+        let sql = "INSERT OR IGNORE INTO \(TableLoginsLocal) (\(shared)\(local)) SELECT \(shared), NULL AS local_modified, 0 AS is_deleted, 0 AS sync_status FROM \(TableLoginsMirror) \(whereClause ?? "")"
+        return self.db.write(sql, withArgs: args)
+    }
+
+    /**
+     * Returns success if either a local row already existed, or
+     * one could be copied from the mirror.
+     */
+    fileprivate func ensureLocalOverlayExistsForGUID(_ guid: GUID) -> Success {
+        let sql = "SELECT guid FROM \(TableLoginsLocal) WHERE guid = ?"
+        let args: Args = [guid]
+        let c = db.runQuery(sql, args: args, factory: { _ in 1 })
+
+        return c >>== { rows in
+            if rows.count > 0 {
+                return succeed()
+            }
+            log.debug("No overlay; cloning one for GUID \(guid).")
+            return self.cloneMirrorToOverlay(guid)
+                >>== { count in
+                    if count > 0 {
+                        return succeed()
+                    }
+                    log.warning("Failed to create local overlay for GUID \(guid).")
+                    return deferMaybe(NoSuchRecordError(guid: guid))
+            }
+        }
+    }
+
+    fileprivate func cloneMirrorToOverlay(_ guid: GUID) -> Deferred<Maybe<Int>> {
+        let whereClause = "WHERE guid = ?"
+        let args: Args = [guid]
+
+        return self.cloneMirrorToOverlay(whereClause: whereClause, args: args)
+    }
+
+    fileprivate func markMirrorAsOverridden(_ guid: GUID) -> Success {
+        let args: Args = [guid]
+        let sql =
+        "UPDATE \(TableLoginsMirror) SET " +
+        "is_overridden = 1 " +
+        "WHERE guid = ?"
+
+        return self.db.run(sql, withArgs: args)
+    }
+
+    /**
+     * Replace the local DB row with the provided GUID.
+     * If no local overlay exists, one is first created.
+     *
+     * If `significant` is `true`, the `sync_status` of the row is bumped to at least `Changed`.
+     * If it's already `New`, it remains marked as `New`.
+     *
+     * This flag allows callers to make minor changes (such as incrementing a usage count)
+     * without triggering an upload or a conflict.
+     */
+    open func updateLoginByGUID(_ guid: GUID, new: LoginData, significant: Bool) -> Success {
+        if let error = new.isValid.failureValue {
+            return deferMaybe(error)
+        }
+
+        // Right now this method is only ever called if the password changes at
+        // point of use, so we always set `timePasswordChanged` and `timeLastUsed`.
+        // We can (but don't) also assume that `significant` will always be `true`,
+        // at least for the time being.
+        let nowMicro = Date.nowMicroseconds()
+        let nowMilli = nowMicro / 1000
+        let dateMicro = nowMicro
+        let dateMilli = nowMilli
+
+        let args: Args = [
+            dateMilli,            // local_modified
+            dateMicro,            // timeLastUsed
+            dateMicro,            // timePasswordChanged
+            new.httpRealm,
+            new.formSubmitURL,
+            new.usernameField,
+            new.passwordField,
+            new.password,
+            new.hostname,
+            new.username,
+            guid,
+        ]
+
+        let update =
+        "UPDATE \(TableLoginsLocal) SET " +
+        "  local_modified = ?, timeLastUsed = ?, timePasswordChanged = ?" +
+        ", httpRealm = ?, formSubmitURL = ?, usernameField = ?" +
+        ", passwordField = ?, timesUsed = timesUsed + 1" +
+        ", password = ?, hostname = ?, username = ?" +
+
+        // We keep rows marked as New in preference to marking them as changed. This allows us to
+        // delete them immediately if they don't reach the server.
+        (significant ? ", sync_status = max(sync_status, 1) " : "") +
+        " WHERE guid = ?"
+
+        return self.ensureLocalOverlayExistsForGUID(guid)
+           >>> { self.markMirrorAsOverridden(guid) }
+           >>> { self.db.run(update, withArgs: args) }
+            >>> effect(self.notifyLoginDidChange)
+    }
+
+    open func addUseOfLoginByGUID(_ guid: GUID) -> Success {
+        let sql =
+        "UPDATE \(TableLoginsLocal) SET " +
+        "timesUsed = timesUsed + 1, timeLastUsed = ?, local_modified = ? " +
+        "WHERE guid = ? AND is_deleted = 0"
+
+        // For now, mere use is not enough to flip sync_status to Changed.
+
+        let nowMicro = Date.nowMicroseconds()
+        let nowMilli = nowMicro / 1000
+        let args: Args = [nowMicro, nowMilli, guid]
+
+        return self.ensureLocalOverlayExistsForGUID(guid)
+           >>> { self.markMirrorAsOverridden(guid) }
+           >>> { self.db.run(sql, withArgs: args) }
+    }
+
+    open func removeLoginByGUID(_ guid: GUID) -> Success {
+        return removeLoginsWithGUIDs([guid])
+    }
+
+    fileprivate func getDeletionStatementsForGUIDs(_ guids: ArraySlice<GUID>, nowMillis: Timestamp) -> [(sql: String, args: Args?)] {
+        let inClause = BrowserDB.varlist(guids.count)
+
+        // Immediately delete anything that's marked as new -- i.e., it's never reached
+        // the server.
+        let delete =
+        "DELETE FROM \(TableLoginsLocal) WHERE guid IN \(inClause) AND sync_status = \(SyncStatus.new.rawValue)"
+
+        // Otherwise, mark it as changed.
+        let update =
+        "UPDATE \(TableLoginsLocal) SET " +
+        " local_modified = \(nowMillis)" +
+        ", sync_status = \(SyncStatus.changed.rawValue)" +
+        ", is_deleted = 1" +
+        ", password = ''" +
+        ", hostname = ''" +
+        ", username = ''" +
+        " WHERE guid IN \(inClause)"
+
+        let markMirrorAsOverridden =
+        "UPDATE \(TableLoginsMirror) SET " +
+            "is_overridden = 1 " +
+            "WHERE guid IN \(inClause)"
+
+        let insert =
+        "INSERT OR IGNORE INTO \(TableLoginsLocal) " +
+            "(guid, local_modified, is_deleted, sync_status, hostname, timeCreated, timePasswordChanged, password, username) " +
+        "SELECT guid, \(nowMillis), 1, \(SyncStatus.changed.rawValue), '', timeCreated, \(nowMillis)000, '', '' FROM \(TableLoginsMirror) WHERE guid IN \(inClause)"
+
+        let args: Args = guids.map { $0 }
+        return [ (delete, args), (update, args), (markMirrorAsOverridden, args), (insert, args)]
+    }
+
+    open func removeLoginsWithGUIDs(_ guids: [GUID]) -> Success {
+        let timestamp = Date.now()
+        return db.run(chunk(guids, by: BrowserDB.MaxVariableNumber).flatMap {
+            self.getDeletionStatementsForGUIDs($0, nowMillis: timestamp)
+        }) >>> effect(self.notifyLoginDidChange)
+    }
+
+    open func removeAll() -> Success {
+        // Immediately delete anything that's marked as new -- i.e., it's never reached
+        // the server. If Sync isn't set up, this will be everything.
+        let delete =
+        "DELETE FROM \(TableLoginsLocal) WHERE sync_status = \(SyncStatus.new.rawValue)"
+
+        let nowMillis = Date.now()
+
+        // Mark anything we haven't already deleted.
+        let update =
+        "UPDATE \(TableLoginsLocal) SET local_modified = \(nowMillis), sync_status = \(SyncStatus.changed.rawValue), is_deleted = 1, password = '', hostname = '', username = '' WHERE is_deleted = 0"
+
+        // Copy all the remaining rows from our mirror, marking them as locally deleted. The
+        // OR IGNORE will cause conflicts due to non-unique guids to be dropped, preserving
+        // anything we already deleted.
+        let insert =
+        "INSERT OR IGNORE INTO \(TableLoginsLocal) (guid, local_modified, is_deleted, sync_status, hostname, timeCreated, timePasswordChanged, password, username) " +
+        "SELECT guid, \(nowMillis), 1, \(SyncStatus.changed.rawValue), '', timeCreated, \(nowMillis)000, '', '' FROM \(TableLoginsMirror)"
+
+        // After that, we mark all of the mirror rows as overridden.
+        return self.db.run(delete)
+           >>> { self.db.run(update) }
+           >>> { self.db.run("UPDATE \(TableLoginsMirror) SET is_overridden = 1") }
+           >>> { self.db.run(insert) }
+            >>> effect(self.notifyLoginDidChange)
+    }
+}
+
+// When a server change is detected (e.g., syncID changes), we should consider shifting the contents
+// of the mirror into the local overlay, allowing a content-based reconciliation to occur on the next
+// full sync. Or we could flag the mirror as to-clear, download the server records and un-clear, and
+// resolve the remainder on completion. This assumes that a fresh start will typically end up with
+// the exact same records, so we might as well keep the shared parents around and double-check.
+extension SQLiteLogins: SyncableLogins {
+    /**
+     * Delete the login with the provided GUID. Succeeds if the GUID is unknown.
+     */
+    public func deleteByGUID(_ guid: GUID, deletedAt: Timestamp) -> Success {
+        // Simply ignore the possibility of a conflicting local change for now.
+        let local = "DELETE FROM \(TableLoginsLocal) WHERE guid = ?"
+        let remote = "DELETE FROM \(TableLoginsMirror) WHERE guid = ?"
+        let args: Args = [guid]
+
+        return self.db.run(local, withArgs: args) >>> { self.db.run(remote, withArgs: args) }
+    }
+
+    func getExistingMirrorRecordByGUID(_ guid: GUID) -> Deferred<Maybe<MirrorLogin?>> {
+        let sql = "SELECT * FROM \(TableLoginsMirror) WHERE guid = ? LIMIT 1"
+        let args: Args = [guid]
+        return self.db.runQuery(sql, args: args, factory: SQLiteLogins.MirrorLoginFactory) >>== { deferMaybe($0[0]) }
+    }
+
+    func getExistingLocalRecordByGUID(_ guid: GUID) -> Deferred<Maybe<LocalLogin?>> {
+        let sql = "SELECT * FROM \(TableLoginsLocal) WHERE guid = ? LIMIT 1"
+        let args: Args = [guid]
+        return self.db.runQuery(sql, args: args, factory: SQLiteLogins.LocalLoginFactory) >>== { deferMaybe($0[0]) }
+    }
+
+    fileprivate func storeReconciledLogin(_ login: Login) -> Success {
+        let dateMilli = Date.now()
+
+        let args: Args = [
+            dateMilli,            // local_modified
+            login.httpRealm,
+            login.formSubmitURL,
+            login.usernameField,
+            login.passwordField,
+            login.timeLastUsed,
+            login.timePasswordChanged,
+            login.timesUsed,
+            login.password,
+            login.hostname,
+            login.username,
+            login.guid,
+        ]
+
+        let update =
+        "UPDATE \(TableLoginsLocal) SET " +
+            "  local_modified = ?" +
+            ", httpRealm = ?, formSubmitURL = ?, usernameField = ?" +
+            ", passwordField = ?, timeLastUsed = ?, timePasswordChanged = ?, timesUsed = ?" +
+            ", password = ?" +
+            ", hostname = ?, username = ?" +
+            ", sync_status = \(SyncStatus.changed.rawValue) " +
+        " WHERE guid = ?"
+
+        return self.db.run(update, withArgs: args)
+    }
+
+    public func applyChangedLogin(_ upstream: ServerLogin) -> Success {
+        // Our login storage tracks the shared parent from the last sync (the "mirror").
+        // This allows us to conclusively determine what changed in the case of conflict.
+        //
+        // Our first step is to determine whether the record is changed or new: i.e., whether
+        // or not it's present in the mirror.
+        //
+        // TODO: these steps can be done in a single query. Make it work, make it right, make it fast.
+        // TODO: if there's no mirror record, all incoming records can be applied in one go; the only
+        // reason we need to fetch first is to establish the shared parent. That would be nice.
+        let guid = upstream.guid
+        return self.getExistingMirrorRecordByGUID(guid) >>== { mirror in
+            return self.getExistingLocalRecordByGUID(guid) >>== { local in
+                return self.applyChangedLogin(upstream, local: local, mirror: mirror)
+            }
+        }
+    }
+
+    fileprivate func applyChangedLogin(_ upstream: ServerLogin, local: LocalLogin?, mirror: MirrorLogin?) -> Success {
+        // Once we have the server record, the mirror record (if any), and the local overlay (if any),
+        // we can always know which state a record is in.
+
+        // If it's present in the mirror, then we can proceed directly to handling the change;
+        // we assume that once a record makes it into the mirror, that the local record association
+        // has already taken place, and we're tracking local changes correctly.
+        if let mirror = mirror {
+            log.debug("Mirror record found for changed record \(mirror.guid).")
+            if let local = local {
+                log.debug("Changed local overlay found for \(local.guid). Resolving conflict with 3WM.")
+                // * Changed remotely and locally (conflict). Resolve the conflict using a three-way merge: the
+                //   local mirror is the shared parent of both the local overlay and the new remote record.
+                //   Apply results as in the co-creation case.
+                return self.resolveConflictBetween(local: local, upstream: upstream, shared: mirror)
+            }
+
+            log.debug("No local overlay found. Updating mirror to upstream.")
+            // * Changed remotely but not locally. Apply the remote changes to the mirror.
+            //   There is no local overlay to discard or resolve against.
+            return self.updateMirrorToLogin(upstream, fromPrevious: mirror)
+        }
+
+        // * New both locally and remotely with no shared parent (cocreation).
+        //   Or we matched the GUID, and we're assuming we just forgot the mirror.
+        //
+        //   Merge and apply the results remotely, writing the result into the mirror and discarding the overlay
+        //   if the upload succeeded. (Doing it in this order allows us to safely replay on failure.)
+        //
+        //   If the local and remote record are the same, this is trivial.
+        //   At this point we also switch our local GUID to match the remote.
+        if let local = local {
+            // We might have randomly computed the same GUID on two devices connected
+            // to the same Sync account.
+            // With our 9-byte GUIDs, the chance of that happening is very small, so we
+            // assume that this device has previously connected to this account, and we
+            // go right ahead with a merge.
+            log.debug("Local record with GUID \(local.guid) but no mirror. This is unusual; assuming disconnect-reconnect scenario. Smushing.")
+            return self.resolveConflictWithoutParentBetween(local: local, upstream: upstream)
+        }
+
+        // If it's not present, we must first check whether we have a local record that's substantially
+        // the same -- the co-creation or re-sync case.
+        //
+        // In this case, we apply the server record to the mirror, change the local record's GUID,
+        // and proceed to reconcile the change on a content basis.
+        return self.findLocalRecordByContent(upstream) >>== { local in
+            if let local = local {
+                log.debug("Local record \(local.guid) content-matches new remote record \(upstream.guid). Smushing.")
+                return self.resolveConflictWithoutParentBetween(local: local, upstream: upstream)
+            }
+
+            // * New upstream only; no local overlay, content-based merge,
+            //   or shared parent in the mirror. Insert it in the mirror.
+            log.debug("Never seen remote record \(upstream.guid). Mirroring.")
+            return self.insertNewMirror(upstream)
+        }
+    }
+
+    // N.B., the final guid is sometimes a WHERE and sometimes inserted.
+    fileprivate func mirrorArgs(_ login: ServerLogin) -> Args {
+        let args: Args = [
+            login.serverModified,
+            login.httpRealm,
+            login.formSubmitURL,
+            login.usernameField,
+            login.passwordField,
+            login.timesUsed,
+            login.timeLastUsed,
+            login.timePasswordChanged,
+            login.timeCreated,
+            login.password,
+            login.hostname,
+            login.username,
+            login.guid,
+        ]
+        return args
+    }
+
+    /**
+     * Called when we have a changed upstream record and no local changes.
+     * There's no need to flip the is_overridden flag.
+     */
+    fileprivate func updateMirrorToLogin(_ login: ServerLogin, fromPrevious previous: Login) -> Success {
+        let args = self.mirrorArgs(login)
+        let sql =
+        "UPDATE \(TableLoginsMirror) SET " +
+        " server_modified = ?" +
+        ", httpRealm = ?, formSubmitURL = ?, usernameField = ?" +
+        ", passwordField = ?" +
+
+        // These we need to coalesce, because we might be supplying zeroes if the remote has
+        // been overwritten by an older client. In this case, preserve the old value in the
+        // mirror.
+        ", timesUsed = coalesce(nullif(?, 0), timesUsed)" +
+        ", timeLastUsed = coalesce(nullif(?, 0), timeLastUsed)" +
+        ", timePasswordChanged = coalesce(nullif(?, 0), timePasswordChanged)" +
+        ", timeCreated = coalesce(nullif(?, 0), timeCreated)" +
+        ", password = ?, hostname = ?, username = ?" +
+        " WHERE guid = ?"
+
+        return self.db.run(sql, withArgs: args)
+    }
+
+    /**
+     * Called when we have a completely new record. Naturally the new record
+     * is marked as non-overridden.
+     */
+    fileprivate func insertNewMirror(_ login: ServerLogin, isOverridden: Int = 0) -> Success {
+        let args = self.mirrorArgs(login)
+        let sql =
+        "INSERT OR IGNORE INTO \(TableLoginsMirror) (" +
+            " is_overridden, server_modified" +
+            ", httpRealm, formSubmitURL, usernameField" +
+            ", passwordField, timesUsed, timeLastUsed, timePasswordChanged, timeCreated" +
+            ", password, hostname, username, guid" +
+        ") VALUES (\(isOverridden), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+
+        return self.db.run(sql, withArgs: args)
+    }
+
+    /**
+     * We assume a local record matches if it has the same username (password can differ),
+     * hostname, httpRealm. We also check that the formSubmitURLs are either blank or have the
+     * same host and port.
+     *
+     * This is roughly the same as desktop's .matches():
+     * <https://mxr.mozilla.org/mozilla-central/source/toolkit/components/passwordmgr/nsLoginInfo.js#41>
+     */
+    fileprivate func findLocalRecordByContent(_ login: Login) -> Deferred<Maybe<LocalLogin?>> {
+        let primary =
+        "SELECT * FROM \(TableLoginsLocal) WHERE " +
+        "hostname IS ? AND httpRealm IS ? AND username IS ?"
+
+        var args: Args = [login.hostname, login.httpRealm, login.username]
+        let sql: String
+
+        if login.formSubmitURL == nil {
+            sql = primary + " AND formSubmitURL IS NULL"
+        } else if login.formSubmitURL!.isEmpty {
+            sql = primary
+        } else {
+            if let hostPort = login.formSubmitURL?.asURL?.hostPort {
+                // Substring check will suffice for now. TODO: proper host/port check after fetching the cursor.
+                sql = primary + " AND (formSubmitURL = '' OR (instr(formSubmitURL, ?) > 0))"
+                args.append(hostPort)
+            } else {
+                log.warning("Incoming formSubmitURL is non-empty but is not a valid URL with a host. Not matching local.")
+                return deferMaybe(nil)
+            }
+        }
+
+        return self.db.runQuery(sql, args: args, factory: SQLiteLogins.LocalLoginFactory)
+          >>== { cursor in
+            switch cursor.count {
+            case 0:
+                return deferMaybe(nil)
+            case 1:
+                // Great!
+                return deferMaybe(cursor[0])
+            default:
+                // TODO: join against the mirror table to exclude local logins that
+                // already match a server record.
+                // Right now just take the first.
+                log.warning("Got \(cursor.count) local logins with matching details! This is most unexpected.")
+                return deferMaybe(cursor[0])
+            }
+        }
+    }
+
+    fileprivate func resolveConflictBetween(local: LocalLogin, upstream: ServerLogin, shared: Login) -> Success {
+        // Attempt to compute two delta sets by comparing each new record to the shared record.
+        // Then we can merge the two delta sets -- either perfectly or by picking a winner in the case
+        // of a true conflict -- and produce a resultant record.
+
+        let localDeltas = (local.localModified, local.deltas(from: shared))
+        let upstreamDeltas = (upstream.serverModified, upstream.deltas(from: shared))
+
+        let mergedDeltas = Login.mergeDeltas(a: localDeltas, b: upstreamDeltas)
+
+        // Not all Sync clients handle the optional timestamp fields introduced in Bug 555755.
+        // We might get a server record with no timestamps, and it will differ from the original
+        // mirror!
+        // We solve that by refusing to generate deltas that discard information. We'll preserve
+        // the local values -- either from the local record or from the last shared parent that
+        // still included them -- and propagate them back to the server.
+        // It's OK for us to reconcile and reupload; it causes extra work for every client, but
+        // should not cause looping.
+        let resultant = shared.applyDeltas(mergedDeltas)
+
+        // We can immediately write the downloaded upstream record -- the old one -- to
+        // the mirror store.
+        // We then apply this record to the local store, and mark it as needing upload.
+        // When the reconciled record is uploaded, it'll be flushed into the mirror
+        // with the correct modified time.
+        return self.updateMirrorToLogin(upstream, fromPrevious: shared)
+            >>> { self.storeReconciledLogin(resultant) }
+    }
+
+    fileprivate func resolveConflictWithoutParentBetween(local: LocalLogin, upstream: ServerLogin) -> Success {
+        // Do the best we can. Either the local wins and will be
+        // uploaded, or the remote wins and we delete our overlay.
+        if local.timePasswordChanged > upstream.timePasswordChanged {
+            log.debug("Conflicting records with no shared parent. Using newer local record.")
+            return self.insertNewMirror(upstream, isOverridden: 1)
+        }
+
+        log.debug("Conflicting records with no shared parent. Using newer remote record.")
+        let args: Args = [local.guid]
+        return self.insertNewMirror(upstream, isOverridden: 0)
+            >>> { self.db.run("DELETE FROM \(TableLoginsLocal) WHERE guid = ?", withArgs: args) }
+    }
+
+    public func getModifiedLoginsToUpload() -> Deferred<Maybe<[Login]>> {
+        let sql =
+        "SELECT * FROM \(TableLoginsLocal) " +
+        "WHERE sync_status IS NOT \(SyncStatus.synced.rawValue) AND is_deleted = 0"
+
+        // Swift 2.0: use Cursor.asArray directly.
+        return self.db.runQuery(sql, args: nil, factory: SQLiteLogins.LoginFactory)
+          >>== { deferMaybe($0.asArray()) }
+    }
+
+    public func getDeletedLoginsToUpload() -> Deferred<Maybe<[GUID]>> {
+        // There are no logins that are marked as deleted that were not originally synced --
+        // others are deleted immediately.
+        let sql =
+        "SELECT guid FROM \(TableLoginsLocal) " +
+        "WHERE is_deleted = 1"
+
+        // Swift 2.0: use Cursor.asArray directly.
+        return self.db.runQuery(sql, args: nil, factory: { return $0["guid"] as! GUID })
+            >>== { deferMaybe($0.asArray()) }
+    }
+
+    /**
+     * Chains through the provided timestamp.
+     */
+    public func markAsSynchronized<T: Collection>(_ guids: T, modified: Timestamp) -> Deferred<Maybe<Timestamp>> where T.Iterator.Element == GUID {
+        // Update the mirror from the local record that we just uploaded.
+        // sqlite doesn't support UPDATE FROM, so instead of running 10 subqueries * n GUIDs,
+        // we issue a single DELETE and a single INSERT on the mirror, then throw away the
+        // local overlay that we just uploaded with another DELETE.
+        log.debug("Marking \(guids.count) GUIDs as synchronized.")
+
+        let queries: [(String, Args?)] = chunkCollection(guids, by: BrowserDB.MaxVariableNumber) { guids in
+            let args: Args = guids.map { $0 }
+            let inClause = BrowserDB.varlist(args.count)
+
+            let delMirror = "DELETE FROM \(TableLoginsMirror) WHERE guid IN \(inClause)"
+
+            let insMirror =
+            "INSERT OR IGNORE INTO \(TableLoginsMirror) (" +
+                " is_overridden, server_modified" +
+                ", httpRealm, formSubmitURL, usernameField" +
+                ", passwordField, timesUsed, timeLastUsed, timePasswordChanged, timeCreated" +
+                ", password, hostname, username, guid" +
+            ") SELECT 0, \(modified)" +
+            ", httpRealm, formSubmitURL, usernameField" +
+            ", passwordField, timesUsed, timeLastUsed, timePasswordChanged, timeCreated" +
+            ", password, hostname, username, guid " +
+            "FROM \(TableLoginsLocal) " +
+            "WHERE guid IN \(inClause)"
+
+            let delLocal = "DELETE FROM \(TableLoginsLocal) WHERE guid IN \(inClause)"
+
+            return [(delMirror, args),
+                    (insMirror, args),
+                    (delLocal, args)]
+        }
+
+        return self.db.run(queries)
+         >>> always(modified)
+    }
+
+    public func markAsDeleted<T: Collection>(_ guids: T) -> Success where T.Iterator.Element == GUID {
+        log.debug("Marking \(guids.count) GUIDs as deleted.")
+
+        let queries: [(String, Args?)] = chunkCollection(guids, by: BrowserDB.MaxVariableNumber) { guids in
+            let args: Args = guids.map { $0 }
+            let inClause = BrowserDB.varlist(args.count)
+            return [("DELETE FROM \(TableLoginsMirror) WHERE guid IN \(inClause)", args),
+                    ("DELETE FROM \(TableLoginsLocal) WHERE guid IN \(inClause)", args)]
+        }
+
+        return self.db.run(queries)
+    }
+
+    public func hasSyncedLogins() -> Deferred<Maybe<Bool>> {
+        let checkLoginsMirror = "SELECT 1 FROM \(TableLoginsMirror)"
+        let checkLoginsLocal = "SELECT 1 FROM \(TableLoginsLocal) WHERE sync_status IS NOT \(SyncStatus.new.rawValue)"
+
+        let sql = "\(checkLoginsMirror) UNION ALL \(checkLoginsLocal)"
+        return self.db.queryReturnsResults(sql)
+    }
+}
+
+extension SQLiteLogins: ResettableSyncStorage {
+    /**
+     * Clean up any metadata.
+     * TODO: is this safe for a regular reset? It forces a content-based merge.
+     */
+    public func resetClient() -> Success {
+        // Clone all the mirrors so we don't lose data.
+        return self.cloneMirrorToOverlay(whereClause: nil, args: nil)
+
+        // Drop all of the mirror data.
+        >>> { self.db.run("DELETE FROM \(TableLoginsMirror)") }
+
+        // Mark all of the local data as new.
+        >>> { self.db.run("UPDATE \(TableLoginsLocal) SET sync_status = \(SyncStatus.new.rawValue)") }
+    }
+}
+
+extension SQLiteLogins: AccountRemovalDelegate {
+    public func onRemovedAccount() -> Success {
+        return self.resetClient()
+    }
+}

mobile/ios/Storage/SQL/SQLiteMetadata.swift

@@ -0,0 +1,58 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Deferred
+import Shared
+
+/// The sqlite-backed implementation of the metadata protocol containing images and content for pages.
+open class SQLiteMetadata {
+    let db: BrowserDB
+
+    required public init(db: BrowserDB) {
+        self.db = db
+    }
+}
+
+extension SQLiteMetadata: Metadata {
+    // A cache key is a conveninent, readable identifier for a site in the metadata database which helps
+    // with deduping entries for the same page.
+    typealias CacheKey = String
+
+    /// Persists the given PageMetadata object to browser.db in the page_metadata table.
+    ///
+    /// - parameter metadata: Metadata object
+    /// - parameter pageURL:  URL of page metadata was fetched from
+    /// - parameter expireAt: Expiration/TTL interval for when this metadata should expire at.
+    ///
+    /// - returns: Deferred on success
+    public func storeMetadata(_ metadata: PageMetadata, forPageURL pageURL: URL,
+                              expireAt: UInt64) -> Success {
+        guard let cacheKey = pageURL.displayURL?.absoluteString else {
+            return succeed()
+        }
+
+        // Replace any matching cache_key entries if they exist
+        let selectUniqueCacheKey = "COALESCE((SELECT cache_key FROM \(TablePageMetadata) WHERE cache_key = ?), ?)"
+        let args: Args = [cacheKey, cacheKey, metadata.siteURL, metadata.mediaURL, metadata.title,
+                          metadata.type, metadata.description, metadata.providerName,
+                          expireAt]
+
+        let insert =
+        "INSERT OR REPLACE INTO \(TablePageMetadata)" +
+        "(cache_key, site_url, media_url, title, type, description, provider_name, expired_at) " +
+        "VALUES ( \(selectUniqueCacheKey), ?, ?, ?, ?, ?, ?, ?)"
+
+        return self.db.run(insert, withArgs: args)
+    }
+
+    /// Purges any metadata items living in page_metadata that are expired.
+    ///
+    /// - returns: Deferred on success
+    public func deleteExpiredMetadata() -> Success {
+        let sql = "DELETE FROM page_metadata WHERE expired_at <= (CAST(strftime('%s', 'now') AS LONG)*1000)"
+        return self.db.run(sql)
+    }
+
+}

mobile/ios/Storage/SQL/SQLiteQueue.swift

@@ -0,0 +1,35 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+open class SQLiteQueue: TabQueue {
+    let db: BrowserDB
+
+    public init(db: BrowserDB) {
+        self.db = db
+    }
+
+    open func addToQueue(_ tab: ShareItem) -> Success {
+        let args: Args = [tab.url, tab.title]
+        return db.run("INSERT OR IGNORE INTO \(TableQueuedTabs) (url, title) VALUES (?, ?)", withArgs: args)
+    }
+
+    fileprivate func factory(_ row: SDRow) -> ShareItem {
+        return ShareItem(url: row["url"] as! String, title: row["title"] as? String, favicon: nil)
+    }
+
+    open func getQueuedTabs() -> Deferred<Maybe<Cursor<ShareItem>>> {
+        return db.runQuery("SELECT url, title FROM \(TableQueuedTabs)", args: nil, factory: self.factory)
+    }
+
+    open func clearQueuedTabs() -> Success {
+        return db.run("DELETE FROM \(TableQueuedTabs)")
+    }
+}

mobile/ios/Storage/SQL/SQLiteRemoteClientsAndTabs.swift

@@ -0,0 +1,382 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+open class SQLiteRemoteClientsAndTabs: RemoteClientsAndTabs {
+    let db: BrowserDB
+
+    public init(db: BrowserDB) {
+        self.db = db
+    }
+    
+    class func remoteClientFactory(_ row: SDRow) -> RemoteClient {
+        let guid = row["guid"] as? String
+        let name = row["name"] as! String
+        let mod = (row["modified"] as! NSNumber).uint64Value
+        let type = row["type"] as? String
+        let form = row["formfactor"] as? String
+        let os = row["os"] as? String
+        let version = row["version"] as? String
+        let fxaDeviceId = row["fxaDeviceId"] as? String
+        return RemoteClient(guid: guid, name: name, modified: mod, type: type, formfactor: form, os: os, version: version, fxaDeviceId: fxaDeviceId)
+    }
+    
+    class func remoteTabFactory(_ row: SDRow) -> RemoteTab {
+        let clientGUID = row["client_guid"] as? String
+        let url = URL(string: row["url"] as! String)! // TODO: find a way to make this less dangerous.
+        let title = row["title"] as! String
+        let history = SQLiteRemoteClientsAndTabs.convertStringToHistory(row["history"] as? String)
+        let lastUsed = row.getTimestamp("last_used")!
+        return RemoteTab(clientGUID: clientGUID, URL: url, title: title, history: history, lastUsed: lastUsed, icon: nil)
+    }
+
+    class func convertStringToHistory(_ history: String?) -> [URL] {
+        guard let data = history?.data(using: String.Encoding.utf8),
+            let decoded = try? JSONSerialization.jsonObject(with: data, options: [JSONSerialization.ReadingOptions.allowFragments]),
+            let urlStrings = decoded as? [String] else {
+                return []
+        }
+        return optFilter(urlStrings.flatMap { URL(string: $0) }) 
+    }
+
+    class func convertHistoryToString(_ history: [URL]) -> String? {
+        let historyAsStrings = optFilter(history.map { $0.absoluteString })
+        
+        guard let data = try? JSONSerialization.data(withJSONObject: historyAsStrings, options: []) else {
+            return nil
+        }
+        return String(data: data, encoding: String.Encoding(rawValue: String.Encoding.utf8.rawValue))
+    }
+
+    open func wipeClients() -> Success {
+        return db.run("DELETE FROM \(TableClients)")
+    }
+
+    open func wipeRemoteTabs() -> Success {
+        return db.run("DELETE FROM \(TableTabs) WHERE client_guid IS NOT NULL")
+    }
+
+    open func wipeTabs() -> Success {
+        return db.run("DELETE FROM \(TableTabs)")
+    }
+
+    open func insertOrUpdateTabs(_ tabs: [RemoteTab]) -> Deferred<Maybe<Int>> {
+        return self.insertOrUpdateTabsForClientGUID(nil, tabs: tabs)
+    }
+
+    open func insertOrUpdateTabsForClientGUID(_ clientGUID: String?, tabs: [RemoteTab]) -> Deferred<Maybe<Int>> {
+        let deleteQuery = "DELETE FROM \(TableTabs) WHERE client_guid IS ?"
+        let deleteArgs: Args = [clientGUID]
+
+        return db.transaction { connection -> Int in
+            // Delete any existing tabs.
+            try connection.executeChange(deleteQuery, withArgs: deleteArgs)
+
+            // Insert replacement tabs.
+            var inserted = 0
+            for tab in tabs {
+                let args: Args = [
+                    tab.clientGUID,
+                    tab.URL.absoluteString,
+                    tab.title,
+                    SQLiteRemoteClientsAndTabs.convertHistoryToString(tab.history),
+                    NSNumber(value: tab.lastUsed)
+                ]
+                
+                let lastInsertedRowID = connection.lastInsertedRowID
+                
+                // We trust that each tab's clientGUID matches the supplied client!
+                // Really tabs shouldn't have a GUID at all. Future cleanup!
+                try connection.executeChange("INSERT INTO \(TableTabs) (client_guid, url, title, history, last_used) VALUES (?, ?, ?, ?, ?)", withArgs: args)
+                
+                if connection.lastInsertedRowID == lastInsertedRowID {
+                    log.debug("Unable to INSERT RemoteTab!")
+                } else {
+                    inserted += 1
+                }
+            }
+
+            return inserted
+        }
+    }
+
+    open func insertOrUpdateClients(_ clients: [RemoteClient]) -> Deferred<Maybe<Int>> {
+        // TODO: insert multiple clients in a single query.
+        // ORM systems are foolish.
+        return db.transaction { connection -> Int in
+            var succeeded = 0
+
+            // Update or insert client records.
+            for client in clients {
+                let args: Args = [
+                    client.name,
+                    NSNumber(value: client.modified),
+                    client.type,
+                    client.formfactor,
+                    client.os,
+                    client.version,
+                    client.fxaDeviceId,
+                    client.guid
+                ]
+                
+                try connection.executeChange("UPDATE \(TableClients) SET name = ?, modified = ?, type = ?, formfactor = ?, os = ?, version = ?, fxaDeviceId = ? WHERE guid = ?", withArgs: args)
+                
+                if connection.numberOfRowsModified == 0 {
+                    let args: Args = [
+                        client.guid,
+                        client.name,
+                        NSNumber(value: client.modified),
+                        client.type,
+                        client.formfactor,
+                        client.os,
+                        client.version,
+                        client.fxaDeviceId
+                    ]
+                    
+                    let lastInsertedRowID = connection.lastInsertedRowID
+                    
+                    try connection.executeChange("INSERT INTO \(TableClients) (guid, name, modified, type, formfactor, os, version, fxaDeviceId) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", withArgs: args)
+                    
+                    if connection.lastInsertedRowID == lastInsertedRowID {
+                        log.debug("INSERT did not change last inserted row ID.")
+                    }
+                }
+                
+                succeeded += 1
+            }
+
+            return succeeded
+        }
+    }
+
+    open func insertOrUpdateClient(_ client: RemoteClient) -> Deferred<Maybe<Int>> {
+        return insertOrUpdateClients([client])
+    }
+
+    open func deleteClient(guid: GUID) -> Success {
+        let deleteTabsQuery = "DELETE FROM \(TableTabs) WHERE client_guid = ?"
+        let deleteClientQuery = "DELETE FROM \(TableClients) WHERE guid = ?"
+        let deleteArgs: Args = [guid]
+
+        return db.transaction { connection -> Void in
+            try connection.executeChange(deleteClientQuery, withArgs: deleteArgs)
+            try connection.executeChange(deleteTabsQuery, withArgs: deleteArgs)
+        }
+    }
+
+    open func getClient(guid: GUID) -> Deferred<Maybe<RemoteClient?>> {
+        let factory = SQLiteRemoteClientsAndTabs.remoteClientFactory
+        return self.db.runQuery("SELECT * FROM \(TableClients) WHERE guid = ?", args: [guid], factory: factory) >>== { deferMaybe($0[0]) }
+    }
+
+    open func getClient(fxaDeviceId: String) -> Deferred<Maybe<RemoteClient?>> {
+        let factory = SQLiteRemoteClientsAndTabs.remoteClientFactory
+        return self.db.runQuery("SELECT * FROM \(TableClients) WHERE fxaDeviceId = ?", args: [fxaDeviceId], factory: factory) >>== { deferMaybe($0[0]) }
+    }
+
+    open func getClientWithId(_ clientID: GUID) -> Deferred<Maybe<RemoteClient?>> {
+        return self.getClient(guid: clientID)
+    }
+
+    open func getClients() -> Deferred<Maybe<[RemoteClient]>> {
+        return db.withConnection { connection -> [RemoteClient] in
+            let cursor = connection.executeQuery("SELECT * FROM \(TableClients) WHERE EXISTS (SELECT 1 FROM \(TableRemoteDevices) rd WHERE rd.guid = fxaDeviceId) ORDER BY modified DESC", factory: SQLiteRemoteClientsAndTabs.remoteClientFactory)
+            defer {
+                cursor.close()
+            }
+
+            return cursor.asArray()
+        }
+    }
+
+    open func getClientGUIDs() -> Deferred<Maybe<Set<GUID>>> {
+        let c = db.runQuery("SELECT guid FROM \(TableClients) WHERE guid IS NOT NULL", args: nil, factory: { $0["guid"] as! String })
+        return c >>== { cursor in
+            let guids = Set<GUID>(cursor.asArray())
+            return deferMaybe(guids)
+        }
+    }
+
+    open func getTabsForClientWithGUID(_ guid: GUID?) -> Deferred<Maybe<[RemoteTab]>> {
+        let tabsSQL: String
+        let clientArgs: Args?
+        if let _ = guid {
+            tabsSQL = "SELECT * FROM \(TableTabs) WHERE client_guid = ?"
+            clientArgs = [guid]
+        } else {
+            tabsSQL = "SELECT * FROM \(TableTabs) WHERE client_guid IS NULL"
+            clientArgs = nil
+        }
+
+        log.debug("Looking for tabs for client with guid: \(guid ?? "nil")")
+        return db.runQuery(tabsSQL, args: clientArgs, factory: SQLiteRemoteClientsAndTabs.remoteTabFactory) >>== {
+            let tabs = $0.asArray()
+            log.debug("Found \(tabs.count) tabs for client with guid: \(guid ?? "nil")")
+            return deferMaybe(tabs)
+        }
+    }
+
+    open func getClientsAndTabs() -> Deferred<Maybe<[ClientAndTabs]>> {
+        return db.withConnection { conn -> ([RemoteClient], [RemoteTab]) in
+            let clientsCursor = conn.executeQuery("SELECT * FROM \(TableClients) WHERE EXISTS (SELECT 1 FROM \(TableRemoteDevices) rd WHERE rd.guid = fxaDeviceId) ORDER BY modified DESC", factory: SQLiteRemoteClientsAndTabs.remoteClientFactory)
+            let tabsCursor = conn.executeQuery("SELECT * FROM \(TableTabs) WHERE client_guid IS NOT NULL ORDER BY client_guid DESC, last_used DESC", factory: SQLiteRemoteClientsAndTabs.remoteTabFactory)
+
+            defer {
+                clientsCursor.close()
+                tabsCursor.close()
+            }
+
+            return (clientsCursor.asArray(), tabsCursor.asArray())
+        } >>== { clients, tabs in
+            var acc = [String: [RemoteTab]]()
+            for tab in tabs {
+                if let guid = tab.clientGUID {
+                    if acc[guid] == nil {
+                        acc[guid] = [tab]
+                    } else {
+                        acc[guid]!.append(tab)
+                    }
+                } else {
+                    log.error("RemoteTab (\(tab)) has a nil clientGUID")
+                }
+            }
+
+            // Most recent first.
+            let fillTabs: (RemoteClient) -> ClientAndTabs = { client in
+                var tabs: [RemoteTab]? = nil
+                if let guid: String = client.guid {
+                    tabs = acc[guid]
+                }
+                return ClientAndTabs(client: client, tabs: tabs ?? [])
+            }
+
+            return deferMaybe(clients.map(fillTabs))
+        }
+    }
+
+    open func deleteCommands() -> Success {
+        return db.run("DELETE FROM \(TableSyncCommands)")
+    }
+
+    open func deleteCommands(_ clientGUID: GUID) -> Success {
+        return db.run("DELETE FROM \(TableSyncCommands) WHERE client_guid = ?", withArgs: [clientGUID] as Args)
+    }
+
+    open func insertCommand(_ command: SyncCommand, forClients clients: [RemoteClient]) -> Deferred<Maybe<Int>> {
+        return insertCommands([command], forClients: clients)
+    }
+
+    open func insertCommands(_ commands: [SyncCommand], forClients clients: [RemoteClient]) -> Deferred<Maybe<Int>> {
+        return db.transaction { connection -> Int in
+            var numberOfInserts = 0
+
+            // Update or insert client records.
+            for command in commands {
+                for client in clients {
+                    do {
+                        if let commandID = try self.insert(connection, sql: "INSERT INTO \(TableSyncCommands) (client_guid, value) VALUES (?, ?)", args: [client.guid, command.value] as Args) {
+                            log.verbose("Inserted command: \(commandID)")
+                            numberOfInserts += 1
+                        } else {
+                            log.warning("Command not inserted, but no error!")
+                        }
+                    } catch let err as NSError {
+                        log.error("insertCommands(_:, forClients:) failed: \(err.localizedDescription) (numberOfInserts: \(numberOfInserts)")
+                        throw err
+                    }
+                }
+            }
+
+            return numberOfInserts
+        }
+    }
+
+    open func getCommands() -> Deferred<Maybe<[GUID: [SyncCommand]]>> {
+        return db.withConnection { connection -> [GUID: [SyncCommand]] in
+            let cursor = connection.executeQuery("SELECT * FROM \(TableSyncCommands)", factory: { row -> SyncCommand in
+                SyncCommand(
+                    id: row["command_id"] as? Int,
+                    value: row["value"] as! String,
+                    clientGUID: row["client_guid"] as? GUID)
+            })
+            defer {
+                cursor.close()
+            }
+
+            return self.clientsFromCommands(cursor.asArray())
+        }
+    }
+
+    func clientsFromCommands(_ commands: [SyncCommand]) -> [GUID: [SyncCommand]] {
+        var syncCommands = [GUID: [SyncCommand]]()
+        for command in commands {
+            var cmds: [SyncCommand] = syncCommands[command.clientGUID!] ?? [SyncCommand]()
+            cmds.append(command)
+            syncCommands[command.clientGUID!] = cmds
+        }
+        return syncCommands
+    }
+    
+    func insert(_ db: SQLiteDBConnection, sql: String, args: Args?) throws -> Int? {
+        let lastID = db.lastInsertedRowID
+        try db.executeChange(sql, withArgs: args)
+        
+        let id = db.lastInsertedRowID
+        if id == lastID {
+            log.debug("INSERT did not change last inserted row ID.")
+            return nil
+        }
+        
+        return id
+    }
+}
+
+extension SQLiteRemoteClientsAndTabs: RemoteDevices {
+    open func replaceRemoteDevices(_ remoteDevices: [RemoteDevice]) -> Success {
+        // Drop corrupted records and our own record too.
+        let remoteDevices = remoteDevices.filter { $0.id != nil && $0.type != nil && !$0.isCurrentDevice }
+
+        return db.transaction { conn -> Void in
+            try conn.executeChange("DELETE FROM \(TableRemoteDevices)")
+
+            let now = Date.now()
+
+            for device in remoteDevices {
+                let sql =
+                    "INSERT INTO \(TableRemoteDevices) (guid, name, type, is_current_device, date_created, date_modified, last_access_time) " +
+                "VALUES (?, ?, ?, ?, ?, ?, ?)"
+                let args: Args = [device.id, device.name, device.type, device.isCurrentDevice, now, now, device.lastAccessTime]
+                try conn.executeChange(sql, withArgs: args)
+            }
+        }
+    }
+}
+
+extension SQLiteRemoteClientsAndTabs: ResettableSyncStorage {
+    public func resetClient() -> Success {
+        // For this engine, resetting is equivalent to wiping.
+        return self.clear()
+    }
+
+    public func clear() -> Success {
+        return db.transaction { conn -> Void in
+            try conn.executeChange("DELETE FROM \(TableTabs) WHERE client_guid IS NOT NULL")
+            try conn.executeChange("DELETE FROM \(TableClients)")
+        }
+    }
+}
+
+extension SQLiteRemoteClientsAndTabs: AccountRemovalDelegate {
+    public func onRemovedAccount() -> Success {
+        log.info("Clearing clients and tabs after account removal.")
+        // TODO: Bug 1168690 - delete our client and tabs records from the server.
+        return self.resetClient()
+    }
+}

mobile/ios/Storage/SQL/Schema.swift

@@ -0,0 +1,23 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+
+/**
+ * Something that knows how to construct a database.
+ */
+public protocol Schema {
+    var name: String { get }
+    var version: Int { get }
+    
+    func create(_ db: SQLiteDBConnection) -> Bool
+    func update(_ db: SQLiteDBConnection, from: Int) -> Bool
+    func drop(_ db: SQLiteDBConnection) -> Bool
+}
+
+enum SchemaUpgradeResult {
+    case success
+    case failure
+    case skipped
+}