Dactyloidae iOS initial commit

mobile/ios/Sync/BatchingClient.swift

@@ -0,0 +1,294 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Alamofire
+import Shared
+import XCGLogger
+import Deferred
+
+open class SerializeRecordFailure<T: CleartextPayloadJSON>: MaybeErrorType, SyncPingFailureFormattable {
+    open let record: Record<T>
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+
+    open var description: String {
+        return "Failed to serialize record: \(record)"
+    }
+
+    public init(record: Record<T>) {
+        self.record = record
+    }
+}
+
+private let log = Logger.syncLogger
+
+private typealias UploadRecord = (guid: GUID, payload: String, sizeBytes: Int)
+public typealias DeferredResponse = Deferred<Maybe<StorageResponse<POSTResult>>>
+
+typealias BatchUploadFunction = (_ lines: [String], _ ifUnmodifiedSince: Timestamp?, _ queryParams: [URLQueryItem]?) -> Deferred<Maybe<StorageResponse<POSTResult>>>
+
+private let commitParam = URLQueryItem(name: "commit", value: "true")
+
+private enum AccumulateRecordError: MaybeErrorType {
+    var description: String {
+        switch self {
+        case .full:
+            return "Batch or payload is full."
+        case .unknown:
+            return "Unknown errored while trying to accumulate records in batch"
+        }
+    }
+
+    case full(uploadOp: DeferredResponse)
+    case unknown
+}
+
+open class TooManyRecordsError: MaybeErrorType, SyncPingFailureFormattable {
+    open var description: String {
+        return "Trying to send too many records in a single batch."
+    }
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+}
+
+open class RecordsFailedToUpload: MaybeErrorType, SyncPingFailureFormattable {
+    open var description: String {
+        return "Some records failed to upload"
+    }
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+}
+
+open class Sync15BatchClient<T: CleartextPayloadJSON> {
+    fileprivate(set) var ifUnmodifiedSince: Timestamp?
+
+    fileprivate let config: InfoConfiguration
+    fileprivate let uploader: BatchUploadFunction
+    fileprivate let serializeRecord: (Record<T>) -> String?
+
+    fileprivate var batchToken: BatchToken?
+
+    // Keep track of the limits of a single batch
+    fileprivate var totalBytes: ByteCount = 0
+    fileprivate var totalRecords: Int = 0
+
+    // Keep track of the limits of a single POST
+    fileprivate var postBytes: ByteCount = 0
+    fileprivate var postRecords: Int = 0
+
+    fileprivate var records = [UploadRecord]()
+
+    fileprivate var onCollectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp
+
+    fileprivate func batchQueryParamWithValue(_ value: String) -> URLQueryItem {
+        return URLQueryItem(name: "batch", value: value)
+    }
+
+    init(config: InfoConfiguration, ifUnmodifiedSince: Timestamp? = nil, serializeRecord: @escaping (Record<T>) -> String?,
+         uploader: @escaping BatchUploadFunction, onCollectionUploaded: @escaping (POSTResult, Timestamp?) -> DeferredTimestamp) {
+        self.config = config
+        self.ifUnmodifiedSince = ifUnmodifiedSince
+        self.uploader = uploader
+        self.serializeRecord = serializeRecord
+
+        self.onCollectionUploaded = onCollectionUploaded
+    }
+
+    open func endBatch() -> Success {
+        guard !records.isEmpty else {
+            return succeed()
+        }
+
+        if let token = self.batchToken {
+            return commitBatch(token) >>> succeed
+        }
+
+        let lines = self.freezePost()
+        return self.uploader(lines, self.ifUnmodifiedSince, nil)
+            >>== effect(moveForward)
+            >>> succeed
+    }
+
+    // If in batch mode, will discard the batch if any record fails
+    open func endSingleBatch() -> Deferred<Maybe<(succeeded: [GUID], lastModified: Timestamp?)>> {
+        return self.start() >>== { response in
+            let succeeded = response.value.success
+            guard let token = self.batchToken else {
+                return deferMaybe((succeeded: succeeded, lastModified: response.metadata.lastModifiedMilliseconds))
+            }
+            guard succeeded.count == self.totalRecords else {
+                return deferMaybe(RecordsFailedToUpload())
+            }
+            return self.commitBatch(token) >>== { commitResp in
+                return deferMaybe((succeeded: succeeded, lastModified: commitResp.metadata.lastModifiedMilliseconds))
+            }
+        }
+    }
+
+    open func addRecords(_ records: [Record<T>], singleBatch: Bool = false) -> Success {
+        guard !records.isEmpty else {
+            return succeed()
+        }
+
+        // Eagerly serializer the record prior to processing them so we can catch any issues
+        // with record sizes before we start uploading to the server.
+        let serializeThunks = records.map { record in
+            return { self.serialize(record) }
+        }
+
+        return accumulate(serializeThunks) >>== {
+            let iter = $0.makeIterator()
+            if singleBatch {
+                return self.addRecordsInSingleBatch(iter)
+            } else {
+                return self.addRecords(iter)
+            }
+        }
+    }
+
+    fileprivate func addRecords(_ generator: IndexingIterator<[UploadRecord]>) -> Success {
+        var mutGenerator = generator
+        while let record = mutGenerator.next() {
+            return accumulateOrUpload(record) >>> { self.addRecords(mutGenerator) }
+        }
+        return succeed()
+    }
+
+    fileprivate func addRecordsInSingleBatch(_ generator: IndexingIterator<[UploadRecord]>) -> Success {
+        var mutGenerator = generator
+        while let record = mutGenerator.next() {
+            guard self.addToPost(record) else {
+                return deferMaybe(TooManyRecordsError())
+            }
+        }
+        return succeed()
+    }
+
+    fileprivate func accumulateOrUpload(_ record: UploadRecord) -> Success {
+        return accumulateRecord(record).bind { result in
+            // Try to add the record to our buffer
+            guard let e = result.failureValue as? AccumulateRecordError else {
+                return succeed()
+            }
+
+            switch e {
+            case .full(let uploadOp):
+                return uploadOp >>> { self.accumulateOrUpload(record) }
+            default:
+                return deferMaybe(e)
+            }
+        }
+    }
+
+    fileprivate func accumulateRecord(_ record: UploadRecord) -> Success {
+        guard let token = self.batchToken else {
+            guard addToPost(record) else {
+                return deferMaybe(AccumulateRecordError.full(uploadOp: self.start()))
+            }
+            return succeed()
+        }
+
+        guard fitsInBatch(record) else {
+            return deferMaybe(AccumulateRecordError.full(uploadOp: self.commitBatch(token)))
+        }
+
+        guard addToPost(record) else {
+            return deferMaybe(AccumulateRecordError.full(uploadOp: self.postInBatch(token)))
+        }
+
+        addToBatch(record)
+        return succeed()
+    }
+
+    fileprivate func serialize(_ record: Record<T>) -> Deferred<Maybe<UploadRecord>> {
+        guard let line = self.serializeRecord(record) else {
+            return deferMaybe(SerializeRecordFailure(record: record))
+        }
+
+        let lineSize = line.utf8.count
+        guard lineSize < Sync15StorageClient.maxRecordSizeBytes else {
+            return deferMaybe(RecordTooLargeError(size: lineSize, guid: record.id))
+        }
+
+        return deferMaybe((record.id, line, lineSize))
+    }
+
+    fileprivate func addToPost(_ record: UploadRecord) -> Bool {
+        guard postRecords + 1 <= config.maxPostRecords && postBytes + record.sizeBytes <= config.maxPostBytes else {
+            return false
+        }
+        postRecords += 1
+        postBytes += record.sizeBytes
+        records.append(record)
+        return true
+    }
+
+    fileprivate func fitsInBatch(_ record: UploadRecord) -> Bool {
+        return totalRecords + 1 <= config.maxTotalRecords && totalBytes + record.sizeBytes <= config.maxTotalBytes
+    }
+
+    fileprivate func addToBatch(_ record: UploadRecord) {
+        totalRecords += 1
+        totalBytes += record.sizeBytes
+    }
+
+    fileprivate func postInBatch(_ token: BatchToken) -> DeferredResponse {
+        // Push up the current payload to the server and reset
+        let lines = self.freezePost()
+        return uploader(lines, self.ifUnmodifiedSince, [batchQueryParamWithValue(token)])
+    }
+
+    fileprivate func commitBatch(_ token: BatchToken) -> DeferredResponse {
+        resetBatch()
+        let lines = self.freezePost()
+        let queryParams = [batchQueryParamWithValue(token), commitParam]
+        return uploader(lines, self.ifUnmodifiedSince, queryParams)
+            >>== effect(moveForward)
+    }
+
+    fileprivate func start() -> DeferredResponse {
+        let postRecordCount = self.postRecords
+        let postBytesCount = self.postBytes
+        let lines = freezePost()
+        return self.uploader(lines, self.ifUnmodifiedSince, [batchQueryParamWithValue("true")])
+             >>== effect(moveForward)
+             >>== { response in
+                if let token = response.value.batchToken {
+                    self.batchToken = token
+
+                    // Now that we've started a batch, make sure to set the counters for the batch to include
+                    // the records we just sent as part of the start call.
+                    self.totalRecords = postRecordCount
+                    self.totalBytes = postBytesCount
+                }
+
+                return deferMaybe(response)
+            }
+    }
+
+    fileprivate func moveForward(_ response: StorageResponse<POSTResult>) {
+        let lastModified = response.metadata.lastModifiedMilliseconds
+        self.ifUnmodifiedSince = lastModified
+        _ = self.onCollectionUploaded(response.value, lastModified)
+    }
+
+    fileprivate func resetBatch() {
+        totalBytes = 0
+        totalRecords = 0
+        self.batchToken = nil
+    }
+
+    fileprivate func freezePost() -> [String] {
+        let lines = records.map { $0.payload }
+        self.records = []
+        self.postBytes = 0
+        self.postRecords = 0
+        return lines
+    }
+}

mobile/ios/Sync/BookmarkPayload.swift

@@ -0,0 +1,571 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+public protocol MirrorItemable {
+    func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem
+}
+
+extension BookmarkMirrorItem {
+    func asPayload() -> BookmarkBasePayload {
+        return BookmarkType.somePayloadFromJSON(self.asJSON())
+    }
+
+    func asPayloadWithChildren(_ children: [GUID]?) -> BookmarkBasePayload {
+        let remappedChildren: [GUID]?
+        if let children = children {
+            if BookmarkRoots.RootGUID == self.guid {
+                // Only the root contains roots, and so only its children
+                // need to be translated.
+                remappedChildren = children.map(BookmarkRoots.translateOutgoingRootGUID)
+            } else {
+                remappedChildren = children
+            }
+        } else {
+            remappedChildren = nil
+        }
+
+        let json = self.asJSONWithChildren(remappedChildren)
+        return BookmarkType.somePayloadFromJSON(json)
+    }
+}
+
+/**
+ * Hierarchy:
+ * - BookmarkBasePayload
+ *   \_ FolderPayload
+ *      \_ LivemarkPayload
+ *   \_ SeparatorPayload
+ *   \_ BookmarkPayload
+ *      \_ BookmarkQueryPayload
+ */
+
+public enum BookmarkType: String {
+    case livemark
+    case separator
+    case folder
+    case bookmark
+    case query
+    case microsummary     // Dead: now a bookmark.
+
+    // The result might be invalid, but it won't be nil.
+    public static func somePayloadFromJSON(_ json: JSON) -> BookmarkBasePayload {
+        return payloadFromJSON(json) ?? BookmarkBasePayload(json)
+    }
+
+    public static func payloadFromJSON(_ json: JSON) -> BookmarkBasePayload? {
+        if json["deleted"].bool ?? false {
+            // Deleted records won't have a type.
+            return BookmarkBasePayload(json)
+        }
+
+        guard let typeString = json["type"].string else {
+            return nil
+        }
+
+        guard let type = BookmarkType(rawValue: typeString) else {
+            return nil
+        }
+
+        switch type {
+        case microsummary:
+            fallthrough
+        case bookmark:
+            return BookmarkPayload(json)
+        case folder:
+            return FolderPayload(json)
+        case livemark:
+            return LivemarkPayload(json)
+        case separator:
+            return SeparatorPayload(json)
+        case query:
+            return BookmarkQueryPayload(json)
+        }
+    }
+
+    public static func isValid(_ type: String?) -> Bool {
+        guard let type = type else {
+            return false
+        }
+
+        return BookmarkType(rawValue: type) != nil
+    }
+}
+
+open class LivemarkPayload: BookmarkBasePayload {
+    open var feedURI: String? {
+        return self["feedUri"].string
+    }
+
+    open var siteURI: String? {
+        return self["siteUri"].string
+    }
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+        return self.hasRequiredStringFields(["feedUri", "siteUri"])
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        guard let p = obj as? LivemarkPayload else {
+            return false
+        }
+
+        if !super.equalPayloads(p) {
+            return false
+        }
+
+        if self.deleted {
+            return true
+        }
+
+        if self.feedURI != p.feedURI {
+            return false
+        }
+
+        if self.siteURI != p.siteURI {
+            return false
+        }
+
+        return true
+    }
+
+    override open func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem {
+        if self.deleted {
+            return BookmarkMirrorItem.deleted(.livemark, guid: self.id, modified: modified)
+        }
+
+        return BookmarkMirrorItem.livemark(
+            self.id,
+            dateAdded: self["dateAdded"].uInt64,
+            modified: modified,
+            hasDupe: self.hasDupe,
+            // TODO: these might need to be weakened if real-world data is dirty.
+            parentID: self["parentid"].stringValue,
+            parentName: self["parentName"].string,
+            title: self["title"].string,
+            description: self["description"].string,
+            feedURI: self.feedURI!,
+            siteURI: self.siteURI!
+        )
+    }
+}
+
+open class SeparatorPayload: BookmarkBasePayload {
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+        if !self["pos"].isInt() {
+            log.warning("Separator \(self.id) missing pos.")
+            return false
+        }
+        return true
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        guard let p = obj as? SeparatorPayload else {
+            return false
+        }
+
+        if !super.equalPayloads(p) {
+            return false
+        }
+
+        if self.deleted {
+            return true
+        }
+
+        if self["pos"].int != p["pos"].int {
+            return false
+        }
+
+        return true
+    }
+
+    override open func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem {
+        if self.deleted {
+            return BookmarkMirrorItem.deleted(.separator, guid: self.id, modified: modified)
+        }
+
+        return BookmarkMirrorItem.separator(
+            self.id,
+            dateAdded: self["dateAdded"].uInt64,
+            modified: modified,
+            hasDupe: self.hasDupe,
+            // TODO: these might need to be weakened if real-world data is dirty.
+            parentID: self["parentid"].string!,
+            parentName: self["parentName"].string,
+            pos: self["pos"].int!
+        )
+    }
+}
+
+open class FolderPayload: BookmarkBasePayload {
+    fileprivate var childrenAreValid: Bool {
+        return self.hasStringArrayField("children")
+    }
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if !self.hasRequiredStringFields(["title"]) {
+            log.warning("Folder \(self.id) missing title.")
+            return false
+        }
+
+        if !self.hasOptionalStringFields(["description"]) {
+            log.warning("Folder \(self.id) missing string description.")
+            return false
+
+        }
+        if !self.childrenAreValid {
+            log.warning("Folder \(self.id) has invalid children.")
+            return false
+        }
+
+        return true
+    }
+
+    open var children: [String] {
+        return self["children"].arrayValue.map { $0.string! }
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        guard let p = obj as? FolderPayload else {
+            return false
+        }
+
+        if !super.equalPayloads(p) {
+            return false
+        }
+
+        if self.deleted {
+            return true
+        }
+
+        if self["title"].string != p["title"].string {
+            return false
+        }
+
+        if self["description"].string != p["description"].string {
+            return false
+        }
+
+        if self.children != p.children {
+            return false
+        }
+
+        return true
+    }
+
+    override open func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem {
+        if self.deleted {
+            return BookmarkMirrorItem.deleted(.folder, guid: self.id, modified: modified)
+        }
+
+        return BookmarkMirrorItem.folder(
+            self.id,
+            dateAdded: self["dateAdded"].uInt64,
+            modified: modified,
+            hasDupe: self.hasDupe,
+            // TODO: these might need to be weakened if real-world data is dirty.
+            parentID: self["parentid"].string!,
+            parentName: self["parentName"].string,
+            title: self["title"].string!,
+            description: self["description"].string,
+            children: self.children
+        )
+    }
+}
+
+open class BookmarkPayload: BookmarkBasePayload {
+    fileprivate static let requiredBookmarkStringFields = ["bmkUri"]
+
+    // Title *should* be required, but can be missing for queries. Great.
+    fileprivate static let optionalBookmarkStringFields = ["title", "keyword", "description"]
+    fileprivate static let optionalBookmarkBooleanFields = ["loadInSidebar"]
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if !self.hasRequiredStringFields(BookmarkPayload.requiredBookmarkStringFields) {
+            log.warning("Bookmark \(self.id) missing required string field.")
+            return false
+        }
+
+        if !self.hasStringArrayField("tags") {
+            log.warning("Bookmark \(self.id) missing tags array. We'll replace with an empty array.")
+            // Ignore.
+        }
+
+        if !self.hasOptionalStringFields(BookmarkPayload.optionalBookmarkStringFields) {
+            return false
+        }
+
+        return self.hasOptionalBooleanFields(BookmarkPayload.optionalBookmarkBooleanFields)
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        guard let p = obj as? BookmarkPayload else {
+            return false
+        }
+
+        if !super.equalPayloads(p) {
+            return false
+        }
+
+        if self.deleted {
+            return true
+        }
+
+        if !BookmarkPayload.requiredBookmarkStringFields.every({ p[$0].string! == self[$0].string! }) {
+            return false
+        }
+
+        // TODO: compare optional fields.
+
+        if Set(self.tags) != Set(p.tags) {
+            return false
+        }
+
+        if self["loadInSidebar"].bool != p["loadInSidebar"].bool {
+            return false
+        }
+
+        return true
+    }
+
+    lazy var tags: [String] = {
+        return self["tags"].arrayValue.flatMap { $0.string } 
+    }()
+
+    lazy var tagsString: String = {
+        if self["tags"].isArray() {
+            return self["tags"].stringValue() ?? "[]"
+        }
+        return "[]"
+    }()
+
+    override open func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem {
+        if self.deleted {
+            return BookmarkMirrorItem.deleted(.bookmark, guid: self.id, modified: modified)
+        }
+
+        return BookmarkMirrorItem.bookmark(
+            self.id,
+            dateAdded: self["dateAdded"].uInt64,
+            modified: modified,
+            hasDupe: self.hasDupe,
+            // TODO: these might need to be weakened if real-world data is dirty.
+            parentID: self["parentid"].string!,
+            parentName: self["parentName"].string,
+            title: self["title"].string ?? "",
+            description: self["description"].string,
+            URI: self["bmkUri"].string!,
+            tags: self.tagsString,           // Stringify it so we can put the array in the DB.
+            keyword: self["keyword"].string
+        )
+    }
+}
+
+open class BookmarkQueryPayload: BookmarkPayload {
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if !self.hasOptionalStringFields(["queryId", "folderName"]) {
+            log.warning("Query \(self.id) missing queryId or folderName.")
+            return false
+        }
+
+        return true
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        guard let p = obj as? BookmarkQueryPayload else {
+            return false
+        }
+
+        if !super.equalPayloads(p) {
+            return false
+        }
+
+        if self.deleted {
+            return true
+        }
+
+        if self["folderName"].string != p["folderName"].string {
+            return false
+        }
+
+        if self["queryId"].string != p["queryId"].string {
+            return false
+        }
+
+        return true
+    }
+
+    override open func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem {
+        if self.deleted {
+            return BookmarkMirrorItem.deleted(.query, guid: self.id, modified: modified)
+        }
+
+        return BookmarkMirrorItem.query(
+            self.id,
+            dateAdded: self["dateAdded"].uInt64,
+            modified: modified,
+            hasDupe: self.hasDupe,
+            parentID: self["parentid"].string!,
+            parentName: self["parentName"].string,
+            title: self["title"].string ?? "",
+            description: self["description"].string,
+            URI: self["bmkUri"].string!,
+            tags: self.tagsString,           // Stringify it so we can put the array in the DB.
+            keyword: self["keyword"].string,
+            folderName: self["folderName"].string,
+            queryID: self["queryID"].string
+        )
+    }
+}
+
+open class BookmarkBasePayload: CleartextPayloadJSON, MirrorItemable {
+    fileprivate static let requiredStringFields: [String] = ["parentid", "type"]
+    fileprivate static let optionalBooleanFields: [String] = ["hasDupe"]
+
+    static func deletedPayload(_ guid: GUID) -> BookmarkBasePayload {
+        let remappedGUID = BookmarkRoots.translateOutgoingRootGUID(guid)
+        return BookmarkBasePayload(JSON(["id": remappedGUID, "deleted": true]))
+    }
+
+    func hasStringArrayField(_ name: String) -> Bool {
+        guard let arr = self[name].array else {
+            return false
+        }
+        return arr.every { $0.isString() }
+    }
+
+    func hasRequiredStringFields(_ fields: [String]) -> Bool {
+        return fields.every { self[$0].isString() }
+    }
+
+    func hasOptionalStringFields(_ fields: [String]) -> Bool {
+        return fields.every { field in
+            let val = self[field]
+            // Yup, 404 is not found, so this means "string or nothing".
+            let valid = val.isString() || val.isNull() || val.isError()
+            if !valid {
+                log.debug("Field \(field) is invalid: \(val).")
+            }
+            return valid
+        }
+    }
+
+    func hasOptionalBooleanFields(_ fields: [String]) -> Bool {
+        return fields.every { field in
+            let val = self[field]
+            // Yup, 404 is not found, so this means "boolean or nothing".
+            let valid = val.isBool() || val.isNull() || val.error?.code == 404
+            if !valid {
+                log.debug("Field \(field) is invalid: \(val).")
+            }
+            return valid
+        }
+    }
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if self["deleted"].bool ?? false {
+            return true
+        }
+
+        // If not deleted, we must be a specific, known, type!
+        if !BookmarkType.isValid(self["type"].string) {
+            return false
+        }
+
+        if !(self["parentName"].isString() || self.id == "places") {
+            if self["parentid"].string! == "places" {
+                log.debug("Accepting root with missing parent name.")
+            } else {
+                // Bug 1318414.
+                log.warning("Accepting bookmark with missing parent name.")
+            }
+        }
+
+        if !self.hasRequiredStringFields(BookmarkBasePayload.requiredStringFields) {
+            log.warning("Item missing required string field.")
+            return false
+        }
+
+        return self.hasOptionalBooleanFields(BookmarkBasePayload.optionalBooleanFields)
+    }
+
+    open var hasDupe: Bool {
+        return self["hasDupe"].bool ?? false
+    }
+
+    /**
+     * This only makes sense for valid payloads.
+     */
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        guard let p = obj as? BookmarkBasePayload else {
+            return false
+        }
+
+        if !super.equalPayloads(p) {
+            return false
+        }
+
+        if self.deleted {
+            return true
+        }
+
+        if p.deleted {
+            return self.deleted == p.deleted
+        }
+
+        // If either record is deleted, these other fields might be missing.
+        // But we just checked, so we're good to roll on.
+
+        let same: (String) -> Bool = { field in
+            let left = self[field].string
+            let right = p[field].string
+            return left == right
+        }
+
+        if !BookmarkBasePayload.requiredStringFields.every(same) {
+            return false
+        }
+
+        if p["parentName"].string != self["parentName"].string {
+            return false
+        }
+
+        return self.hasDupe == p.hasDupe
+    }
+
+    // This goes here because extensions cannot override methods yet.
+    open func toMirrorItem(_ modified: Timestamp) -> BookmarkMirrorItem {
+        precondition(self.deleted, "Non-deleted items should have a specific type.")
+        return BookmarkMirrorItem.deleted(.bookmark, guid: self.id, modified: modified)
+    }
+}

mobile/ios/Sync/BookmarkTelemetryPing.swift

@@ -0,0 +1,53 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+import SwiftyJSON
+import Shared
+
+public func makeAdHocBookmarkMergePing(_ bundle: Bundle, clientID: String, attempt: Int32, bufferRows: Int?, valid: [String: Bool], clientCount: Int) -> JSON {
+    let anyFailed = valid.reduce(false, { $0 || $1.1 })
+
+    var out: [String: Any] = [
+        "v": 1,
+        "appV": AppInfo.appVersion,
+        "build": AppInfo.buildNumber,
+        "id": clientID,
+        "attempt": Int(attempt),
+        "success": !anyFailed,
+        "date": Date().description,
+        "clientCount": clientCount,
+    ]
+
+    if let bufferRows = bufferRows {
+        out["rows"] = bufferRows
+    }
+
+    if anyFailed {
+        valid.forEach { key, value in
+            out[key] = value
+        }
+    }
+
+    return JSON(out)
+}
+
+public func makeAdHocSyncStatusPing(_ bundle: Bundle, clientID: String, statusObject: [String: String]?, engineResults: [String: String]?, resultsFailure: MaybeErrorType?, clientCount: Int) -> JSON {
+
+    let statusObject: Any = statusObject ?? JSON.null
+    let engineResults: Any = engineResults ?? JSON.null
+    let resultsFailure: Any = resultsFailure?.description ?? JSON.null
+
+    let out: [String: Any] = [
+        "v": 1,
+        "appV": AppInfo.appVersion,
+        "build": AppInfo.buildNumber,
+        "id": clientID,
+        "date": Date().description,
+        "clientCount": clientCount,
+        "statusObject": statusObject,
+        "engineResults": engineResults,
+        "resultsFailure": resultsFailure
+    ]
+
+    return JSON(out)
+}

mobile/ios/Sync/CleartextPayloadJSON.swift

@@ -0,0 +1,61 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import SwiftyJSON
+
+open class BasePayloadJSON {
+    let json: JSON
+    required public init(_ jsonString: String) {
+        self.json = JSON(parseJSON: jsonString)
+    }
+
+    public init(_ json: JSON) {
+        self.json = json
+    }
+
+    // Override me.
+    fileprivate func isValid() -> Bool {
+        return self.json.type != .unknown &&
+               self.json.error == nil
+    }
+
+    subscript(key: String) -> JSON {
+        get {
+            return json[key]
+        }
+    }
+}
+
+/**
+ * http://docs.services.mozilla.com/sync/objectformats.html
+ * "In addition to these custom collection object structures, the
+ *  Encrypted DataObject adds fields like id and deleted."
+ */
+open class CleartextPayloadJSON: BasePayloadJSON {
+    // Override me.
+    override open func isValid() -> Bool {
+        return super.isValid() && self["id"].isString()
+    }
+
+    open var id: String {
+        return self["id"].string!
+    }
+
+    open var deleted: Bool {
+        let d = self["deleted"]
+        if let bool = d.bool {
+            return bool
+        } else {
+            return false
+        }
+    }
+
+    // Override me.
+    // Doesn't check id. Should it?
+    open func equalPayloads (_ obj: CleartextPayloadJSON) -> Bool {
+        return self.deleted == obj.deleted
+    }
+}

mobile/ios/Sync/ClientPayload.swift

@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import SwiftyJSON
+
+open class ClientPayload: CleartextPayloadJSON {
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if self["deleted"].bool ?? false {
+            return true
+        }
+
+        return self["name"].isString() &&
+               self["type"].isString()
+    }
+
+    var commands: [JSON] {
+        return self["commands"].array ?? []   // It might not be present at all.
+    }
+
+    var name: String {
+        return self["name"].stringValue
+    }
+
+    var clientType: String {
+        return self["type"].stringValue
+    }
+    
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        if !(obj is ClientPayload) {
+            return false
+        }
+
+        if !super.equalPayloads(obj) {
+            return false
+        }
+
+        let p = obj as! ClientPayload
+        if p.name != self.name {
+            return false
+        }
+        
+        if p.clientType != self.clientType {
+            return false
+        }
+
+        return true
+    }
+
+    // TODO: version, protocols.
+}

mobile/ios/Sync/EncryptedJSON.swift

@@ -0,0 +1,169 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import FxA
+import Account
+import XCGLogger
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+/**
+ * Turns JSON of the form
+ *
+ *  { ciphertext: ..., hmac: ..., iv: ...}
+ *
+ * into a new JSON object resulting from decrypting and parsing the ciphertext.
+ */
+open class EncryptedJSON {
+    var json: JSON
+    var _cleartext: JSON?               // Cache decrypted cleartext.
+    var _ciphertextBytes: Data?       // Cache decoded ciphertext.
+    var _hmacBytes: Data?             // Cache decoded HMAC.
+    var _ivBytes: Data?               // Cache decoded IV.
+
+    var valid: Bool = false
+    var validated: Bool = false
+
+    let keyBundle: KeyBundle
+
+    public init(json: String, keyBundle: KeyBundle) {
+        self.keyBundle = keyBundle
+        self.json = JSON(parseJSON: json)
+    }
+
+    public init(json: JSON, keyBundle: KeyBundle) {
+        self.keyBundle = keyBundle
+        self.json = json
+    }
+
+    // For validating HMAC: the raw ciphertext as bytes without decoding.
+    fileprivate var ciphertextB64: Data? {
+        if let ct = self["ciphertext"].string {
+            return Bytes.dataFromBase64(ct)
+        }
+        return nil
+    }
+
+    /**
+     * You probably want to call validate() and then use .ciphertext.
+     */
+    fileprivate var ciphertextBytes: Data? {
+        return Bytes.decodeBase64(self["ciphertext"].string!)
+    }
+
+    fileprivate func validate() -> Bool {
+        if validated {
+            return valid
+        }
+
+        defer { validated = true }
+
+        guard self["ciphertext"].isString() &&
+              self["hmac"].isString() &&
+              self["IV"].isString() else {
+            valid = false
+            return false
+        }
+
+        guard let ciphertextForHMAC = self.ciphertextB64 else {
+            valid = false
+            return false
+        }
+
+        guard keyBundle.verify(hmac: self.hmac, ciphertextB64: ciphertextForHMAC) else {
+            valid = false
+            return false
+        }
+
+        // I guess we called validate twice…
+        if self._ciphertextBytes != nil {
+            valid = true
+            return true
+        }
+
+        // Also verify that the ciphertext is valid base64. Do this by
+        // retrieving the value in a failable way, leaving the accessors
+        // to take the dangerous/simple path.
+        // We can force-unwrap self["ciphertext"] because we already checked
+        // it when verifying the HMAC above.
+        guard let data = self.ciphertextBytes else {
+            log.error("Unable to decode ciphertext base64 in record \(self["id"].string ?? "<unknown>")")
+            valid = false
+            return false
+        }
+
+        self._ciphertextBytes = data
+        valid = true
+        return valid
+    }
+
+    open func isValid() -> Bool {
+        return !json.isError() && self.validate()
+    }
+
+    /**
+     * Make sure you call isValid first. This API force-unwraps for simplicity.
+     */
+    var ciphertext: Data {
+        if _ciphertextBytes != nil {
+            return _ciphertextBytes!
+        }
+
+        _ciphertextBytes = self.ciphertextBytes
+        return _ciphertextBytes!
+    }
+
+    var hmac: Data {
+        if _hmacBytes != nil {
+            return _hmacBytes!
+        }
+        //NSData(base16EncodedString: self["hmac"].asString!, options: NSDataBase16DecodingOptions.Default)
+        _hmacBytes = NSData(base16EncodedString: self["hmac"].stringValue, options: []) as Data
+        return _hmacBytes!
+    }
+
+    var iv: Data {
+        if _ivBytes != nil {
+            return _ivBytes!
+        }
+
+        _ivBytes = Bytes.decodeBase64(self["IV"].string!)
+        return _ivBytes!
+    }
+
+    // Returns nil on error.
+    open var cleartext: JSON? {
+        if _cleartext != nil {
+            return _cleartext
+        }
+
+        if !isValid() {
+            log.error("Failed to validate.")
+            return nil
+        }
+
+        let decrypted: String? = keyBundle.decrypt(self.ciphertext, iv: self.iv)
+        if decrypted == nil {
+            log.error("Failed to decrypt.")
+            valid = false
+            return nil
+        }
+
+        _cleartext = JSON(parseJSON: decrypted!)
+        return _cleartext!
+    }
+
+    subscript(key: String) -> JSON {
+        get {
+            return json[key]
+        }
+
+        set {
+            json[key] = newValue
+        }
+    }
+}

mobile/ios/Sync/EnvelopeJSON.swift

@@ -0,0 +1,73 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import SwiftyJSON
+
+open class EnvelopeJSON {
+    fileprivate let json: JSON
+
+    public init(_ jsonString: String) {
+        self.json = JSON(parseJSON: jsonString)
+    }
+
+    public init(_ json: JSON) {
+        self.json = json
+    }
+
+    open func isValid() -> Bool {
+        return !self.json.isError() &&
+            self.json["id"].isString() &&
+            //self["collection"].isString &&
+            self.json["payload"].isString()
+    }
+
+    open var id: String {
+        return self.json["id"].string!
+    }
+
+    open var collection: String {
+        return self.json["collection"].string ?? ""
+    }
+
+    open var payload: String {
+        return self.json["payload"].string!
+    }
+
+    open var sortindex: Int {
+        let s = self.json["sortindex"]
+        return s.int ?? 0
+    }
+
+    open var modified: Timestamp {
+//        if let intValue = self.json["modified"].int64 {
+//            return Timestamp(intValue) * 1000
+//        }
+
+        if let doubleValue = self.json["modified"].double {
+            return Timestamp(1000 * (doubleValue))
+        }
+
+        return 0
+    }
+
+    open func toString() -> String {
+        return self.json.stringValue()!
+    }
+
+    open func withModified(_ now: Timestamp) -> EnvelopeJSON {
+        if var d = self.json.dictionary {
+            d["modified"] = JSON(Double(now) / 1000)
+            return EnvelopeJSON(JSON(d))
+        }
+        return EnvelopeJSON(JSON(parseJSON: "!")) // Intentionally bad JSON.
+    }
+}
+
+extension EnvelopeJSON {
+    func asJSON() -> JSON {
+        return self.json
+    }
+}

mobile/ios/Sync/HistoryPayload.swift

@@ -0,0 +1,82 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import SwiftyJSON
+
+open class HistoryPayload: CleartextPayloadJSON {
+    open class func fromJSON(_ json: JSON) -> HistoryPayload? {
+        let p = HistoryPayload(json)
+        if p.isValid() {
+            return p
+        }
+        return nil
+    }
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if self["deleted"].bool ?? false {
+            return true
+        }
+
+        return self["histUri"].string != nil &&      // TODO: validate URI.
+               self["title"].isStringOrNull() &&
+               self["visits"].isArray()
+    }
+
+    open func asPlace() -> Place {
+        return Place(guid: self.id, url: self.histURI, title: self.title)
+    }
+
+    var visits: [Visit] {
+        let visits = self["visits"].arrayObject as! [[String: Any]]
+        return optFilter(visits.map(Visit.fromJSON))
+    }
+
+    fileprivate var histURI: String {
+        return self["histUri"].string!
+    }
+
+    var historyURI: URL {
+        return self.histURI.asURL!
+    }
+
+    var title: String {
+        return self["title"].string ?? ""
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        if let p = obj as? HistoryPayload {
+            if !super.equalPayloads(p) {
+                return false
+            }
+
+            if p.deleted {
+                return self.deleted == p.deleted
+            }
+
+            // If either record is deleted, these other fields might be missing.
+            // But we just checked, so we're good to roll on.
+
+            if p.title != self.title {
+                return false
+            }
+
+            if p.historyURI != self.historyURI {
+                return false
+            }
+
+            // TODO: compare visits.
+
+            return true
+        }
+
+        return false
+    }
+}

mobile/ios/Sync/Info.plist

@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>FMWK</string>
+	<key>CFBundleShortVersionString</key>
+	<string>10.6</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+	<key>NSPrincipalClass</key>
+	<string></string>
+</dict>
+</plist>

mobile/ios/Sync/Info.swift

@@ -0,0 +1,75 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import SwiftyJSON
+
+open class InfoCollections {
+    fileprivate let collections: [String: Timestamp]
+
+    init(collections: [String: Timestamp]) {
+        self.collections = collections
+    }
+
+    open class func fromJSON(_ json: JSON) -> InfoCollections? {
+        if let dict = json.dictionary {
+            var coll = [String: Timestamp]()
+            for (key, value) in dict {
+                if let value = value.double {
+                    coll[key] = Timestamp(value * 1000)
+                } else {
+                    return nil       // Invalid, so bail out.
+                }
+            }
+            return InfoCollections(collections: coll)
+        }
+        return nil
+    }
+
+    open func collectionNames() -> [String] {
+        return Array(self.collections.keys)
+    }
+
+    open func modified(_ collection: String) -> Timestamp? {
+        return self.collections[collection]
+    }
+
+    // Two I/Cs are the same if they have the same modified times for a set of
+    // collections. If no collections are specified, they're considered the same
+    // if the other I/C has the same values for this I/C's collections, and
+    // they have the same collection array.
+    open func same(_ other: InfoCollections, collections: [String]?) -> Bool {
+        if let collections = collections {
+            return collections.every({ self.modified($0) == other.modified($0) })
+        }
+
+        // Same collections?
+        let ours = self.collectionNames()
+        let theirs = other.collectionNames()
+        return ours.sameElements(theirs) && same(other, collections: ours)
+    }
+}
+
+// Response object from https://<sync-endpoint-url>/info/configuration
+public struct InfoConfiguration {
+
+    // Maximum size in bytes of the overall HTTP request body.
+    public let maxRequestBytes: Int
+
+    // Maximum number of records that can be uploaded to a collection in a single POST request.
+    public let maxPostRecords: Int
+
+    // Maximum combined size in bytes of the record payloads that can be uploaded to a collection in
+    // a single POST request.
+    public let maxPostBytes: Int
+
+    // Maximum total number of records that can be uploaded to a collection as part of a batched upload.
+    public let maxTotalRecords: Int
+
+    // Maximum total combined size in bytes of the record payloads that can be uploaded to a collection
+    // as part of a batched upload.
+    public let maxTotalBytes: Int
+}
+

mobile/ios/Sync/KeyBundle.swift

@@ -0,0 +1,319 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import FxA
+import Account
+import SwiftyJSON
+
+private let KeyLength = 32
+
+open class KeyBundle: Hashable {
+    let encKey: Data
+    let hmacKey: Data
+
+    open class func fromKB(_ kB: Data) -> KeyBundle {
+        let salt = Data()
+        let contextInfo = FxAClient10.KW("oldsync")
+        let len: UInt = 64               // KeyLength + KeyLength, without type nonsense.
+        let derived = (kB as NSData).deriveHKDFSHA256Key(withSalt: salt, contextInfo: contextInfo, length: len)!
+        return KeyBundle(encKey: derived.subdata(in: 0..<KeyLength),
+                         hmacKey: derived.subdata(in: KeyLength..<(2 * KeyLength)))
+    }
+
+    open class func random() -> KeyBundle {
+        // Bytes.generateRandomBytes uses SecRandomCopyBytes, which hits /dev/random, which
+        // on iOS is populated by the OS from kernel-level sources of entropy.
+        // That should mean that we don't need to seed or initialize anything before calling
+        // this. That is probably not true on (some versions of) OS X.
+        return KeyBundle(encKey: Bytes.generateRandomBytes(32), hmacKey: Bytes.generateRandomBytes(32))
+    }
+
+    open class var invalid: KeyBundle {
+        return KeyBundle(encKeyB64: "deadbeef", hmacKeyB64: "deadbeef")!
+    }
+
+    public init?(encKeyB64: String, hmacKeyB64: String) {
+        guard let e = Bytes.decodeBase64(encKeyB64),
+              let h = Bytes.decodeBase64(hmacKeyB64) else {
+            return nil
+        }
+        self.encKey = e
+        self.hmacKey = h
+    }
+
+    public init(encKey: Data, hmacKey: Data) {
+        self.encKey = encKey
+        self.hmacKey = hmacKey
+    }
+
+    fileprivate func _hmac(_ ciphertext: Data) -> (data: UnsafeMutablePointer<CUnsignedChar>, len: Int) {
+        let hmacAlgorithm = CCHmacAlgorithm(kCCHmacAlgSHA256)
+        let digestLen: Int = Int(CC_SHA256_DIGEST_LENGTH)
+        let result = UnsafeMutablePointer<CUnsignedChar>.allocate(capacity: digestLen)
+        CCHmac(hmacAlgorithm, hmacKey.getBytes(), hmacKey.count, ciphertext.getBytes(), ciphertext.count, result)
+        return (result, digestLen)
+    }
+
+    open func hmac(_ ciphertext: Data) -> Data {
+        let (result, digestLen) = _hmac(ciphertext)
+        let data = NSMutableData(bytes: result, length: digestLen)
+
+        result.deinitialize()
+        result.deallocate(capacity: digestLen)
+        return data as Data
+    }
+
+    /**
+     * Returns a hex string for the HMAC.
+     */
+    open func hmacString(_ ciphertext: Data) -> String {
+        let (result, digestLen) = _hmac(ciphertext)
+        let hash = NSMutableString()
+        for i in 0..<digestLen {
+            hash.appendFormat("%02x", result[i])
+        }
+
+        result.deinitialize()
+        result.deallocate(capacity: digestLen)
+        return String(hash)
+    }
+
+    open func encrypt(_ cleartext: Data, iv: Data?=nil) -> (ciphertext: Data, iv: Data)? {
+        let iv = iv ?? Bytes.generateRandomBytes(16)
+
+        let (success, b, copied) = self.crypt(cleartext, iv: iv, op: CCOperation(kCCEncrypt))
+        let byteCount = cleartext.count + kCCBlockSizeAES128
+        if success == CCCryptorStatus(kCCSuccess) {
+            // Hooray!
+            let d = Data(bytes: b, count: Int(copied))
+            b.deallocate(bytes: byteCount, alignedTo: MemoryLayout<Void>.size)
+            return (d, iv)
+        }
+
+        b.deallocate(bytes: byteCount, alignedTo: MemoryLayout<Void>.size)
+        return nil
+    }
+
+    // You *must* verify HMAC before calling this.
+    open func decrypt(_ ciphertext: Data, iv: Data) -> String? {
+        let (success, b, copied) = self.crypt(ciphertext, iv: iv, op: CCOperation(kCCDecrypt))
+        let byteCount = ciphertext.count + kCCBlockSizeAES128
+        if success == CCCryptorStatus(kCCSuccess) {
+            // Hooray!
+            let d = Data(bytes: b, count: Int(copied))
+            let s = NSString(data: d, encoding: String.Encoding.utf8.rawValue)
+            b.deallocate(bytes: byteCount, alignedTo: MemoryLayout<Void>.size)
+            return s as String?
+        }
+
+        b.deallocate(bytes: byteCount, alignedTo: MemoryLayout<Void>.size)
+        return nil
+    }
+
+    fileprivate func crypt(_ input: Data, iv: Data, op: CCOperation) -> (status: CCCryptorStatus, buffer: UnsafeMutableRawPointer, count: Int) {
+        let resultSize = input.count + kCCBlockSizeAES128
+        var copied: Int = 0
+        let result = UnsafeMutableRawPointer.allocate(bytes: resultSize, alignedTo: MemoryLayout<Void>.size)
+
+        let success: CCCryptorStatus =
+        CCCrypt(op,
+                CCHmacAlgorithm(kCCAlgorithmAES128),
+                CCOptions(kCCOptionPKCS7Padding),
+                encKey.getBytes(),
+                kCCKeySizeAES256,
+                iv.getBytes(),
+                input.getBytes(),
+                input.count,
+                result,
+                resultSize,
+                &copied
+        )
+
+        return (success, result, copied)
+    }
+
+    open func verify(hmac: Data, ciphertextB64: Data) -> Bool {
+        let expectedHMAC = hmac
+        let computedHMAC = self.hmac(ciphertextB64)
+        return (expectedHMAC == computedHMAC)
+    }
+
+    /**
+     * Swift can't do functional factories. I would like to have one of the following
+     * approaches be viable:
+     *
+     * 1. Derive the constructor from the consumer of the factory.
+     * 2. Accept a type as input.
+     *
+     * Neither of these are viable, so we instead pass an explicit constructor closure.
+     *
+     * Most of these approaches produce either odd compiler errors, or -- worse --
+     * compile and then yield runtime EXC_BAD_ACCESS (see Radar 20230159).
+     *
+     * For this reason, be careful trying to simplify or improve this code.
+     */
+    open func factory<T: CleartextPayloadJSON>(_ f: @escaping (JSON) -> T) -> (String) -> T? {
+        return { (payload: String) -> T? in
+            let potential = EncryptedJSON(json: payload, keyBundle: self)
+            if !potential.isValid() {
+                return nil
+            }
+
+            let cleartext = potential.cleartext
+            if cleartext == nil {
+                return nil
+            }
+            return f(cleartext!)
+        }
+    }
+
+    // TODO: how much do we want to move this into EncryptedJSON?
+    open func serializer<T: CleartextPayloadJSON>(_ f: @escaping (T) -> JSON) -> (Record<T>) -> JSON? {
+        return { (record: Record<T>) -> JSON? in
+            let json = f(record.payload)
+            if json.isNull() {
+                // This should never happen, but if it does, we don't want to leak this
+                // record to the server!
+                return nil
+            }
+
+            let bytes: Data
+            do {
+                // Get the most basic kind of encoding: no pretty printing.
+                // This can throw; if so, we return nil.
+                // `rawData` simply calls JSONSerialization.dataWithJSONObject:options:error, which
+                // guarantees UTF-8 encoded output.
+                bytes = try json.rawData(options: [])
+            } catch {
+                return nil
+            }
+
+            // Given a valid non-null JSON object, we don't ever expect a round-trip to fail.
+            assert(!JSON(bytes).isNull())
+
+            // We pass a null IV, which means "generate me a new one".
+            // We then include the generated IV in the resulting record.
+            if let (ciphertext, iv) = self.encrypt(bytes, iv: nil) {
+                // So we have the encrypted payload. Now let's build the envelope around it.
+                let ciphertext = ciphertext.base64EncodedString
+
+                // The HMAC is computed over the base64 string. As bytes. Yes, I know.
+                if let encodedCiphertextBytes = ciphertext.data(using: String.Encoding.ascii, allowLossyConversion: false) {
+                    let hmac = self.hmacString(encodedCiphertextBytes)
+                    let iv = iv.base64EncodedString
+
+                    // The payload is stringified JSON. Yes, I know.
+                    let payload: Any = JSON(object: ["ciphertext": ciphertext, "IV": iv, "hmac": hmac]).stringValue()! as Any
+                    let obj = ["id": record.id,
+                               "sortindex": record.sortindex,
+                               // This is how SwiftyJSON wants us to express a null that we want to
+                               // serialize. Yes, this is gross.
+                               "ttl": record.ttl ?? NSNull(),
+                               "payload": payload]
+                    return JSON(object: obj)
+                }
+            }
+            return nil
+        }
+    }
+
+    open func asPair() -> [String] {
+        return [self.encKey.base64EncodedString, self.hmacKey.base64EncodedString]
+    }
+
+    open var hashValue: Int {
+        return "\(self.encKey.base64EncodedString) \(self.hmacKey.base64EncodedString)".hashValue
+    }
+}
+
+public func == (lhs: KeyBundle, rhs: KeyBundle) -> Bool {
+    return (lhs.encKey == rhs.encKey) &&
+           (lhs.hmacKey == rhs.hmacKey)
+}
+
+open class Keys: Equatable {
+    let valid: Bool
+    let defaultBundle: KeyBundle
+    var collectionKeys: [String: KeyBundle] = [String: KeyBundle]()
+
+    public init(defaultBundle: KeyBundle) {
+        self.defaultBundle = defaultBundle
+        self.valid = true
+    }
+
+    public init(payload: KeysPayload?) {
+        if let payload = payload, payload.isValid() {
+            if let keys = payload.defaultKeys {
+                self.defaultBundle = keys
+                self.collectionKeys = payload.collectionKeys
+                self.valid = true
+                return
+            }
+        }
+        self.defaultBundle = KeyBundle.invalid
+        self.valid = false
+    }
+
+    public convenience init(downloaded: EnvelopeJSON, master: KeyBundle) {
+        let f: (JSON) -> KeysPayload = { KeysPayload($0) }
+        let keysRecord = Record<KeysPayload>.fromEnvelope(downloaded, payloadFactory: master.factory(f))
+        self.init(payload: keysRecord?.payload)
+    }
+
+    open class func random() -> Keys {
+        return Keys(defaultBundle: KeyBundle.random())
+    }
+
+    open func forCollection(_ collection: String) -> KeyBundle {
+        if let bundle = collectionKeys[collection] {
+            return bundle
+        }
+        return defaultBundle
+    }
+
+    open func encrypter<T>(_ collection: String, encoder: RecordEncoder<T>) -> RecordEncrypter<T> {
+        return RecordEncrypter(bundle: forCollection(collection), encoder: encoder)
+    }
+
+    open func asPayload() -> KeysPayload {
+        let json: JSON = JSON([
+            "id": "keys",
+            "collection": "crypto",
+            "default": self.defaultBundle.asPair(),
+            "collections": mapValues(self.collectionKeys, f: { $0.asPair() })
+        ])
+        return KeysPayload(json)
+    }
+}
+
+/**
+ * Yup, these are basically typed tuples.
+ */
+public struct RecordEncoder<T: CleartextPayloadJSON> {
+    let decode: (JSON) -> T
+    let encode: (T) -> JSON
+}
+
+public struct RecordEncrypter<T: CleartextPayloadJSON> {
+    let serializer: (Record<T>) -> JSON?
+    let factory: (String) -> T?
+
+    init(bundle: KeyBundle, encoder: RecordEncoder<T>) {
+        self.serializer = bundle.serializer(encoder.encode)
+        self.factory = bundle.factory(encoder.decode)
+    }
+
+    init(serializer: @escaping (Record<T>) -> JSON?, factory: @escaping (String) -> T?) {
+        self.serializer = serializer
+        self.factory = factory
+    }
+}
+
+public func ==(lhs: Keys, rhs: Keys) -> Bool {
+    return lhs.valid == rhs.valid &&
+           lhs.defaultBundle == rhs.defaultBundle &&
+           lhs.collectionKeys == rhs.collectionKeys
+}

mobile/ios/Sync/KeysPayload.swift

@@ -0,0 +1,55 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import SwiftyJSON
+
+open class KeysPayload: CleartextPayloadJSON {
+    override open func isValid() -> Bool {
+        return super.isValid() &&
+               self["default"].isArray()
+    }
+    
+    fileprivate func keyBundleFromPair(_ input: JSON) -> KeyBundle? {
+        if let pair: [JSON] = input.array {
+            if let encKey = pair[0].string {
+                if let hmacKey = pair[1].string {
+                    return KeyBundle(encKeyB64: encKey, hmacKeyB64: hmacKey)
+                }
+            }
+        }
+        return nil
+    }
+
+    var defaultKeys: KeyBundle? {
+        return self.keyBundleFromPair(self["default"])
+    }
+
+    var collectionKeys: [String: KeyBundle] {
+        if let collections: [String: JSON] = self["collections"].dictionary {
+            return optFilter(mapValues(collections, f: self.keyBundleFromPair))
+        }
+        return [:]
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        if !(obj is KeysPayload) {
+            return false
+        }
+        
+        if !super.equalPayloads(obj) {
+            return false
+        }
+        
+        let p = obj as! KeysPayload
+        if p.defaultKeys != self.defaultKeys {
+            return false
+        }
+
+        // TODO: check collections.
+        
+        return true
+    }
+}

mobile/ios/Sync/LoginPayload.swift

@@ -0,0 +1,156 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+open class LoginPayload: CleartextPayloadJSON {
+    fileprivate static let OptionalStringFields = [
+        "formSubmitURL",
+        "httpRealm",
+    ]
+
+    fileprivate static let OptionalNumericFields = [
+        "timeLastUsed",
+        "timeCreated",
+        "timePasswordChanged",
+        "timesUsed",
+    ]
+
+    fileprivate static let RequiredStringFields = [
+        "hostname",
+        "username",
+        "password",
+        "usernameField",
+        "passwordField",
+    ]
+
+    open class func fromJSON(_ json: JSON) -> LoginPayload? {
+        let p = LoginPayload(json)
+        if p.isValid() {
+            return p
+        }
+        return nil
+    }
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if self["deleted"].bool ?? false {
+            return true
+        }
+
+        if !LoginPayload.RequiredStringFields.every({ self[$0].isString() }) {
+            return false
+        }
+
+        if !LoginPayload.OptionalStringFields.every({ field in
+            let val = self[field]
+            // Yup, 404 is not found, so this means "string or nothing".
+            let valid = val.isString() || val.isNull() || val.error?.code == 404
+            if !valid {
+                log.debug("Field \(field) is invalid: \(val)")
+            }
+            return valid
+        }) {
+            return false
+        }
+
+        if !LoginPayload.OptionalNumericFields.every({ field in
+            let val = self[field]
+            // Yup, 404 is not found, so this means "number or nothing".
+            // We only check for number because we're including timestamps as NSNumbers.
+            let valid = val.isNumber() || val.isNull() || val.error?.code == 404
+            if !valid {
+                log.debug("Field \(field) is invalid: \(val)")
+            }
+            return valid
+        }) {
+            return false
+        }
+
+        return true
+    }
+
+    open var hostname: String {
+        return self["hostname"].string!
+    }
+
+    open var username: String {
+        return self["username"].string!
+    }
+
+    open var password: String {
+        return self["password"].string!
+    }
+
+    open var usernameField: String {
+        return self["usernameField"].string!
+    }
+
+    open var passwordField: String {
+        return self["passwordField"].string!
+    }
+
+    open var formSubmitURL: String? {
+        return self["formSubmitURL"].string
+    }
+
+    open var httpRealm: String? {
+        return self["httpRealm"].string
+    }
+
+    fileprivate func timestamp(_ field: String) -> Timestamp? {
+        let json = self[field]
+        if let i = json.int64, i > 0 {
+            return Timestamp(i)
+        }
+        return nil
+    }
+
+    open var timesUsed: Int? {
+        return self["timesUsed"].int
+    }
+
+    open var timeCreated: Timestamp? {
+        return self.timestamp("timeCreated")
+    }
+
+    open var timeLastUsed: Timestamp? {
+        return self.timestamp("timeLastUsed")
+    }
+
+    open var timePasswordChanged: Timestamp? {
+        return self.timestamp("timePasswordChanged")
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        if let p = obj as? LoginPayload {
+            if !super.equalPayloads(p) {
+                return false
+            }
+
+            if p.deleted || self.deleted {
+                return self.deleted == p.deleted
+            }
+
+            // If either record is deleted, these other fields might be missing.
+            // But we just checked, so we're good to roll on.
+
+            return LoginPayload.RequiredStringFields.every({ field in
+                p[field].string == self[field].string
+            })
+
+            // TODO: optional fields.
+        }
+
+        return false
+    }
+}

mobile/ios/Sync/Record.swift

@@ -0,0 +1,98 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+
+private let log = Logger.syncLogger
+
+let ONE_YEAR_IN_SECONDS = 365 * 24 * 60 * 60
+
+/**
+ * Immutable representation for Sync records.
+ *
+ * Envelopes consist of:
+ *   Required: "id", "collection", "payload".
+ *   Optional: "modified", "sortindex", "ttl".
+ *
+ * Deletedness is a property of the payload.
+ */
+open class Record<T: CleartextPayloadJSON> {
+    open let id: String
+    open let payload: T
+
+    open let modified: Timestamp
+    open let sortindex: Int
+    open let ttl: Int?              // Seconds. Can be null, which means 'don't expire'.
+
+    // This is a hook for decryption.
+    // Right now it only parses the string. In subclasses, it'll parse the
+    // string, decrypt the contents, and return the data as a JSON object.
+    // From the docs:
+    //
+    //   payload  none  string 256k
+    //   A string containing a JSON structure encapsulating the data of the record.
+    //   This structure is defined separately for each WBO type.
+    //   Parts of the structure may be encrypted, in which case the structure
+    //   should also specify a record for decryption.
+    //
+    // @seealso EncryptedRecord.
+    open class func payloadFromPayloadString(_ envelope: EnvelopeJSON, payload: String) -> T? {
+        return T(payload)
+    }
+
+    // TODO: consider using error tuples.
+    open class func fromEnvelope(_ envelope: EnvelopeJSON, payloadFactory: (String) -> T?) -> Record<T>? {
+        if !(envelope.isValid()) {
+            log.error("Invalid envelope.")
+            return nil
+        }
+        guard let payload = payloadFactory(envelope.payload) else {
+            log.error("Unable to parse payload.")
+            return nil
+        }
+
+        if !payload.isValid() {
+            log.error("Invalid payload \(envelope.payload).")
+            return nil
+        }
+
+        return Record<T>(envelope: envelope, payload: payload)
+    }
+
+    /**
+     * Accepts an envelope and a decrypted payload.
+     * Inputs are not validated. Use `fromEnvelope` above.
+     */
+    convenience init(envelope: EnvelopeJSON, payload: T) {
+        // TODO: modified, sortindex, ttl
+        self.init(id: envelope.id, payload: payload, modified: envelope.modified, sortindex: envelope.sortindex)
+    }
+
+    init(id: GUID, payload: T, modified: Timestamp = Timestamp(time(nil)), sortindex: Int = 0, ttl: Int? = nil) {
+        self.id = id
+
+        self.payload = payload
+
+        self.modified = modified
+        self.sortindex = sortindex
+        self.ttl = ttl
+    }
+
+    func equalIdentifiers(_ rec: Record) -> Bool {
+        return rec.id == self.id
+    }
+
+    // Override me.
+    func equalPayloads(_ rec: Record) -> Bool {
+        return equalIdentifiers(rec) && rec.payload.deleted == self.payload.deleted
+    }
+
+    func equals(_ rec: Record) -> Bool {
+        return rec.sortindex == self.sortindex &&
+               rec.modified == self.modified &&
+               equalPayloads(rec)
+    }
+}

mobile/ios/Sync/RequestExtensions.swift

@@ -0,0 +1,72 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Alamofire
+import Shared
+import SwiftyJSON
+
+extension DataRequest {
+    public func responsePartialParsedJSON(_ completionHandler: @escaping (DataResponse<JSON>) -> Void) -> Self {
+        return response(responseSerializer: parsedJSONResponseSerializer(), completionHandler: completionHandler)
+    }
+
+    public func responsePartialParsedJSON(queue: DispatchQueue, completionHandler: @escaping (DataResponse<JSON>) -> Void) -> Self {
+        return response(queue: queue, responseSerializer: partialParsedJSONResponseSerializer(), completionHandler: completionHandler)
+    }
+
+    public func responseParsedJSON(_ partial: Bool, completionHandler: @escaping (DataResponse<JSON>) -> Void) -> Self {
+        return response(responseSerializer: parsedJSONResponseSerializer(), completionHandler: completionHandler)
+    }
+
+    public func responseParsedJSON(queue: DispatchQueue, completionHandler: @escaping (DataResponse<JSON>) -> Void) -> Self {
+        return response(queue: queue, responseSerializer: parsedJSONResponseSerializer(), completionHandler: completionHandler)
+    }
+}
+
+public enum JSONSerializeError: Error {
+    case noData
+    case parseError
+}
+
+private func parsedJSONResponseSerializer() -> DataResponseSerializer<JSON> {
+    return DataResponseSerializer() { (request, response, data, error) -> Alamofire.Result<JSON> in
+        guard let data = data, !data.isEmpty else {
+            return .failure(JSONSerializeError.noData)
+        }
+
+        let json = JSON(data: data)
+        if json.isError() {
+            return .failure(JSONSerializeError.parseError)
+        }
+
+        return .success(json)
+    }
+}
+
+private func partialParsedJSONResponseSerializer() -> DataResponseSerializer<JSON> {
+    return DataResponseSerializer() { (request, response, data, error) -> Alamofire.Result<JSON> in
+        guard let data = data, !data.isEmpty else {
+            return .failure(JSONSerializeError.noData)
+        }
+
+        let o: Any?
+        do {
+            try o = JSONSerialization.jsonObject(with: data, options: .allowFragments)
+        } catch {
+            return .failure(JSONSerializeError.parseError)
+        }
+
+        guard let object = o else {
+            return .failure(JSONSerializeError.noData)
+        }
+
+        let json = JSON(object)
+        if json.isError() {
+            return .failure(JSONSerializeError.parseError)
+        }
+
+        return .success(json)
+    }
+}

mobile/ios/Sync/State.swift

@@ -0,0 +1,579 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import XCGLogger
+import SwiftKeychainWrapper
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+/*
+ * This file includes types that manage intra-sync and inter-sync metadata
+ * for the use of synchronizers and the state machine.
+ *
+ * See docs/sync.md for details on what exactly we need to persist.
+ */
+
+public struct Fetched<T: Equatable>: Equatable {
+    let value: T
+    let timestamp: Timestamp
+}
+
+public func ==<T>(lhs: Fetched<T>, rhs: Fetched<T>) -> Bool {
+    return lhs.timestamp == rhs.timestamp &&
+           lhs.value == rhs.value
+}
+
+public enum LocalCommand: CustomStringConvertible, Hashable {
+    // We've seen something (a blank server, a changed global sync ID, a
+    // crypto/keys with a different meta/global) that requires us to reset all
+    // local engine timestamps (save the ones listed) and possibly re-upload.
+    case resetAllEngines(except: Set<String>)
+
+    // We've seen something (a changed engine sync ID, a crypto/keys with a
+    // different per-engine bulk key) that requires us to reset our local engine
+    // timestamp and possibly re-upload.
+    case resetEngine(engine: String)
+
+    // We've seen a change in meta/global: an engine has come or gone.
+    case enableEngine(engine: String)
+    case disableEngine(engine: String)
+
+    public func toJSON() -> JSON {
+        switch self {
+        case let .resetAllEngines(except):
+            return JSON(["type": "ResetAllEngines", "except": Array(except).sorted()])
+
+        case let .resetEngine(engine):
+            return JSON(["type": "ResetEngine", "engine": engine])
+
+        case let .enableEngine(engine):
+            return JSON(["type": "EnableEngine", "engine": engine])
+
+        case let .disableEngine(engine):
+            return JSON(["type": "DisableEngine", "engine": engine])
+        }
+    }
+
+    public static func fromJSON(_ json: JSON) -> LocalCommand? {
+        if json.isError() {
+            return nil
+        }
+        guard let type = json["type"].string else {
+            return nil
+        }
+        switch type {
+        case "ResetAllEngines":
+            if let except = json["except"].array, except.every({$0.isString()}) {
+                return .resetAllEngines(except: Set(except.map({$0.stringValue})))
+            }
+            return nil
+        case "ResetEngine":
+            if let engine = json["engine"].string {
+                return .resetEngine(engine: engine)
+            }
+            return nil
+        case "EnableEngine":
+            if let engine = json["engine"].string {
+                return .enableEngine(engine: engine)
+            }
+            return nil
+        case "DisableEngine":
+            if let engine = json["engine"].string {
+                return .disableEngine(engine: engine)
+            }
+            return nil
+        default:
+            return nil
+        }
+    }
+
+    public var description: String {
+        return self.toJSON().description
+    }
+
+    public var hashValue: Int {
+        return self.description.hashValue
+    }
+}
+
+public func ==(lhs: LocalCommand, rhs: LocalCommand) -> Bool {
+    switch (lhs, rhs) {
+    case (let .resetAllEngines(exceptL), let .resetAllEngines(exceptR)):
+        return exceptL == exceptR
+
+    case (let .resetEngine(engineL), let .resetEngine(engineR)):
+        return engineL == engineR
+
+    case (let .enableEngine(engineL), let .enableEngine(engineR)):
+        return engineL == engineR
+
+    case (let .disableEngine(engineL), let .disableEngine(engineR)):
+        return engineL == engineR
+
+    default:
+        return false
+    }
+}
+
+/*
+ * Persistence pref names.
+ * Note that syncKeyBundle isn't persisted by us.
+ *
+ * Note also that fetched keys aren't kept in prefs: we keep the timestamp ("PrefKeysTS"),
+ * and we keep a 'label'. This label is used to find the real fetched keys in the Keychain.
+ */
+
+private let PrefVersion = "_v"
+private let PrefGlobal = "global"
+private let PrefGlobalTS = "globalTS"
+private let PrefKeyLabel = "keyLabel"
+private let PrefKeysTS = "keysTS"
+private let PrefLastFetched = "lastFetched"
+private let PrefLocalCommands = "localCommands"
+private let PrefClientName = "clientName"
+private let PrefClientGUID = "clientGUID"
+private let PrefHashedUID = "hashedUID"
+private let PrefEngineConfiguration = "engineConfiguration"
+private let PrefDeviceID = "deviceID"
+
+class PrefsBackoffStorage: BackoffStorage {
+    let prefs: Prefs
+    fileprivate let key = "timestamp"
+
+    init(prefs: Prefs) {
+        self.prefs = prefs
+    }
+
+    var serverBackoffUntilLocalTimestamp: Timestamp? {
+        get {
+            return self.prefs.unsignedLongForKey(self.key)
+        }
+
+        set(value) {
+            if let value = value {
+                self.prefs.setLong(value, forKey: self.key)
+            } else {
+                self.prefs.removeObjectForKey(self.key)
+            }
+        }
+    }
+
+    func clearServerBackoff() {
+        self.prefs.removeObjectForKey(self.key)
+    }
+
+    func isInBackoff(_ now: Timestamp) -> Timestamp? {
+        if let ts = self.serverBackoffUntilLocalTimestamp, now < ts {
+            return ts
+        }
+        return nil
+    }
+}
+
+/**
+ * The scratchpad consists of the following:
+ *
+ * 1. Cached records. We cache meta/global and crypto/keys until they change.
+ * 2. Metadata like timestamps, both for cached records and for server fetches.
+ * 3. User preferences -- engine enablement.
+ * 4. Client record state.
+ * 5. Local commands that have yet to be processed.
+ *
+ * Note that the scratchpad itself is immutable, but is a class passed by reference.
+ * Its mutable fields can be mutated, but you can't accidentally e.g., switch out
+ * meta/global and get confused.
+ *
+ * TODO: the Scratchpad needs to be loaded from persistent storage, and written
+ * back at certain points in the state machine (after a replayable action is taken).
+ */
+open class Scratchpad {
+    open class Builder {
+        var syncKeyBundle: KeyBundle         // For the love of god, if you change this, invalidate keys, too!
+        fileprivate var global: Fetched<MetaGlobal>?
+        fileprivate var keys: Fetched<Keys>?
+        fileprivate var keyLabel: String
+        var localCommands: Set<LocalCommand>
+        var engineConfiguration: EngineConfiguration?
+        var clientGUID: String
+        var clientName: String
+        var fxaDeviceId: String
+        var hashedUID: String?
+        var prefs: Prefs
+
+        init(p: Scratchpad) {
+            self.syncKeyBundle = p.syncKeyBundle
+            self.prefs = p.prefs
+
+            self.global = p.global
+
+            self.keys = p.keys
+            self.keyLabel = p.keyLabel
+            self.localCommands = p.localCommands
+            self.engineConfiguration = p.engineConfiguration
+            self.clientGUID = p.clientGUID
+            self.clientName = p.clientName
+            self.fxaDeviceId = p.fxaDeviceId
+            self.hashedUID = p.hashedUID
+        }
+
+        open func clearLocalCommands() -> Builder {
+            self.localCommands.removeAll()
+            return self
+        }
+
+        open func addLocalCommandsFromKeys(_ keys: Fetched<Keys>?) -> Builder {
+            // Getting new keys can force local collection resets.
+            guard let freshKeys = keys?.value, let staleKeys = self.keys?.value, staleKeys.valid else {
+                // Removing keys, or new keys and either we didn't have old keys or they weren't valid.  Everybody gets a reset!
+                self.localCommands.insert(LocalCommand.resetAllEngines(except: []))
+                return self
+            }
+
+            // New keys, and we have valid old keys.
+            if freshKeys.defaultBundle != staleKeys.defaultBundle {
+                // Default bundle has changed.  Reset everything but collections that have unchanged bulk keys.
+                var except: Set<String> = Set()
+                // Symmetric difference, like an animal.  Swift doesn't allow Hashable tuples; don't fight it.
+                for (collection, keyBundle) in staleKeys.collectionKeys {
+                    if keyBundle == freshKeys.forCollection(collection) {
+                        except.insert(collection)
+                    }
+                }
+                for (collection, keyBundle) in freshKeys.collectionKeys {
+                    if keyBundle == staleKeys.forCollection(collection) {
+                        except.insert(collection)
+                    }
+                }
+                self.localCommands.insert(.resetAllEngines(except: except))
+            } else {
+                // Default bundle is the same.  Reset collections that have changed bulk keys.
+                for (collection, keyBundle) in staleKeys.collectionKeys {
+                    if keyBundle != freshKeys.forCollection(collection) {
+                        self.localCommands.insert(.resetEngine(engine: collection))
+                    }
+                }
+                for (collection, keyBundle) in freshKeys.collectionKeys {
+                    if keyBundle != staleKeys.forCollection(collection) {
+                        self.localCommands.insert(.resetEngine(engine: collection))
+                    }
+                }
+            }
+            return self
+        }
+
+        open func setKeys(_ keys: Fetched<Keys>?) -> Builder {
+            self.keys = keys
+            return self
+        }
+
+        open func setGlobal(_ global: Fetched<MetaGlobal>?) -> Builder {
+            self.global = global
+            if let global = global {
+                // We always take the incoming meta/global's engine configuration.
+                self.engineConfiguration = global.value.engineConfiguration()
+            }
+            return self
+        }
+
+        open func setEngineConfiguration(_ engineConfiguration: EngineConfiguration?) -> Builder {
+            self.engineConfiguration = engineConfiguration
+            return self
+        }
+
+        open func build() -> Scratchpad {
+            return Scratchpad(
+                    b: self.syncKeyBundle,
+                    m: self.global,
+                    k: self.keys,
+                    keyLabel: self.keyLabel,
+                    localCommands: self.localCommands,
+                    engines: self.engineConfiguration,
+                    clientGUID: self.clientGUID,
+                    clientName: self.clientName,
+                    fxaDeviceId: self.fxaDeviceId,
+                    hashedUID: self.hashedUID,
+                    persistingTo: self.prefs
+            )
+        }
+    }
+
+    open lazy var backoffStorage: BackoffStorage = {
+        return PrefsBackoffStorage(prefs: self.prefs.branch("backoff.storage"))
+    }()
+
+    open func evolve() -> Scratchpad.Builder {
+        return Scratchpad.Builder(p: self)
+    }
+
+    // This is never persisted.
+    let syncKeyBundle: KeyBundle
+
+    // Cached records.
+    // This cached meta/global is what we use to add or remove enabled engines. See also
+    // engineConfiguration, below.
+    // We also use it to detect when meta/global hasn't changed -- compare timestamps.
+    //
+    // Note that a Scratchpad held by a Ready state will have the current server meta/global
+    // here. That means we don't need to track syncIDs separately (which is how desktop and
+    // Android are implemented).
+    // If we don't have a meta/global, and thus we don't know syncIDs, it means we haven't
+    // synced with this server before, and we'll do a fresh sync.
+    let global: Fetched<MetaGlobal>?
+
+    // We don't store these keys (so-called "collection keys" or "bulk keys") in Prefs.
+    // Instead, we store a label, which is seeded when you first create a Scratchpad.
+    // This label is used to retrieve the real keys from your Keychain.
+    //
+    // Note that we also don't store the syncKeyBundle here. That's always created from kB,
+    // provided by the Firefox Account.
+    //
+    // Why don't we derive the label from your Sync Key? Firstly, we'd like to be able to
+    // clean up without having your key. Secondly, we don't want to accidentally load keys
+    // from the Keychain just because the Sync Key is the same -- e.g., after a node
+    // reassignment. Randomly generating a label offers all of the benefits with none of the
+    // problems, with only the cost of persisting that label alongside the rest of the state.
+    let keys: Fetched<Keys>?
+    let keyLabel: String
+
+    // Local commands.
+    var localCommands: Set<LocalCommand>
+
+    // Enablement states.
+    let engineConfiguration: EngineConfiguration?
+
+    // What's our client name?
+    let clientName: String
+    let clientGUID: String
+    let fxaDeviceId: String
+    let hashedUID: String?
+
+    var hashedDeviceID: String? {
+        guard let hashedUID = hashedUID else {
+            return nil
+        }
+        return (fxaDeviceId + hashedUID).sha256.hexEncodedString
+    }
+
+    // Where do we persist when told?
+    let prefs: Prefs
+
+    init(b: KeyBundle,
+         m: Fetched<MetaGlobal>?,
+         k: Fetched<Keys>?,
+         keyLabel: String,
+         localCommands: Set<LocalCommand>,
+         engines: EngineConfiguration?,
+         clientGUID: String,
+         clientName: String,
+         fxaDeviceId: String,
+         hashedUID: String?,
+         persistingTo prefs: Prefs
+        ) {
+        self.syncKeyBundle = b
+        self.prefs = prefs
+
+        self.keys = k
+        self.keyLabel = keyLabel
+        self.global = m
+        self.engineConfiguration = engines
+        self.localCommands = localCommands
+        self.clientGUID = clientGUID
+        self.clientName = clientName
+        self.fxaDeviceId = fxaDeviceId
+        self.hashedUID = hashedUID
+    }
+
+    // This should never be used in the end; we'll unpickle instead.
+    // This should be a convenience initializer, but... Swift compiler bug?
+    init(b: KeyBundle, persistingTo prefs: Prefs) {
+        self.syncKeyBundle = b
+        self.prefs = prefs
+
+        self.keys = nil
+        self.keyLabel = Bytes.generateGUID()
+        self.global = nil
+        self.engineConfiguration = nil
+        self.localCommands = Set()
+        self.clientGUID = Bytes.generateGUID()
+        self.clientName = DeviceInfo.defaultClientName()
+
+        self.fxaDeviceId = "unknown_fxaDeviceId"
+
+        self.hashedUID = nil
+    }
+
+    func freshStartWithGlobal(_ global: Fetched<MetaGlobal>) -> Scratchpad {
+        // TODO: I *think* a new keyLabel is unnecessary.
+        return self.evolve()
+                   .setGlobal(global)
+                   .addLocalCommandsFromKeys(nil)
+                   .setKeys(nil)
+                   .build()
+    }
+
+    fileprivate class func unpickleV1FromPrefs(_ prefs: Prefs, syncKeyBundle: KeyBundle) -> Scratchpad {
+        let b = Scratchpad(b: syncKeyBundle, persistingTo: prefs).evolve()
+
+        if let mg = prefs.stringForKey(PrefGlobal) {
+            if let mgTS = prefs.unsignedLongForKey(PrefGlobalTS) {
+                if let global = MetaGlobal.fromJSON(JSON(parseJSON: mg)) {
+                    _ = b.setGlobal(Fetched(value: global, timestamp: mgTS))
+                } else {
+                    log.error("Malformed meta/global in prefs. Ignoring.")
+                }
+            } else {
+                // This should never happen.
+                log.error("Found global in prefs, but not globalTS!")
+            }
+        }
+
+        if let keyLabel = prefs.stringForKey(PrefKeyLabel) {
+            b.keyLabel = keyLabel
+            if let ckTS = prefs.unsignedLongForKey(PrefKeysTS) {
+                let key = "keys." + keyLabel
+                KeychainWrapper.sharedAppContainerKeychain.ensureStringItemAccessibility(.afterFirstUnlock, forKey: key)
+                if let keys = KeychainWrapper.sharedAppContainerKeychain.string(forKey: key) {
+                    // We serialize as JSON.
+                    let keys = Keys(payload: KeysPayload(keys))
+                    if keys.valid {
+                        log.debug("Read keys from Keychain with label \(keyLabel).")
+                        _ = b.setKeys(Fetched(value: keys, timestamp: ckTS))
+                    } else {
+                        log.error("Invalid keys extracted from Keychain. Discarding.")
+                    }
+                } else {
+                    log.error("Found keysTS in prefs, but didn't find keys in Keychain!")
+                }
+            }
+        }
+
+        b.clientGUID = prefs.stringForKey(PrefClientGUID) ?? {
+            log.error("No value found in prefs for client GUID! Generating one.")
+            return Bytes.generateGUID()
+        }()
+
+        b.clientName = prefs.stringForKey(PrefClientName) ?? {
+            log.error("No value found in prefs for client name! Using default.")
+            return DeviceInfo.defaultClientName()
+        }()
+
+        b.hashedUID = prefs.stringForKey(PrefHashedUID)
+
+        b.fxaDeviceId = prefs.stringForKey(PrefDeviceID) ?? {
+            // Migrate from previous way of storing device id.
+            // This code will only be run once – the id will be stored
+            // in PrefDeviceID.
+            let PrefDeviceRegistration = "deviceRegistration"
+            if let string = prefs.stringForKey(PrefDeviceRegistration) {
+                let json = JSON(parseJSON: string)
+                if let id = json["id"].string {
+                    return id
+                }
+                prefs.removeObjectForKey(PrefDeviceRegistration)
+            }
+            // This is run the first time we sync with a new account.
+            // It will be replaced by a real fxaDeviceId, from account.deviceRegistration?.id.
+            log.warning("No value found in prefs for fxaDeviceId! Will overwrite on first sync")
+            return "unknown_fxaDeviceId"
+        }()
+
+        if let localCommands: [String] = prefs.stringArrayForKey(PrefLocalCommands) {
+            b.localCommands = Set(localCommands.flatMap({LocalCommand.fromJSON(JSON(parseJSON: $0))}))
+        }
+
+        if let engineConfigurationString = prefs.stringForKey(PrefEngineConfiguration) {
+            if let engineConfiguration = EngineConfiguration.fromJSON(JSON(parseJSON: engineConfigurationString)) {
+                b.engineConfiguration = engineConfiguration
+            } else {
+                log.error("Invalid engineConfiguration found in prefs. Discarding.")
+            }
+        }
+
+        return b.build()
+    }
+
+    /**
+     * Remove anything that might be left around after prefs is wiped.
+     */
+    open class func clearFromPrefs(_ prefs: Prefs) {
+        if let keyLabel = prefs.stringForKey(PrefKeyLabel) {
+            log.debug("Removing saved key from keychain.")
+            KeychainWrapper.sharedAppContainerKeychain.removeObject(forKey: keyLabel)
+        } else {
+            log.debug("No key label; nothing to remove from keychain.")
+        }
+    }
+
+    open class func restoreFromPrefs(_ prefs: Prefs, syncKeyBundle: KeyBundle) -> Scratchpad? {
+        if let ver = prefs.intForKey(PrefVersion) {
+            switch ver {
+            case 1:
+                return unpickleV1FromPrefs(prefs, syncKeyBundle: syncKeyBundle)
+            default:
+                return nil
+            }
+        }
+
+        log.debug("No scratchpad found in prefs.")
+        return nil
+    }
+
+    /**
+     * Persist our current state to our origin prefs.
+     * Note that calling this from multiple threads with either mutated or evolved
+     * scratchpads will cause sadness — individual writes are thread-safe, but the
+     * overall pseudo-transaction is not atomic.
+     */
+    open func checkpoint() -> Scratchpad {
+        return pickle(self.prefs)
+    }
+
+    func pickle(_ prefs: Prefs) -> Scratchpad {
+        prefs.setInt(1, forKey: PrefVersion)
+        if let global = global {
+            prefs.setLong(global.timestamp, forKey: PrefGlobalTS)
+            prefs.setString(global.value.asPayload().json.stringValue()!, forKey: PrefGlobal)
+        } else {
+            prefs.removeObjectForKey(PrefGlobal)
+            prefs.removeObjectForKey(PrefGlobalTS)
+        }
+
+        // We store the meat of your keys in the Keychain, using a random identifier that we persist in prefs.
+        prefs.setString(self.keyLabel, forKey: PrefKeyLabel)
+        if let keys = self.keys,
+            let payload = keys.value.asPayload().json.stringValue() {
+            let label = "keys." + self.keyLabel
+            log.debug("Storing keys in Keychain with label \(label).")
+            prefs.setString(self.keyLabel, forKey: PrefKeyLabel)
+            prefs.setLong(keys.timestamp, forKey: PrefKeysTS)
+            KeychainWrapper.sharedAppContainerKeychain.set(payload, forKey: label, withAccessibility: .afterFirstUnlock)
+        } else {
+            log.debug("Removing keys from Keychain.")
+            KeychainWrapper.sharedAppContainerKeychain.removeObject(forKey: self.keyLabel)
+        }
+
+        prefs.setString(clientName, forKey: PrefClientName)
+        prefs.setString(clientGUID, forKey: PrefClientGUID)
+
+        if let uid = hashedUID {
+            prefs.setString(uid, forKey: PrefHashedUID)
+        }
+
+        prefs.setString(fxaDeviceId, forKey: PrefDeviceID)
+
+        let localCommands: [String] = Array(self.localCommands).map({$0.toJSON().stringValue()!})
+        prefs.setObject(localCommands, forKey: PrefLocalCommands)
+
+        if let engineConfiguration = self.engineConfiguration {
+            prefs.setString(engineConfiguration.toJSON().stringValue()!, forKey: PrefEngineConfiguration)
+        } else {
+            prefs.removeObjectForKey(PrefEngineConfiguration)
+        }
+
+        return self
+    }
+}

mobile/ios/Sync/StorageClient.swift

@@ -0,0 +1,820 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Alamofire
+import Shared
+import Account
+import XCGLogger
+import Deferred
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+// Not an error that indicates a server problem, but merely an
+// error that encloses a StorageResponse.
+open class StorageResponseError<T>: MaybeErrorType, SyncPingFailureFormattable {
+    open let response: StorageResponse<T>
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .httpError
+    }
+
+    public init(_ response: StorageResponse<T>) {
+        self.response = response
+    }
+
+    open var description: String {
+        return "Error."
+    }
+}
+
+open class RequestError: MaybeErrorType, SyncPingFailureFormattable {
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .httpError
+    }
+
+    open var description: String {
+        return "Request error."
+    }
+}
+
+open class BadRequestError<T>: StorageResponseError<T> {
+    open let request: URLRequest?
+
+    public init(request: URLRequest?, response: StorageResponse<T>) {
+        self.request = request
+        super.init(response)
+    }
+
+    override open var description: String {
+        return "Bad request."
+    }
+}
+
+open class ServerError<T>: StorageResponseError<T> {
+    override open var description: String {
+        return "Server error."
+    }
+
+    override public init(_ response: StorageResponse<T>) {
+        super.init(response)
+    }
+}
+
+open class NotFound<T>: StorageResponseError<T> {
+    override open var description: String {
+        return "Not found. (\(T.self))"
+    }
+
+    override public init(_ response: StorageResponse<T>) {
+        super.init(response)
+    }
+}
+
+open class RecordParseError: MaybeErrorType, SyncPingFailureFormattable {
+    open var description: String {
+        return "Failed to parse record."
+    }
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+}
+
+open class MalformedMetaGlobalError: MaybeErrorType, SyncPingFailureFormattable {
+    open var description: String {
+        return "Supplied meta/global for upload did not serialize to valid JSON."
+    }
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+}
+
+open class RecordTooLargeError: MaybeErrorType, SyncPingFailureFormattable {
+    open let guid: GUID
+    open let size: ByteCount
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+
+    public init(size: ByteCount, guid: GUID) {
+        self.size = size
+        self.guid = guid
+    }
+
+    open var description: String {
+        return "Record \(self.guid) too large: \(size) bytes."
+    }
+}
+
+/**
+ * Raised when the storage client is refusing to make a request due to a known
+ * server backoff.
+ * If you want to bypass this, remove the backoff from the BackoffStorage that
+ * the storage client is using.
+ */
+open class ServerInBackoffError: MaybeErrorType, SyncPingFailureFormattable {
+    fileprivate let until: Timestamp
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+
+    open var description: String {
+        let formatter = DateFormatter()
+        formatter.dateStyle = DateFormatter.Style.short
+        formatter.timeStyle = DateFormatter.Style.medium
+        let s = formatter.string(from: Date.fromTimestamp(self.until))
+        return "Server in backoff until \(s)."
+    }
+
+    public init(until: Timestamp) {
+        self.until = until
+    }
+}
+
+// Returns milliseconds. Handles decimals.
+private func optionalSecondsHeader(_ input: AnyObject?) -> Timestamp? {
+    if input == nil {
+        return nil
+    }
+
+    if let val = input as? String {
+        if let timestamp = decimalSecondsStringToTimestamp(val) {
+            return timestamp
+        }
+    }
+
+    if let seconds: Double = input as? Double {
+        // Oh for a BigDecimal library.
+        return Timestamp(seconds * 1000)
+    }
+
+    if let seconds: NSNumber = input as? NSNumber {
+        // Who knows.
+        return seconds.uint64Value * 1000
+    }
+
+    return nil
+}
+
+private func optionalIntegerHeader(_ input: AnyObject?) -> Int64? {
+    if input == nil {
+        return nil
+    }
+
+    if let val = input as? String {
+        return Scanner(string: val).scanLongLong()
+    }
+
+    if let val: Double = input as? Double {
+        // Oh for a BigDecimal library.
+        return Int64(val)
+    }
+
+    if let val: NSNumber = input as? NSNumber {
+        // Who knows.
+        return val.int64Value
+    }
+
+    return nil
+}
+
+private func optionalUIntegerHeader(_ input: AnyObject?) -> Timestamp? {
+    if input == nil {
+        return nil
+    }
+
+    if let val = input as? String {
+        return Scanner(string: val).scanUnsignedLongLong()
+    }
+
+    if let val: Double = input as? Double {
+        // Oh for a BigDecimal library.
+        return Timestamp(val)
+    }
+
+    if let val: NSNumber = input as? NSNumber {
+        // Who knows.
+        return val.uint64Value
+    }
+
+    return nil
+}
+
+public enum SortOption: String {
+    case NewestFirst = "newest"
+    case OldestFirst = "oldest"
+    case Index = "index"
+}
+
+public struct ResponseMetadata {
+    public let status: Int
+    public let alert: String?
+    public let nextOffset: String?
+    public let records: UInt64?
+    public let quotaRemaining: Int64?
+    public let timestampMilliseconds: Timestamp         // Non-optional. Server timestamp when handling request.
+    public let lastModifiedMilliseconds: Timestamp?     // Included for all success responses. Collection or record timestamp.
+    public let backoffMilliseconds: UInt64?
+    public let retryAfterMilliseconds: UInt64?
+
+    public init(response: HTTPURLResponse) {
+        self.init(status: response.statusCode, headers: response.allHeaderFields)
+    }
+
+    init(status: Int, headers: [AnyHashable: Any]) {
+        self.status = status
+        alert = headers["X-Weave-Alert"] as? String
+        nextOffset = headers["X-Weave-Next-Offset"] as? String
+        records = optionalUIntegerHeader(headers["X-Weave-Records"] as AnyObject?)
+        quotaRemaining = optionalIntegerHeader(headers["X-Weave-Quota-Remaining"] as AnyObject?)
+        timestampMilliseconds = optionalSecondsHeader(headers["X-Weave-Timestamp"] as AnyObject?) ?? 0
+        lastModifiedMilliseconds = optionalSecondsHeader(headers["X-Last-Modified"] as AnyObject?)
+        backoffMilliseconds = optionalSecondsHeader(headers["X-Weave-Backoff"] as AnyObject?) ??
+                              optionalSecondsHeader(headers["X-Backoff"] as AnyObject?)
+        retryAfterMilliseconds = optionalSecondsHeader(headers["Retry-After"] as AnyObject?)
+    }
+}
+
+public struct StorageResponse<T> {
+    public let value: T
+    public let metadata: ResponseMetadata
+
+    init(value: T, metadata: ResponseMetadata) {
+        self.value = value
+        self.metadata = metadata
+    }
+
+    init(value: T, response: HTTPURLResponse) {
+        self.value = value
+        self.metadata = ResponseMetadata(response: response)
+    }
+}
+
+public typealias BatchToken = String
+
+public typealias ByteCount = Int
+
+public struct POSTResult {
+    public let success: [GUID]
+    public let failed: [GUID: String]
+    public let batchToken: BatchToken?
+
+    public init(success: [GUID], failed: [GUID: String], batchToken: BatchToken? = nil) {
+        self.success = success
+        self.failed = failed
+        self.batchToken = batchToken
+    }
+
+    public static func fromJSON(_ json: JSON) -> POSTResult? {
+        if json.isError() {
+            return nil
+        }
+
+        let batchToken = json["batch"].string
+
+        if let s = json["success"].array,
+           let f = json["failed"].dictionary {
+            var failed = false
+            let stringOrFail: (JSON) -> String = { $0.string ?? { failed = true; return "" }() }
+
+            // That's the basic structure. Now let's transform the contents.
+            let successGUIDs = s.map(stringOrFail)
+            if failed {
+                return nil
+            }
+            let failedGUIDs = mapValues(f, f: stringOrFail)
+            if failed {
+                return nil
+            }
+            return POSTResult(success: successGUIDs, failed: failedGUIDs, batchToken: batchToken)
+        }
+        return nil
+    }
+}
+
+public typealias Authorizer = (URLRequest) -> URLRequest
+
+// TODO: don't be so naïve. Use a combination of uptime and wall clock time.
+public protocol BackoffStorage {
+    var serverBackoffUntilLocalTimestamp: Timestamp? { get set }
+    func clearServerBackoff()
+    func isInBackoff(_ now: Timestamp) -> Timestamp?   // Returns 'until' for convenience.
+}
+
+// Don't forget to batch downloads.
+open class Sync15StorageClient {
+    fileprivate let authorizer: Authorizer
+    fileprivate let serverURI: URL
+
+    open static let maxRecordSizeBytes: Int = 262_140       // A shade under 256KB.
+    open static let maxPayloadSizeBytes: Int = 1_000_000    // A shade under 1MB.
+    open static let maxPayloadItemCount: Int = 100          // Bug 1250747 will raise this.
+
+    var backoff: BackoffStorage
+
+    let workQueue: DispatchQueue
+    let resultQueue: DispatchQueue
+
+    public init(token: TokenServerToken, workQueue: DispatchQueue, resultQueue: DispatchQueue, backoff: BackoffStorage) {
+        self.workQueue = workQueue
+        self.resultQueue = resultQueue
+        self.backoff = backoff
+
+        // This is a potentially dangerous assumption, but failable initializers up the stack are a giant pain.
+        // We want the serverURI to *not* have a trailing slash: to efficiently wipe a user's storage, we delete
+        // the user root (like /1.5/1234567) and not an "empty collection" (like /1.5/1234567/); the storage
+        // server treats the first like a DROP table and the latter like a DELETE *, and the former is more
+        // efficient than the latter.
+        self.serverURI = URL(string: token.api_endpoint.endsWith("/")
+            ? token.api_endpoint.substring(to: token.api_endpoint.index(before: token.api_endpoint.endIndex))
+            : token.api_endpoint)!
+        self.authorizer = {
+            (r: URLRequest) -> URLRequest in
+            var req = r
+            let helper = HawkHelper(id: token.id, key: token.key.data(using: String.Encoding.utf8, allowLossyConversion: false)!)
+            req.setValue(helper.getAuthorizationValueFor(r), forHTTPHeaderField: "Authorization")
+            return req
+        }
+    }
+
+    public init(serverURI: URL, authorizer: @escaping Authorizer, workQueue: DispatchQueue, resultQueue: DispatchQueue, backoff: BackoffStorage) {
+        self.serverURI = serverURI
+        self.authorizer = authorizer
+        self.workQueue = workQueue
+        self.resultQueue = resultQueue
+        self.backoff = backoff
+    }
+
+    func updateBackoffFromResponse<T>(_ response: StorageResponse<T>) {
+        // N.B., we would not have made this request if a backoff were set, so
+        // we can safely avoid doing the write if there's no backoff in the
+        // response.
+        // This logic will have to change if we ever invalidate that assumption.
+        if let ms = response.metadata.backoffMilliseconds ?? response.metadata.retryAfterMilliseconds {
+            log.info("Backing off for \(ms)ms.")
+            self.backoff.serverBackoffUntilLocalTimestamp = ms + Date.now()
+        }
+    }
+
+    func errorWrap<T, U>(_ deferred: Deferred<Maybe<T>>, handler: @escaping (DataResponse<U>) -> Void) -> (DataResponse<U>) -> Void {
+        return { response in
+            log.verbose("Response is \(response.response ??? "nil").")
+
+            /**
+             * Returns true if handled.
+             */
+            func failFromResponse(_ HTTPResponse: HTTPURLResponse?) -> Bool {
+                guard let HTTPResponse = HTTPResponse else {
+                    // TODO: better error.
+                    log.error("No response")
+                    let result = Maybe<T>(failure: RecordParseError())
+                    deferred.fill(result)
+                    return true
+                }
+
+                log.debug("Status code: \(HTTPResponse.statusCode).")
+
+                let storageResponse = StorageResponse(value: HTTPResponse, metadata: ResponseMetadata(response: HTTPResponse))
+
+                self.updateBackoffFromResponse(storageResponse)
+
+                if HTTPResponse.statusCode >= 500 {
+                    log.debug("ServerError.")
+                    let result = Maybe<T>(failure: ServerError(storageResponse))
+                    deferred.fill(result)
+                    return true
+                }
+
+                if HTTPResponse.statusCode == 404 {
+                    log.debug("NotFound<\(T.self)>.")
+                    let result = Maybe<T>(failure: NotFound(storageResponse))
+                    deferred.fill(result)
+                    return true
+                }
+
+                if HTTPResponse.statusCode >= 400 {
+                    log.debug("BadRequestError.")
+                    let result = Maybe<T>(failure: BadRequestError(request: response.request, response: storageResponse))
+                    deferred.fill(result)
+                    return true
+                }
+
+                return false
+            }
+
+            // Check for an error from the request processor.
+            if response.result.isFailure {
+                log.error("Response: \(response.response?.statusCode ?? 0). Got error \(response.result.error ??? "nil").")
+
+                // If we got one, we don't want to hit the response nil case above and
+                // return a RecordParseError, because a RequestError is more fitting.
+                if let response = response.response {
+                    if failFromResponse(response) {
+                        log.error("This was a failure response. Filled specific error type.")
+                        return
+                    }
+                }
+
+                log.error("Filling generic RequestError.")
+                deferred.fill(Maybe<T>(failure: RequestError()))
+                return
+            }
+
+            if failFromResponse(response.response) {
+                return
+            }
+
+            handler(response)
+        }
+    }
+
+    lazy fileprivate var alamofire: SessionManager = {
+        let ua = UserAgent.syncUserAgent
+        let configuration = URLSessionConfiguration.ephemeral
+        var defaultHeaders = SessionManager.default.session.configuration.httpAdditionalHeaders ?? [:]
+        defaultHeaders["User-Agent"] = ua
+        configuration.httpAdditionalHeaders = defaultHeaders
+        return SessionManager(configuration: configuration)
+    }()
+
+    func requestGET(_ url: URL) -> DataRequest {
+        var req = URLRequest(url: url as URL)
+        req.httpMethod = URLRequest.Method.get.rawValue
+        req.setValue("application/json", forHTTPHeaderField: "Accept")
+        let authorized: URLRequest = self.authorizer(req)
+        return alamofire.request(authorized)
+                        .validate(contentType: ["application/json"])
+    }
+
+    func requestDELETE(_ url: URL) -> DataRequest {
+        var req = URLRequest(url: url as URL)
+        req.httpMethod = URLRequest.Method.delete.rawValue
+        req.setValue("1", forHTTPHeaderField: "X-Confirm-Delete")
+        let authorized: URLRequest = self.authorizer(req)
+        return alamofire.request(authorized)
+    }
+
+    func requestWrite(_ url: URL, method: String, body: String, contentType: String, ifUnmodifiedSince: Timestamp?) -> Request {
+        var req = URLRequest(url: url as URL)
+        req.httpMethod = method
+        req.setValue(contentType, forHTTPHeaderField: "Content-Type")
+
+        if let ifUnmodifiedSince = ifUnmodifiedSince {
+            req.setValue(millisecondsToDecimalSeconds(ifUnmodifiedSince), forHTTPHeaderField: "X-If-Unmodified-Since")
+        }
+
+        req.httpBody = body.data(using: String.Encoding.utf8)!
+        let authorized: URLRequest = self.authorizer(req)
+        return alamofire.request(authorized)
+    }
+
+    func requestPUT(_ url: URL, body: JSON, ifUnmodifiedSince: Timestamp?) -> Request {
+        return self.requestWrite(url, method: URLRequest.Method.put.rawValue, body: body.stringValue()!, contentType: "application/json;charset=utf-8", ifUnmodifiedSince: ifUnmodifiedSince)
+    }
+
+    func requestPOST(_ url: URL, body: JSON, ifUnmodifiedSince: Timestamp?) -> Request {
+        return self.requestWrite(url, method: URLRequest.Method.post.rawValue, body: body.stringValue()!, contentType: "application/json;charset=utf-8", ifUnmodifiedSince: ifUnmodifiedSince)
+    }
+
+    func requestPOST(_ url: URL, body: [String], ifUnmodifiedSince: Timestamp?) -> Request {
+        let content = body.joined(separator: "\n")
+        return self.requestWrite(url, method: URLRequest.Method.post.rawValue, body: content, contentType: "application/newlines", ifUnmodifiedSince: ifUnmodifiedSince)
+    }
+
+    func requestPOST(_ url: URL, body: [JSON], ifUnmodifiedSince: Timestamp?) -> Request {
+        return self.requestPOST(url, body: body.map { $0.stringValue()! }, ifUnmodifiedSince: ifUnmodifiedSince)
+    }
+
+    /**
+     * Returns true and fills the provided Deferred if our state shows that we're in backoff.
+     * Returns false otherwise.
+     */
+    fileprivate func checkBackoff<T>(_ deferred: Deferred<Maybe<T>>) -> Bool {
+        if let until = self.backoff.isInBackoff(Date.now()) {
+            deferred.fill(Maybe<T>(failure: ServerInBackoffError(until: until)))
+            return true
+        }
+        return false
+    }
+
+    fileprivate func doOp<T>(_ op: (URL) -> DataRequest, path: String, f: @escaping (JSON) -> T?) -> Deferred<Maybe<StorageResponse<T>>> {
+
+        let deferred = Deferred<Maybe<StorageResponse<T>>>(defaultQueue: self.resultQueue)
+
+        if self.checkBackoff(deferred) {
+            return deferred
+        }
+
+        // Special case "": we want /1.5/1234567 and not /1.5/1234567/.  See note about trailing slashes above.
+        let url: URL
+        if path == "" {
+            url = self.serverURI // No trailing slash.
+        } else {
+            url = self.serverURI.appendingPathComponent(path)
+
+        }
+
+        let req = op(url)
+        let handler = self.errorWrap(deferred) { (response: DataResponse<JSON>) in
+            if let json: JSON = response.result.value {
+                if let v = f(json) {
+                    let storageResponse = StorageResponse<T>(value: v, response: response.response!)
+                    deferred.fill(Maybe(success: storageResponse))
+                } else {
+                    deferred.fill(Maybe(failure: RecordParseError()))
+                }
+                return
+            }
+
+            deferred.fill(Maybe(failure: RecordParseError()))
+        }
+
+        _ = req.responseParsedJSON(true, completionHandler: handler)
+        return deferred
+    }
+
+    // Sync storage responds with a plain timestamp to a PUT, not with a JSON body.
+    fileprivate func putResource<T>(_ path: String, body: JSON, ifUnmodifiedSince: Timestamp?, parser: @escaping (String) -> T?) -> Deferred<Maybe<StorageResponse<T>>> {
+        let url = self.serverURI.appendingPathComponent(path)
+        return self.putResource(url, body: body, ifUnmodifiedSince: ifUnmodifiedSince, parser: parser)
+    }
+
+    fileprivate func putResource<T>(_ URL: Foundation.URL, body: JSON, ifUnmodifiedSince: Timestamp?, parser: @escaping (String) -> T?) -> Deferred<Maybe<StorageResponse<T>>> {
+
+        let deferred = Deferred<Maybe<StorageResponse<T>>>(defaultQueue: self.resultQueue)
+        if self.checkBackoff(deferred) {
+            return deferred
+        }
+
+        let req = self.requestPUT(URL, body: body, ifUnmodifiedSince: ifUnmodifiedSince) as! DataRequest
+        let handler = self.errorWrap(deferred) { (response: DataResponse<String>) in
+            if let data = response.result.value {
+                if let v = parser(data) {
+                    let storageResponse = StorageResponse<T>(value: v, response: response.response!)
+                    deferred.fill(Maybe(success: storageResponse))
+                } else {
+                    deferred.fill(Maybe(failure: RecordParseError()))
+                }
+                return
+            }
+            deferred.fill(Maybe(failure: RecordParseError()))
+        }
+        req.responseString(completionHandler: handler)
+        return deferred
+    }
+
+    fileprivate func getResource<T>(_ path: String, f: @escaping (JSON) -> T?) -> Deferred<Maybe<StorageResponse<T>>> {
+        return doOp(self.requestGET, path: path, f: f)
+    }
+
+    fileprivate func deleteResource<T>(_ path: String, f: @escaping (JSON) -> T?) -> Deferred<Maybe<StorageResponse<T>>> {
+        return doOp(self.requestDELETE, path: path, f: f)
+    }
+
+    func wipeStorage() -> Deferred<Maybe<StorageResponse<JSON>>> {
+        // In Sync 1.5 it's preferred that we delete the root, not /storage.
+        return deleteResource("", f: { $0 })
+    }
+
+    func getInfoCollections() -> Deferred<Maybe<StorageResponse<InfoCollections>>> {
+        return getResource("info/collections", f: InfoCollections.fromJSON)
+    }
+
+    func getMetaGlobal() -> Deferred<Maybe<StorageResponse<MetaGlobal>>> {
+        return getResource("storage/meta/global") { json in
+            // We have an envelope.  Parse the meta/global record embedded in the 'payload' string.
+            let envelope = EnvelopeJSON(json)
+            if envelope.isValid() {
+                return MetaGlobal.fromJSON(JSON(parseJSON: envelope.payload))
+            }
+            return nil
+        }
+    }
+
+    func getCryptoKeys(_ syncKeyBundle: KeyBundle, ifUnmodifiedSince: Timestamp?) -> Deferred<Maybe<StorageResponse<Record<KeysPayload>>>> {
+        let syncKey = Keys(defaultBundle: syncKeyBundle)
+        let encoder = RecordEncoder<KeysPayload>(decode: { KeysPayload($0) }, encode: { $0.json })
+        let encrypter = syncKey.encrypter("keys", encoder: encoder)
+        let client = self.clientForCollection("crypto", encrypter: encrypter)
+        return client.get("keys")
+    }
+
+    func uploadMetaGlobal(_ metaGlobal: MetaGlobal, ifUnmodifiedSince: Timestamp?) -> Deferred<Maybe<StorageResponse<Timestamp>>> {
+        let payload = metaGlobal.asPayload()
+        if payload.json.isError() {
+            return Deferred(value: Maybe(failure: MalformedMetaGlobalError()))
+        }
+
+        let record: JSON = JSON(object: ["payload": payload.json.stringValue() ?? JSON.null as Any, "id": "global"])
+        return putResource("storage/meta/global", body: record, ifUnmodifiedSince: ifUnmodifiedSince, parser: decimalSecondsStringToTimestamp)
+    }
+
+    // The crypto/keys record is a special snowflake: it is encrypted with the Sync key bundle.  All other records are
+    // encrypted with the bulk key bundle (including possibly a per-collection bulk key) stored in crypto/keys.
+    func uploadCryptoKeys(_ keys: Keys, withSyncKeyBundle syncKeyBundle: KeyBundle, ifUnmodifiedSince: Timestamp?) -> Deferred<Maybe<StorageResponse<Timestamp>>> {
+        let syncKey = Keys(defaultBundle: syncKeyBundle)
+        let encoder = RecordEncoder<KeysPayload>(decode: { KeysPayload($0) }, encode: { $0.json })
+        let encrypter = syncKey.encrypter("keys", encoder: encoder)
+        let client = self.clientForCollection("crypto", encrypter: encrypter)
+
+        let record = Record(id: "keys", payload: keys.asPayload())
+        return client.put(record, ifUnmodifiedSince: ifUnmodifiedSince)
+    }
+
+    // It would be convenient to have the storage client manage Keys, but of course we need to use a different set of
+    // keys to fetch crypto/keys itself.  See uploadCryptoKeys.
+    func clientForCollection<T>(_ collection: String, encrypter: RecordEncrypter<T>) -> Sync15CollectionClient<T> {
+        let storage = self.serverURI.appendingPathComponent("storage", isDirectory: true)
+        return Sync15CollectionClient(client: self, serverURI: storage, collection: collection, encrypter: encrypter)
+    }
+}
+
+private let DefaultInfoConfiguration = InfoConfiguration(maxRequestBytes: 1_048_576,
+                                                         maxPostRecords: 100,
+                                                         maxPostBytes: 1_048_576,
+                                                         maxTotalRecords: 10_000,
+                                                         maxTotalBytes: 104_857_600)
+
+/**
+ * We'd love to nest this in the overall storage client, but Swift
+ * forbids the nesting of a generic class inside another class.
+ */
+open class Sync15CollectionClient<T: CleartextPayloadJSON> {
+    fileprivate let client: Sync15StorageClient
+    fileprivate let encrypter: RecordEncrypter<T>
+    fileprivate let collectionURI: URL
+    fileprivate let collectionQueue = DispatchQueue(label: "com.mozilla.sync.collectionclient", attributes: [])
+    fileprivate let infoConfig = DefaultInfoConfiguration
+
+    public init(client: Sync15StorageClient, serverURI: URL, collection: String, encrypter: RecordEncrypter<T>) {
+        self.client = client
+        self.encrypter = encrypter
+        self.collectionURI = serverURI.appendingPathComponent(collection, isDirectory: false)
+    }
+
+    var maxBatchPostRecords: Int {
+        get {
+            return infoConfig.maxPostRecords
+        }
+    }
+
+    fileprivate func uriForRecord(_ guid: String) -> URL {
+        return self.collectionURI.appendingPathComponent(guid)
+    }
+
+    open func newBatch(ifUnmodifiedSince: Timestamp? = nil, onCollectionUploaded: @escaping (POSTResult, Timestamp?) -> DeferredTimestamp) -> Sync15BatchClient<T> {
+        return Sync15BatchClient(config: infoConfig,
+                                 ifUnmodifiedSince: ifUnmodifiedSince,
+                                 serializeRecord: self.serializeRecord,
+                                 uploader: self.post,
+                                 onCollectionUploaded: onCollectionUploaded)
+    }
+
+    // Exposed so we can batch by size.
+    open func serializeRecord(_ record: Record<T>) -> String? {
+        return self.encrypter.serializer(record)?.stringValue()
+    }
+
+    open func post(_ lines: [String], ifUnmodifiedSince: Timestamp?, queryParams: [URLQueryItem]? = nil) -> Deferred<Maybe<StorageResponse<POSTResult>>> {
+        let deferred = Deferred<Maybe<StorageResponse<POSTResult>>>(defaultQueue: client.resultQueue)
+
+        if self.client.checkBackoff(deferred) {
+            return deferred
+        }
+
+        let requestURI: URL
+        if let queryParams = queryParams {
+            requestURI = self.collectionURI.withQueryParams(queryParams)
+        } else {
+            requestURI = self.collectionURI
+        }
+
+        let req = client.requestPOST(requestURI, body: lines, ifUnmodifiedSince: ifUnmodifiedSince) as! DataRequest
+        _ = req.responsePartialParsedJSON(queue: collectionQueue, completionHandler: self.client.errorWrap(deferred) { (response: DataResponse<JSON>) in
+            if let json: JSON = response.result.value,
+               let result = POSTResult.fromJSON(json) {
+                let storageResponse = StorageResponse(value: result, response: response.response!)
+                deferred.fill(Maybe(success: storageResponse))
+                return
+            } else {
+                log.warning("Couldn't parse JSON response.")
+            }
+            deferred.fill(Maybe(failure: RecordParseError()))
+        })
+
+        return deferred
+    }
+
+    open func post(_ records: [Record<T>], ifUnmodifiedSince: Timestamp?, queryParams: [URLQueryItem]? = nil) -> Deferred<Maybe<StorageResponse<POSTResult>>> {
+        // TODO: charset
+        // TODO: if any of these fail, we should do _something_. Right now we just ignore them.
+        let lines = optFilter(records.map(self.serializeRecord))
+        return self.post(lines, ifUnmodifiedSince: ifUnmodifiedSince, queryParams: queryParams)
+    }
+
+    open func put(_ record: Record<T>, ifUnmodifiedSince: Timestamp?) -> Deferred<Maybe<StorageResponse<Timestamp>>> {
+        if let body = self.encrypter.serializer(record) {
+            return self.client.putResource(uriForRecord(record.id), body: body, ifUnmodifiedSince: ifUnmodifiedSince, parser: decimalSecondsStringToTimestamp)
+        }
+        return deferMaybe(RecordParseError())
+    }
+
+    open func get(_ guid: String) -> Deferred<Maybe<StorageResponse<Record<T>>>> {
+        let deferred = Deferred<Maybe<StorageResponse<Record<T>>>>(defaultQueue: client.resultQueue)
+
+        if self.client.checkBackoff(deferred) {
+            return deferred
+        }
+
+        let req = client.requestGET(uriForRecord(guid))
+        _ = req.responsePartialParsedJSON(queue: collectionQueue, completionHandler: self.client.errorWrap(deferred) { (response: DataResponse<JSON>) in
+
+            if let json: JSON = response.result.value {
+                let envelope = EnvelopeJSON(json)
+                let record = Record<T>.fromEnvelope(envelope, payloadFactory: self.encrypter.factory)
+                if let record = record {
+                    let storageResponse = StorageResponse(value: record, response: response.response!)
+                    deferred.fill(Maybe(success: storageResponse))
+                    return
+                }
+            } else {
+                log.warning("Couldn't parse JSON response.")
+            }
+
+            deferred.fill(Maybe(failure: RecordParseError()))
+        })
+
+        return deferred
+    }
+
+    /**
+     * Unlike every other Sync client, we use the application/json format for fetching
+     * multiple requests. The others use application/newlines. We don't want to write
+     * another Serializer, and we're loading everything into memory anyway.
+     *
+     * It is the caller's responsibility to check whether the returned payloads are invalid.
+     *
+     * Only non-JSON and malformed envelopes will be dropped.
+     */
+    open func getSince(_ since: Timestamp, sort: SortOption?=nil, limit: Int?=nil, offset: String?=nil) -> Deferred<Maybe<StorageResponse<[Record<T>]>>> {
+        let deferred = Deferred<Maybe<StorageResponse<[Record<T>]>>>(defaultQueue: client.resultQueue)
+
+        // Fills the Deferred for us.
+        if self.client.checkBackoff(deferred) {
+            return deferred
+        }
+
+        var params: [URLQueryItem] = [
+            URLQueryItem(name: "full", value: "1"),
+            URLQueryItem(name: "newer", value: millisecondsToDecimalSeconds(since)),
+        ]
+
+        if let offset = offset {
+            params.append(URLQueryItem(name: "offset", value: offset))
+        }
+
+        if let limit = limit {
+            params.append(URLQueryItem(name: "limit", value: "\(limit)"))
+        }
+
+        if let sort = sort {
+            params.append(URLQueryItem(name: "sort", value: sort.rawValue))
+        }
+
+        log.debug("Issuing GET with newer = \(since), offset = \(offset ??? "nil"), sort = \(sort ??? "nil").")
+        let req = client.requestGET(self.collectionURI.withQueryParams(params))
+
+        _ = req.responsePartialParsedJSON(queue: collectionQueue, completionHandler: self.client.errorWrap(deferred) { (response: DataResponse<JSON>) in
+
+            log.verbose("Response is \(response).")
+            guard let json: JSON = response.result.value else {
+                log.warning("Non-JSON response.")
+                deferred.fill(Maybe(failure: RecordParseError()))
+                return
+            }
+
+            guard let arr = json.array else {
+                log.warning("Non-array response.")
+                deferred.fill(Maybe(failure: RecordParseError()))
+                return
+            }
+
+            func recordify(_ json: JSON) -> Record<T>? {
+                let envelope = EnvelopeJSON(json)
+                return Record<T>.fromEnvelope(envelope, payloadFactory: self.encrypter.factory)
+            }
+
+            let records = arr.flatMap(recordify)
+            let response = StorageResponse(value: records, response: response.response!)
+            deferred.fill(Maybe(success: response))
+        })
+
+        return deferred
+    }
+}

mobile/ios/Sync/Sync-Bridging-Header.h

@@ -0,0 +1,8 @@
+#ifndef Client_Sync_Bridging_Header_h
+#define Client_Sync_Bridging_Header_h
+
+#import <Foundation/Foundation.h>
+#import "Shared-Bridging-Header.h"
+#import "Storage-Bridging-Header.h"
+
+#endif

mobile/ios/Sync/SyncConstants.swift

@@ -0,0 +1,12 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+
+public struct SyncConstants {
+    // Suitable for use in dispatch_time().
+    public static let SyncDelayTriggered: Int = 3000
+    public static let SyncOnForegroundMinimumDelayMillis: UInt64 = 5 * 60 * 1000
+    public static let SyncOnForegroundAfterMillis: Int64 = 5000
+}

mobile/ios/Sync/SyncMeta.swift

@@ -0,0 +1,149 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import SwiftyJSON
+
+// Our engine choices need to persist across server changes.
+// Note that EngineConfiguration is not enough to evolve an existing meta/global:
+// a meta/global generated from this will have different syncIDs and will
+// always use this device's engine versions.
+open class EngineConfiguration: Equatable {
+    let enabled: [String]
+    let declined: [String]
+    public init(enabled: [String], declined: [String]) {
+        self.enabled = enabled
+        self.declined = declined
+    }
+
+    open class func fromJSON(_ json: JSON) -> EngineConfiguration? {
+        if json.isError() {
+            return nil
+        }
+        if let enabled = jsonsToStrings(json["enabled"].array) {
+            if let declined = jsonsToStrings(json["declined"].array) {
+                return EngineConfiguration(enabled: enabled, declined: declined)
+            }
+        }
+        return nil
+    }
+
+    open func toJSON() -> JSON {
+        let json: [String: AnyObject] = ["enabled": self.enabled as AnyObject, "declined": self.declined as AnyObject]
+        return JSON(json)
+    }
+}
+
+public func ==(lhs: EngineConfiguration, rhs: EngineConfiguration) -> Bool {
+    return Set(lhs.enabled) == Set(rhs.enabled)
+}
+
+extension EngineConfiguration: CustomStringConvertible {
+    public var description: String {
+        return "EngineConfiguration(enabled: \(self.enabled.sorted()), declined: \(self.declined.sorted()))"
+    }
+}
+
+// Equivalent to Android Sync's EngineSettings, but here
+// we use them for meta/global itself.
+public struct EngineMeta: Equatable {
+    let version: Int
+    let syncID: String
+
+    public static func fromJSON(_ json: JSON) -> EngineMeta? {
+        if let syncID = json["syncID"].string {
+            if let version = json["version"].int {
+                return EngineMeta(version: version, syncID: syncID)
+            }
+        }
+        return nil
+    }
+
+    public static func mapFromJSON(_ map: [String: JSON]?) -> [String: EngineMeta]? {
+        if let map = map {
+            return optFilter(mapValues(map, f: EngineMeta.fromJSON))
+        }
+        return nil
+    }
+
+    public func toJSON() -> JSON {
+        let json: [String: AnyObject] = ["version": self.version as AnyObject, "syncID": self.syncID as AnyObject]
+        return JSON(json)
+    }
+}
+
+public func ==(lhs: EngineMeta, rhs: EngineMeta) -> Bool {
+    return (lhs.version == rhs.version) && (lhs.syncID == rhs.syncID)
+}
+
+public struct MetaGlobal: Equatable {
+    let syncID: String
+    let storageVersion: Int
+    let engines: [String: EngineMeta]
+    let declined: [String]
+
+    // TODO: is it more useful to support partial globals?
+    // TODO: how do we return error states here?
+    public static func fromJSON(_ json: JSON) -> MetaGlobal? {
+        if json.isError() {
+            return nil
+        }
+        if let syncID = json["syncID"].string {
+            if let storageVersion = json["storageVersion"].int {
+                let engines = EngineMeta.mapFromJSON(json["engines"].dictionary) ?? [:]
+                let declined = json["declined"].array ?? []
+                return MetaGlobal(syncID: syncID,
+                                  storageVersion: storageVersion,
+                                  engines: engines,
+                                  declined: jsonsToStrings(declined) ?? [])
+            }
+        }
+        return nil
+    }
+
+    public func enginesPayload() -> JSON {
+        return JSON(mapValues(engines, f: { $0.toJSON() }))
+    }
+
+    // TODO: make a whole record JSON for this.
+    public func asPayload() -> CleartextPayloadJSON {
+        let json: JSON = JSON([
+            "syncID": self.syncID,
+            "storageVersion": self.storageVersion,
+            "engines": enginesPayload().dictionaryObject as Any,
+            "declined": self.declined
+        ])
+        return CleartextPayloadJSON(json)
+    }
+
+    public func withSyncID(_ syncID: String) -> MetaGlobal {
+        return MetaGlobal(syncID: syncID, storageVersion: self.storageVersion, engines: self.engines, declined: self.declined)
+    }
+
+    public func engineConfiguration() -> EngineConfiguration {
+        return EngineConfiguration(enabled: Array(engines.keys), declined: declined)
+    }
+}
+
+public func ==(lhs: MetaGlobal, rhs: MetaGlobal) -> Bool {
+    return (lhs.syncID == rhs.syncID) &&
+           (lhs.storageVersion == rhs.storageVersion) &&
+           optArrayEqual(lhs.declined, rhs: rhs.declined) &&
+           optDictionaryEqual(lhs.engines, rhs: rhs.engines)
+}
+
+/**
+ * Encapsulates a meta/global, identity-derived keys, and keys.
+ */
+open class SyncMeta {
+    let syncKey: KeyBundle
+
+    var keys: Keys?
+    var global: MetaGlobal?
+
+    public init(syncKey: KeyBundle) {
+        self.syncKey = syncKey
+    }
+}

mobile/ios/Sync/SyncStateMachine.swift

@@ -0,0 +1,966 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Account
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+private let StorageVersionCurrent = 5
+
+// Names of collections for which a synchronizer is implemented locally.
+private let LocalEngines: [String] = [
+    "bookmarks",
+    "clients",
+    "history",
+    "passwords",
+    "tabs",
+]
+
+// Names of collections which will appear in a default meta/global produced locally.
+// Map collection name to engine version.  See http://docs.services.mozilla.com/sync/objectformats.html.
+private let DefaultEngines: [String: Int] = [
+    "bookmarks": BookmarksStorageVersion,
+    "clients": ClientsStorageVersion,
+    "history": HistoryStorageVersion,
+    "passwords": PasswordsStorageVersion,
+    "tabs": TabsStorageVersion,
+    // We opt-in to syncing collections we don't know about, since no client offers to sync non-enabled,
+    // non-declined engines yet.  See Bug 969669.
+    "forms": 1,
+    "addons": 1,
+    "prefs": 2,
+]
+
+// Names of collections which will appear as declined in a default
+// meta/global produced locally.
+private let DefaultDeclined: [String] = [String]()
+
+// public for testing.
+public func createMetaGlobalWithEngineConfiguration(_ engineConfiguration: EngineConfiguration) -> MetaGlobal {
+    var engines: [String: EngineMeta] = [:]
+    for engine in engineConfiguration.enabled {
+        // We take this device's version, or, if we don't know the correct version, 0.  Another client should recognize
+        // the engine, see an old version, wipe and start again.
+        // TODO: this client does not yet do this wipe-and-update itself!
+        let version = DefaultEngines[engine] ?? 0
+        engines[engine] = EngineMeta(version: version, syncID: Bytes.generateGUID())
+    }
+    return MetaGlobal(syncID: Bytes.generateGUID(), storageVersion: StorageVersionCurrent, engines: engines, declined: engineConfiguration.declined)
+}
+
+public func createMetaGlobal() -> MetaGlobal {
+    let engineConfiguration = EngineConfiguration(enabled: Array(DefaultEngines.keys), declined: DefaultDeclined)
+    return createMetaGlobalWithEngineConfiguration(engineConfiguration)
+}
+
+public typealias TokenSource = () -> Deferred<Maybe<TokenServerToken>>
+public typealias ReadyDeferred = Deferred<Maybe<Ready>>
+
+// See docs in docs/sync.md.
+
+// You might be wondering why this doesn't have a Sync15StorageClient like FxALoginStateMachine
+// does. Well, such a client is pinned to a particular server, and this state machine must
+// acknowledge that a Sync client occasionally must migrate between two servers, preserving
+// some state from the last.
+// The resultant 'Ready' will be able to provide a suitably initialized storage client.
+open class SyncStateMachine {
+    // The keys are used as a set, to prevent cycles in the state machine.
+    var stateLabelsSeen = [SyncStateLabel: Bool]()
+    var stateLabelSequence = [SyncStateLabel]()
+
+    let stateLabelsAllowed: Set<SyncStateLabel>
+
+    let scratchpadPrefs: Prefs
+
+    /// Use this set of states to constrain the state machine to attempt the barest 
+    /// minimum to get to Ready. This is suitable for extension uses. If it is not possible,
+    /// then no destructive or expensive actions are taken (e.g. total HTTP requests, 
+    /// duration, records processed, database writes, fsyncs, blanking any local collections)
+    public static let OptimisticStates = Set(SyncStateLabel.optimisticValues)
+
+    /// The default set of states that the state machine is allowed to use.
+    public static let AllStates = Set(SyncStateLabel.allValues)
+
+    public init(prefs: Prefs, allowingStates labels: Set<SyncStateLabel> = SyncStateMachine.AllStates) {
+        self.scratchpadPrefs = prefs.branch("scratchpad")
+        self.stateLabelsAllowed = labels
+    }
+
+    open class func clearStateFromPrefs(_ prefs: Prefs) {
+        log.debug("Clearing all Sync prefs.")
+        Scratchpad.clearFromPrefs(prefs.branch("scratchpad")) // XXX this is convoluted.
+        prefs.clearAll()
+    }
+
+    fileprivate func advanceFromState(_ state: SyncState) -> ReadyDeferred {
+        log.info("advanceFromState: \(state.label)")
+
+        // Record visibility before taking any action.
+        let labelAlreadySeen = self.stateLabelsSeen.updateValue(true, forKey: state.label) != nil
+        stateLabelSequence.append(state.label)
+
+        if let ready = state as? Ready {
+            // Sweet, we made it!
+            return deferMaybe(ready)
+        }
+
+        // Cycles are not necessarily a problem, but seeing the same (recoverable) error condition is a problem.
+        if state is RecoverableSyncState && labelAlreadySeen {
+            return deferMaybe(StateMachineCycleError())
+        }
+
+        guard stateLabelsAllowed.contains(state.label) else {
+            return deferMaybe(DisallowedStateError(state.label, allowedStates: stateLabelsAllowed))
+        }
+
+        return state.advance() >>== self.advanceFromState
+    }
+
+    open func toReady(_ authState: SyncAuthState) -> ReadyDeferred {
+        let token = authState.token(Date.now(), canBeExpired: false)
+        return chainDeferred(token, f: { (token, kB) in
+            log.debug("Got token from auth state.")
+            if Logger.logPII {
+                log.debug("Server is \(token.api_endpoint).")
+            }
+            let prior = Scratchpad.restoreFromPrefs(self.scratchpadPrefs, syncKeyBundle: KeyBundle.fromKB(kB))
+            if prior == nil {
+                log.info("No persisted Sync state. Starting over.")
+            }
+            var scratchpad = prior ?? Scratchpad(b: KeyBundle.fromKB(kB), persistingTo: self.scratchpadPrefs)
+
+            // Take the scratchpad and add the fxaDeviceId from the state, and hashedUID from the token
+            let b = Scratchpad.Builder(p: scratchpad)
+            if let deviceID = authState.deviceID {
+                b.fxaDeviceId = deviceID
+            } else {
+                // Either deviceRegistration hasn't occurred yet (our bug) or
+                // FxA has given us an UnknownDevice error.
+                log.warning("Device registration has not taken place before sync.")
+            }
+            b.hashedUID = token.hashedFxAUID
+
+            // Detect if we've changed anything in our client record from the last time we synced…
+            let ourClientUnchanged = (b.fxaDeviceId == scratchpad.fxaDeviceId)
+
+            // …and if so, trigger a reset of clients.
+            if !ourClientUnchanged {
+                b.localCommands.insert(LocalCommand.resetEngine(engine: "clients"))
+            }
+
+            scratchpad = b.build()
+
+            log.info("Advancing to InitialWithLiveToken.")
+            let state = InitialWithLiveToken(scratchpad: scratchpad, token: token)
+
+            // Start with fresh visibility data.
+            self.stateLabelsSeen = [:]
+            self.stateLabelSequence = []
+
+            return self.advanceFromState(state)
+        })
+    }
+}
+
+public enum SyncStateLabel: String {
+    case Stub = "STUB"     // For 'abstract' base classes.
+
+    case InitialWithExpiredToken = "initialWithExpiredToken"
+    case InitialWithExpiredTokenAndInfo = "initialWithExpiredTokenAndInfo"
+    case InitialWithLiveToken = "initialWithLiveToken"
+    case InitialWithLiveTokenAndInfo = "initialWithLiveTokenAndInfo"
+    case ResolveMetaGlobalVersion = "resolveMetaGlobalVersion"
+    case ResolveMetaGlobalContent = "resolveMetaGlobalContent"
+    case NeedsFreshMetaGlobal = "needsFreshMetaGlobal"
+    case NewMetaGlobal = "newMetaGlobal"
+    case HasMetaGlobal = "hasMetaGlobal"
+    case NeedsFreshCryptoKeys = "needsFreshCryptoKeys"
+    case HasFreshCryptoKeys = "hasFreshCryptoKeys"
+    case Ready = "ready"
+    case FreshStartRequired = "freshStartRequired"                                  // Go around again... once only, perhaps.
+    case ServerConfigurationRequired = "serverConfigurationRequired"
+
+    case ChangedServer = "changedServer"
+    case MissingMetaGlobal = "missingMetaGlobal"
+    case MissingCryptoKeys = "missingCryptoKeys"
+    case MalformedCryptoKeys = "malformedCryptoKeys"
+    case SyncIDChanged = "syncIDChanged"
+    case RemoteUpgradeRequired = "remoteUpgradeRequired"
+    case ClientUpgradeRequired = "clientUpgradeRequired"
+
+    static let allValues: [SyncStateLabel] = [
+        InitialWithExpiredToken,
+        InitialWithExpiredTokenAndInfo,
+        InitialWithLiveToken,
+        InitialWithLiveTokenAndInfo,
+        NeedsFreshMetaGlobal,
+        ResolveMetaGlobalVersion,
+        ResolveMetaGlobalContent,
+        NewMetaGlobal,
+        HasMetaGlobal,
+        NeedsFreshCryptoKeys,
+        HasFreshCryptoKeys,
+        Ready,
+
+        FreshStartRequired,
+        ServerConfigurationRequired,
+
+        ChangedServer,
+        MissingMetaGlobal,
+        MissingCryptoKeys,
+        MalformedCryptoKeys,
+        SyncIDChanged,
+        RemoteUpgradeRequired,
+        ClientUpgradeRequired,
+    ]
+
+    // This is the list of states needed to get to Ready, or failing.
+    // This is useful in circumstances where it is important to conserve time and/or battery, and failure 
+    // to timely sync is acceptable.
+    static let optimisticValues: [SyncStateLabel] = [
+        InitialWithLiveToken,
+        InitialWithLiveTokenAndInfo,
+        HasMetaGlobal,
+        HasFreshCryptoKeys,
+        Ready,
+    ]
+}
+
+/**
+ * States in this state machine all implement SyncState.
+ *
+ * States are either successful main-flow states, or (recoverable) error states.
+ * Errors that aren't recoverable are simply errors.
+ * Main-flow states flow one to one.
+ *
+ * (Terminal failure states might be introduced at some point.)
+ *
+ * Multiple error states (but typically only one) can arise from each main state transition.
+ * For example, parsing meta/global can result in a number of different non-routine situations.
+ *
+ * For these reasons, and the lack of useful ADTs in Swift, we model the main flow as
+ * the success branch of a Result, and the recovery flows as a part of the failure branch.
+ *
+ * We could just as easily use a ternary Either-style operator, but thanks to Swift's
+ * optional-cast-let it's no saving to do so.
+ *
+ * Because of the lack of type system support, all RecoverableSyncStates must have the same
+ * signature. That signature implies a possibly multi-state transition; individual states
+ * will have richer type signatures.
+ */
+public protocol SyncState {
+    var label: SyncStateLabel { get }
+
+    func advance() -> Deferred<Maybe<SyncState>>
+}
+
+/*
+ * Base classes to avoid repeating initializers all over the place.
+ */
+open class BaseSyncState: SyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.Stub }
+
+    open let client: Sync15StorageClient!
+    let token: TokenServerToken    // Maybe expired.
+    var scratchpad: Scratchpad
+
+    // TODO: 304 for i/c.
+    open func getInfoCollections() -> Deferred<Maybe<InfoCollections>> {
+        return chain(self.client.getInfoCollections(), f: {
+            return $0.value
+        })
+    }
+
+    public init(client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken) {
+        self.scratchpad = scratchpad
+        self.token = token
+        self.client = client
+        log.info("Inited \(self.label.rawValue)")
+    }
+
+    open func synchronizer<T: Synchronizer>(_ synchronizerClass: T.Type, delegate: SyncDelegate, prefs: Prefs, why: SyncReason) -> T {
+        return T(scratchpad: self.scratchpad, delegate: delegate, basePrefs: prefs, why: why)
+    }
+
+    // This isn't a convenience initializer 'cos subclasses can't call convenience initializers.
+    public init(scratchpad: Scratchpad, token: TokenServerToken) {
+        let workQueue = DispatchQueue.global()
+        let resultQueue = DispatchQueue.main
+        let backoff = scratchpad.backoffStorage
+        let client = Sync15StorageClient(token: token, workQueue: workQueue, resultQueue: resultQueue, backoff: backoff)
+        self.scratchpad = scratchpad
+        self.token = token
+        self.client = client
+        log.info("Inited \(self.label.rawValue)")
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        return deferMaybe(StubStateError())
+    }
+}
+
+open class BaseSyncStateWithInfo: BaseSyncState {
+    open let info: InfoCollections
+
+    init(client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections) {
+        self.info = info
+        super.init(client: client, scratchpad: scratchpad, token: token)
+    }
+
+    init(scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections) {
+        self.info = info
+        super.init(scratchpad: scratchpad, token: token)
+    }
+}
+
+/*
+ * Error types.
+ */
+public protocol SyncError: MaybeErrorType, SyncPingFailureFormattable {}
+
+extension SyncError {
+    public var failureReasonName: SyncPingFailureReasonName {
+        return .unexpectedError
+    }
+}
+
+open class UnknownError: SyncError {
+    open var description: String {
+        return "Unknown error."
+    }
+}
+
+open class StateMachineCycleError: SyncError {
+    open var description: String {
+        return "The Sync state machine encountered a cycle. This is a coding error."
+    }
+}
+
+open class CouldNotFetchMetaGlobalError: SyncError {
+    open var description: String {
+        return "Could not fetch meta/global."
+    }
+}
+
+open class CouldNotFetchKeysError: SyncError {
+    open var description: String {
+        return "Could not fetch crypto/keys."
+    }
+}
+
+open class StubStateError: SyncError {
+    open var description: String {
+        return "Unexpectedly reached a stub state. This is a coding error."
+    }
+}
+
+open class ClientUpgradeRequiredError: SyncError {
+    let targetStorageVersion: Int
+
+    public init(target: Int) {
+        self.targetStorageVersion = target
+    }
+
+    open var description: String {
+        return "Client upgrade required to work with storage version \(self.targetStorageVersion)."
+    }
+}
+
+open class InvalidKeysError: SyncError {
+    let keys: Keys
+
+    public init(_ keys: Keys) {
+        self.keys = keys
+    }
+
+    open var description: String {
+        return "Downloaded crypto/keys, but couldn't parse them."
+    }
+}
+
+open class DisallowedStateError: SyncError {
+    let state: SyncStateLabel
+    let allowedStates: Set<SyncStateLabel>
+
+    public init(_ state: SyncStateLabel, allowedStates: Set<SyncStateLabel>) {
+        self.state = state
+        self.allowedStates = allowedStates
+    }
+
+    open var description: String {
+        return "Sync state machine reached \(String(describing: state)) state, which is disallowed. Legal states are: \(String(describing: allowedStates))"
+    }
+}
+
+/**
+ * Error states. These are errors that can be recovered from by taking actions.  We use RecoverableSyncState as a
+ * sentinel: if we see the same recoverable state twice, we bail out and complain that we've seen a cycle.  (Seeing
+ * some states -- principally initial states -- twice is fine.)
+*/
+
+public protocol RecoverableSyncState: SyncState {
+}
+
+/**
+ * Recovery: discard our local timestamps and sync states; discard caches.
+ * Be prepared to handle a conflict between our selected engines and the new
+ * server's meta/global; if an engine is selected locally but not declined
+ * remotely, then we'll need to upload a new meta/global and sync that engine.
+ */
+open class ChangedServerError: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.ChangedServer }
+
+    let newToken: TokenServerToken
+    let newScratchpad: Scratchpad
+
+    public init(scratchpad: Scratchpad, token: TokenServerToken) {
+        self.newToken = token
+        self.newScratchpad = Scratchpad(b: scratchpad.syncKeyBundle, persistingTo: scratchpad.prefs)
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        // TODO: mutate local storage to allow for a fresh start.
+        let state = InitialWithLiveToken(scratchpad: newScratchpad.checkpoint(), token: newToken)
+        return deferMaybe(state)
+    }
+}
+
+/**
+ * Recovery: same as for changed server, but no need to upload a new meta/global.
+ */
+open class SyncIDChangedError: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.SyncIDChanged }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+    fileprivate let newMetaGlobal: Fetched<MetaGlobal>
+
+    public init(previousState: BaseSyncStateWithInfo, newMetaGlobal: Fetched<MetaGlobal>) {
+        self.previousState = previousState
+        self.newMetaGlobal = newMetaGlobal
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        // TODO: mutate local storage to allow for a fresh start.
+        let s = self.previousState.scratchpad.evolve().setGlobal(self.newMetaGlobal).setKeys(nil).build().checkpoint()
+        let state = HasMetaGlobal(client: self.previousState.client, scratchpad: s, token: self.previousState.token, info: self.previousState.info)
+        return deferMaybe(state)
+    }
+}
+
+/**
+ * Recovery: configure the server.
+ */
+open class ServerConfigurationRequiredError: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.ServerConfigurationRequired }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+
+    public init(previousState: BaseSyncStateWithInfo) {
+        self.previousState = previousState
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        let client = self.previousState.client!
+        let s = self.previousState.scratchpad.evolve()
+                .setGlobal(nil)
+                .addLocalCommandsFromKeys(nil)
+                .setKeys(nil)
+                .build().checkpoint()
+        // Upload a new meta/global ...
+        let metaGlobal: MetaGlobal
+        if let oldEngineConfiguration = s.engineConfiguration {
+            metaGlobal = createMetaGlobalWithEngineConfiguration(oldEngineConfiguration)
+        } else {
+            metaGlobal = createMetaGlobal()
+        }
+        return client.uploadMetaGlobal(metaGlobal, ifUnmodifiedSince: nil)
+            // ... and a new crypto/keys.
+            >>> { return client.uploadCryptoKeys(Keys.random(), withSyncKeyBundle: s.syncKeyBundle, ifUnmodifiedSince: nil) }
+            >>> { return deferMaybe(InitialWithLiveToken(client: client, scratchpad: s, token: self.previousState.token)) }
+    }
+}
+
+/**
+ * Recovery: wipe the server (perhaps unnecessarily) and proceed to configure the server.
+ */
+open class FreshStartRequiredError: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.FreshStartRequired }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+
+    public init(previousState: BaseSyncStateWithInfo) {
+        self.previousState = previousState
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        let client = self.previousState.client!
+        return client.wipeStorage()
+            >>> { return deferMaybe(ServerConfigurationRequiredError(previousState: self.previousState)) }
+    }
+}
+
+open class MissingMetaGlobalError: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.MissingMetaGlobal }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+
+    public init(previousState: BaseSyncStateWithInfo) {
+        self.previousState = previousState
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        return deferMaybe(FreshStartRequiredError(previousState: self.previousState))
+    }
+}
+
+open class MissingCryptoKeysError: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.MissingCryptoKeys }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+
+    public init(previousState: BaseSyncStateWithInfo) {
+        self.previousState = previousState
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        return deferMaybe(FreshStartRequiredError(previousState: self.previousState))
+    }
+}
+
+open class RemoteUpgradeRequired: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.RemoteUpgradeRequired }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+
+    public init(previousState: BaseSyncStateWithInfo) {
+        self.previousState = previousState
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        return deferMaybe(FreshStartRequiredError(previousState: self.previousState))
+    }
+}
+
+open class ClientUpgradeRequired: RecoverableSyncState {
+    open var label: SyncStateLabel { return SyncStateLabel.ClientUpgradeRequired }
+
+    fileprivate let previousState: BaseSyncStateWithInfo
+    let targetStorageVersion: Int
+
+    public init(previousState: BaseSyncStateWithInfo, target: Int) {
+        self.previousState = previousState
+        self.targetStorageVersion = target
+    }
+
+    open func advance() -> Deferred<Maybe<SyncState>> {
+        return deferMaybe(ClientUpgradeRequiredError(target: self.targetStorageVersion))
+    }
+}
+
+/*
+ * Non-error states.
+ */
+
+open class InitialWithLiveToken: BaseSyncState {
+    open override var label: SyncStateLabel { return SyncStateLabel.InitialWithLiveToken }
+
+    // This looks totally redundant, but try taking it out, I dare you.
+    public override init(scratchpad: Scratchpad, token: TokenServerToken) {
+        super.init(scratchpad: scratchpad, token: token)
+    }
+
+    // This looks totally redundant, but try taking it out, I dare you.
+    public override init(client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken) {
+        super.init(client: client, scratchpad: scratchpad, token: token)
+    }
+
+    func advanceWithInfo(_ info: InfoCollections) -> SyncState {
+        return InitialWithLiveTokenAndInfo(scratchpad: self.scratchpad, token: self.token, info: info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        return chain(getInfoCollections(), f: self.advanceWithInfo)
+    }
+}
+
+/**
+ * Each time we fetch a new meta/global, we need to reconcile it with our
+ * current state.
+ *
+ * It might be identical to our current meta/global, in which case we can short-circuit.
+ *
+ * We might have no previous meta/global at all, in which case this state
+ * simply configures local storage to be ready to sync according to the
+ * supplied meta/global. (Not necessarily datatype elections: those will be per-device.)
+ *
+ * Or it might be different. In this case the previous m/g and our local user preferences
+ * are compared to the new, resulting in some actions and a final state.
+ *
+ * This states are similar in purpose to GlobalSession.processMetaGlobal in Android Sync.
+ */
+
+open class ResolveMetaGlobalVersion: BaseSyncStateWithInfo {
+    let fetched: Fetched<MetaGlobal>
+
+    init(fetched: Fetched<MetaGlobal>, client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections) {
+        self.fetched = fetched
+        super.init(client: client, scratchpad: scratchpad, token: token, info: info)
+    }
+    open override var label: SyncStateLabel { return SyncStateLabel.ResolveMetaGlobalVersion }
+
+    class func fromState(_ state: BaseSyncStateWithInfo, fetched: Fetched<MetaGlobal>) -> ResolveMetaGlobalVersion {
+        return ResolveMetaGlobalVersion(fetched: fetched, client: state.client, scratchpad: state.scratchpad, token: state.token, info: state.info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        // First: check storage version.
+        let v = fetched.value.storageVersion
+        if v > StorageVersionCurrent {
+            // New storage version?  Uh-oh.  No recovery possible here.
+            log.info("Client upgrade required for storage version \(v)")
+            return deferMaybe(ClientUpgradeRequired(previousState: self, target: v))
+        }
+
+        if v < StorageVersionCurrent {
+            // Old storage version?  Uh-oh.  Wipe and upload both meta/global and crypto/keys.
+            log.info("Server storage version \(v) is outdated.")
+            return deferMaybe(RemoteUpgradeRequired(previousState: self))
+        }
+
+        return deferMaybe(ResolveMetaGlobalContent.fromState(self, fetched: self.fetched))
+    }
+}
+
+open class ResolveMetaGlobalContent: BaseSyncStateWithInfo {
+    let fetched: Fetched<MetaGlobal>
+
+    init(fetched: Fetched<MetaGlobal>, client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections) {
+        self.fetched = fetched
+        super.init(client: client, scratchpad: scratchpad, token: token, info: info)
+    }
+    open override var label: SyncStateLabel { return SyncStateLabel.ResolveMetaGlobalContent }
+
+    class func fromState(_ state: BaseSyncStateWithInfo, fetched: Fetched<MetaGlobal>) -> ResolveMetaGlobalContent {
+        return ResolveMetaGlobalContent(fetched: fetched, client: state.client, scratchpad: state.scratchpad, token: state.token, info: state.info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        // Check global syncID and contents.
+        if let previous = self.scratchpad.global?.value {
+            // Do checks that only apply when we're coming from a previous meta/global.
+            if previous.syncID != fetched.value.syncID {
+                log.info("Remote global sync ID has changed. Dropping keys and resetting all local collections.")
+                let s = self.scratchpad.freshStartWithGlobal(fetched).checkpoint()
+                return deferMaybe(HasMetaGlobal.fromState(self, scratchpad: s))
+            }
+
+            let b = self.scratchpad.evolve()
+                .setGlobal(fetched) // We always adopt the upstream meta/global record.
+
+            let previousEngines = Set(previous.engines.keys)
+            let remoteEngines = Set(fetched.value.engines.keys)
+
+            for engine in previousEngines.subtracting(remoteEngines) {
+                log.info("Remote meta/global disabled previously enabled engine \(engine).")
+                b.localCommands.insert(.disableEngine(engine: engine))
+            }
+
+            for engine in remoteEngines.subtracting(previousEngines) {
+                log.info("Remote meta/global enabled previously disabled engine \(engine).")
+                b.localCommands.insert(.enableEngine(engine: engine))
+            }
+
+            for engine in remoteEngines.intersection(previousEngines) {
+                let remoteEngine = fetched.value.engines[engine]!
+                let previousEngine = previous.engines[engine]!
+                if previousEngine.syncID != remoteEngine.syncID {
+                    log.info("Remote sync ID for \(engine) has changed. Resetting local.")
+                    b.localCommands.insert(.resetEngine(engine: engine))
+                }
+            }
+
+            let s = b.build().checkpoint()
+            return deferMaybe(HasMetaGlobal.fromState(self, scratchpad: s))
+        }
+
+        // No previous meta/global. Adopt the new meta/global.
+        let s = self.scratchpad.freshStartWithGlobal(fetched).checkpoint()
+        return deferMaybe(HasMetaGlobal.fromState(self, scratchpad: s))
+    }
+}
+
+private func processFailure(_ failure: MaybeErrorType?) -> MaybeErrorType {
+    if let failure = failure as? ServerInBackoffError {
+        log.warning("Server in backoff. Bailing out. \(failure.description)")
+        return failure
+    }
+
+    // TODO: backoff etc. for all of these.
+    if let failure = failure as? ServerError<HTTPURLResponse> {
+        // Be passive.
+        log.error("Server error. Bailing out. \(failure.description)")
+        return failure
+    }
+
+    if let failure = failure as? BadRequestError<HTTPURLResponse> {
+        // Uh oh.
+        log.error("Bad request. Bailing out. \(failure.description)")
+        return failure
+    }
+
+    log.error("Unexpected failure. \(failure?.description ?? "nil")")
+    return failure ?? UnknownError()
+}
+
+open class InitialWithLiveTokenAndInfo: BaseSyncStateWithInfo {
+    open override var label: SyncStateLabel { return SyncStateLabel.InitialWithLiveTokenAndInfo }
+
+    // This method basically hops over HasMetaGlobal, because it's not a state
+    // that we expect consumers to know about.
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        // Either m/g and c/k are in our local cache, and they're up-to-date with i/c,
+        // or we need to fetch them.
+        // Cached and not changed in i/c? Use that.
+        // This check would be inaccurate if any other fields were stored in meta/; this
+        // has been the case in the past, with the Sync 1.1 migration indicator.
+        if let global = self.scratchpad.global {
+            if let metaModified = self.info.modified("meta") {
+                // We check the last time we fetched the record, and that can be
+                // later than the collection timestamp. All we care about here is if the
+                // server might have a newer record.
+                if global.timestamp >= metaModified {
+                    log.debug("Cached meta/global fetched at \(global.timestamp), newer than server modified \(metaModified). Using cached meta/global.")
+                    // Strictly speaking we can avoid fetching if this condition is not true,
+                    // but if meta/ is modified for a different reason -- store timestamps
+                    // for the last collection fetch. This will do for now.
+                    return deferMaybe(HasMetaGlobal.fromState(self))
+                }
+                log.info("Cached meta/global fetched at \(global.timestamp) older than server modified \(metaModified). Fetching fresh meta/global.")
+            } else {
+                // No known modified time for meta/. That means the server has no meta/global.
+                // Drop our cached value and fall through; we'll try to fetch, fail, and
+                // go through the usual failure flow.
+                log.warning("Local meta/global fetched at \(global.timestamp) found, but no meta collection on server. Dropping cached meta/global.")
+                // If we bail because we've been overly optimistic, then we nil out the current (broken)
+                // meta/global. Next time around, we end up in the "No cached meta/global found" branch.
+                self.scratchpad = self.scratchpad.evolve().setGlobal(nil).setKeys(nil).build().checkpoint()
+            }
+        } else {
+            log.debug("No cached meta/global found. Fetching fresh meta/global.")
+        }
+
+        return deferMaybe(NeedsFreshMetaGlobal.fromState(self))
+    }
+}
+
+/*
+ * We've reached NeedsFreshMetaGlobal somehow, but we haven't yet done anything about it
+ * (e.g. fetch a new one with GET /storage/meta/global ).
+ *
+ * If we don't want to hit the network (e.g. from an extension), we should stop if we get to this state.
+ */
+open class NeedsFreshMetaGlobal: BaseSyncStateWithInfo {
+    open override var label: SyncStateLabel { return SyncStateLabel.NeedsFreshMetaGlobal }
+
+    class func fromState(_ state: BaseSyncStateWithInfo) -> NeedsFreshMetaGlobal {
+        return NeedsFreshMetaGlobal(client: state.client, scratchpad: state.scratchpad, token: state.token, info: state.info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        // Fetch.
+        return self.client.getMetaGlobal().bind { result in
+            if let resp = result.successValue {
+                // We use the server's timestamp, rather than the record's modified field.
+                // Either can be made to work, but the latter has suffered from bugs: see Bug 1210625.
+                let fetched = Fetched(value: resp.value, timestamp: resp.metadata.timestampMilliseconds)
+                return deferMaybe(ResolveMetaGlobalVersion.fromState(self, fetched: fetched))
+            }
+
+            if let _ = result.failureValue as? NotFound<HTTPURLResponse> {
+                // OK, this is easy.
+                // This state is responsible for creating the new m/g, uploading it, and
+                // restarting with a clean scratchpad.
+                return deferMaybe(MissingMetaGlobalError(previousState: self))
+            }
+
+            // Otherwise, we have a failure state.  Die on the sword!
+            return deferMaybe(processFailure(result.failureValue))
+        }
+    }
+}
+
+open class HasMetaGlobal: BaseSyncStateWithInfo {
+    open override var label: SyncStateLabel { return SyncStateLabel.HasMetaGlobal }
+
+    class func fromState(_ state: BaseSyncStateWithInfo) -> HasMetaGlobal {
+        return HasMetaGlobal(client: state.client, scratchpad: state.scratchpad, token: state.token, info: state.info)
+    }
+
+    class func fromState(_ state: BaseSyncStateWithInfo, scratchpad: Scratchpad) -> HasMetaGlobal {
+        return HasMetaGlobal(client: state.client, scratchpad: scratchpad, token: state.token, info: state.info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        // Check if crypto/keys is fresh in the cache already.
+        if let keys = self.scratchpad.keys, keys.value.valid {
+            if let cryptoModified = self.info.modified("crypto") {
+                // Both of these are server timestamps. If the record we stored was fetched after the last time the record was modified, as represented by the "crypto" entry in info/collections, and we're fetching from the
+                // same server, then the record must be identical, and we can use it directly.  If are ever additional records in the crypto collection, this will fetch keys too frequently.  In that case, we should use X-I-U-S and expect some 304 responses.
+                if keys.timestamp >= cryptoModified {
+                    log.debug("Cached keys fetched at \(keys.timestamp), newer than server modified \(cryptoModified). Using cached keys.")
+                    return deferMaybe(HasFreshCryptoKeys.fromState(self, scratchpad: self.scratchpad, collectionKeys: keys.value))
+                }
+
+                // The server timestamp is newer, so there might be new keys.
+                // Re-fetch keys and check to see if the actual contents differ.
+                // If the keys are the same, we can ignore this change. If they differ,
+                // we need to re-sync any collection whose keys just changed.
+                log.info("Cached keys fetched at \(keys.timestamp) older than server modified \(cryptoModified). Fetching fresh keys.")
+                return deferMaybe(NeedsFreshCryptoKeys.fromState(self, scratchpad: self.scratchpad, staleCollectionKeys: keys.value))
+            } else {
+                // No known modified time for crypto/. That likely means the server has no keys.
+                // Drop our cached value and fall through; we'll try to fetch, fail, and
+                // go through the usual failure flow.
+                log.warning("Local keys fetched at \(keys.timestamp) found, but no crypto collection on server. Dropping cached keys.")
+                self.scratchpad = self.scratchpad.evolve().setKeys(nil).build().checkpoint()
+            }
+        } else {
+            log.debug("No cached keys found. Fetching fresh keys.")
+        }
+
+        return deferMaybe(NeedsFreshCryptoKeys.fromState(self, scratchpad: self.scratchpad, staleCollectionKeys: nil))
+    }
+}
+
+open class NeedsFreshCryptoKeys: BaseSyncStateWithInfo {
+    open override var label: SyncStateLabel { return SyncStateLabel.NeedsFreshCryptoKeys }
+    let staleCollectionKeys: Keys?
+
+    class func fromState(_ state: BaseSyncStateWithInfo, scratchpad: Scratchpad, staleCollectionKeys: Keys?) -> NeedsFreshCryptoKeys {
+        return NeedsFreshCryptoKeys(client: state.client, scratchpad: scratchpad, token: state.token, info: state.info, keys: staleCollectionKeys)
+    }
+
+    public init(client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections, keys: Keys?) {
+        self.staleCollectionKeys = keys
+        super.init(client: client, scratchpad: scratchpad, token: token, info: info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        // Fetch crypto/keys.
+        return self.client.getCryptoKeys(self.scratchpad.syncKeyBundle, ifUnmodifiedSince: nil).bind { result in
+            if let resp = result.successValue {
+                let collectionKeys = Keys(payload: resp.value.payload)
+                if !collectionKeys.valid {
+                    log.error("Unexpectedly invalid crypto/keys during a successful fetch.")
+                    return Deferred(value: Maybe(failure: InvalidKeysError(collectionKeys)))
+                }
+
+                let fetched = Fetched(value: collectionKeys, timestamp: resp.metadata.timestampMilliseconds)
+                let s = self.scratchpad.evolve()
+                        .addLocalCommandsFromKeys(fetched)
+                        .setKeys(fetched)
+                        .build().checkpoint()
+                return deferMaybe(HasFreshCryptoKeys.fromState(self, scratchpad: s, collectionKeys: collectionKeys))
+            }
+
+            if let _ = result.failureValue as? NotFound<HTTPURLResponse> {
+                // No crypto/keys?  We can handle this.  Wipe and upload both meta/global and crypto/keys.
+                return deferMaybe(MissingCryptoKeysError(previousState: self))
+            }
+
+            // Otherwise, we have a failure state.
+            return deferMaybe(processFailure(result.failureValue))
+        }
+    }
+}
+
+open class HasFreshCryptoKeys: BaseSyncStateWithInfo {
+    open override var label: SyncStateLabel { return SyncStateLabel.HasFreshCryptoKeys }
+    let collectionKeys: Keys
+
+    class func fromState(_ state: BaseSyncStateWithInfo, scratchpad: Scratchpad, collectionKeys: Keys) -> HasFreshCryptoKeys {
+        return HasFreshCryptoKeys(client: state.client, scratchpad: scratchpad, token: state.token, info: state.info, keys: collectionKeys)
+    }
+
+    public init(client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections, keys: Keys) {
+        self.collectionKeys = keys
+        super.init(client: client, scratchpad: scratchpad, token: token, info: info)
+    }
+
+    override open func advance() -> Deferred<Maybe<SyncState>> {
+        return deferMaybe(Ready(client: self.client, scratchpad: self.scratchpad, token: self.token, info: self.info, keys: self.collectionKeys))
+    }
+}
+
+public protocol EngineStateChanges {
+    func collectionsThatNeedLocalReset() -> [String]
+    func enginesEnabled() -> [String]
+    func enginesDisabled() -> [String]
+    func clearLocalCommands()
+}
+
+open class Ready: BaseSyncStateWithInfo {
+    open override var label: SyncStateLabel { return SyncStateLabel.Ready }
+    let collectionKeys: Keys
+
+    public var hashedFxADeviceID: String {
+        return (scratchpad.fxaDeviceId + token.hashedFxAUID).sha256.hexEncodedString
+    }
+
+    public init(client: Sync15StorageClient, scratchpad: Scratchpad, token: TokenServerToken, info: InfoCollections, keys: Keys) {
+        self.collectionKeys = keys
+        super.init(client: client, scratchpad: scratchpad, token: token, info: info)
+    }
+}
+
+extension Ready: EngineStateChanges {
+    public func collectionsThatNeedLocalReset() -> [String] {
+        var needReset: Set<String> = Set()
+        for command in self.scratchpad.localCommands {
+            switch command {
+            case let .resetAllEngines(except: except):
+                needReset.formUnion(Set(LocalEngines).subtracting(except))
+            case let .resetEngine(engine):
+                needReset.insert(engine)
+            case .enableEngine, .disableEngine:
+                break
+            }
+        }
+        return Array(needReset).sorted()
+    }
+
+    public func enginesEnabled() -> [String] {
+        var engines: Set<String> = Set()
+        for command in self.scratchpad.localCommands {
+            switch command {
+            case let .enableEngine(engine):
+                engines.insert(engine)
+            default:
+                break
+            }
+        }
+        return Array(engines).sorted()
+    }
+
+    public func enginesDisabled() -> [String] {
+        var engines: Set<String> = Set()
+        for command in self.scratchpad.localCommands {
+            switch command {
+            case let .disableEngine(engine):
+                engines.insert(engine)
+            default:
+                break
+            }
+        }
+        return Array(engines).sorted()
+    }
+
+    public func clearLocalCommands() {
+        self.scratchpad = self.scratchpad.evolve().clearLocalCommands().build().checkpoint()
+    }
+}

mobile/ios/Sync/SyncTelemetryUtils.swift

@@ -0,0 +1,356 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Account
+import Storage
+import SwiftyJSON
+import SyncTelemetry
+import Deferred
+
+fileprivate let log = Logger.syncLogger
+
+public let PrefKeySyncEvents = "sync.telemetry.events"
+
+public enum SyncReason: String {
+    case startup = "startup"
+    case scheduled = "scheduled"
+    case backgrounded = "backgrounded"
+    case user = "user"
+    case syncNow = "syncNow"
+    case didLogin = "didLogin"
+    case push = "push"
+}
+
+public enum SyncPingReason: String {
+    case shutdown = "shutdown"
+    case schedule = "schedule"
+    case idChanged = "idchanged"
+}
+
+public protocol Stats {
+    func hasData() -> Bool
+}
+
+private protocol DictionaryRepresentable {
+    func asDictionary() -> [String: Any]
+}
+
+public struct SyncUploadStats: Stats {
+    var sent: Int = 0
+    var sentFailed: Int = 0
+
+    public func hasData() -> Bool {
+        return sent > 0 || sentFailed > 0
+    }
+}
+
+extension SyncUploadStats: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        return [
+            "sent": sent,
+            "sentFailed": sentFailed
+        ]
+    }
+}
+
+public struct SyncDownloadStats: Stats {
+    var applied: Int = 0
+    var succeeded: Int = 0
+    var failed: Int = 0
+    var newFailed: Int = 0
+    var reconciled: Int = 0
+
+    public func hasData() -> Bool {
+        return applied > 0 ||
+               succeeded > 0 ||
+               failed > 0 ||
+               newFailed > 0 ||
+               reconciled > 0
+    }
+}
+
+extension SyncDownloadStats: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        return [
+            "applied": applied,
+            "succeeded": succeeded,
+            "failed": failed,
+            "newFailed": newFailed,
+            "reconciled": reconciled
+        ]
+    }
+}
+
+public struct ValidationStats: Stats, DictionaryRepresentable {
+    let problems: [ValidationProblem]
+    let took: Int64
+    let checked: Int?
+
+    public func hasData() -> Bool {
+        return !problems.isEmpty
+    }
+
+    func asDictionary() -> [String: Any] {
+        var dict: [String: Any] = [
+            "problems": problems.map { $0.asDictionary() },
+            "took": took
+        ]
+        if let checked = self.checked {
+            dict["checked"] = checked
+        }
+        return dict
+    }
+}
+
+public struct ValidationProblem: DictionaryRepresentable {
+    let name: String
+    let count: Int
+
+    func asDictionary() -> [String: Any] {
+        return ["name": name, "count": count]
+    }
+}
+
+public class StatsSession {
+    var took: Int64 = 0
+    var when: Timestamp?
+
+    private var startUptimeNanos: UInt64?
+
+    public func start(when: UInt64 = Date.now()) {
+        self.when = when
+        self.startUptimeNanos = DispatchTime.now().uptimeNanoseconds
+    }
+
+    public func hasStarted() -> Bool {
+        return startUptimeNanos != nil
+    }
+
+    public func end() -> Self {
+        guard let startUptime = startUptimeNanos else {
+            assertionFailure("SyncOperationStats called end without first calling start!")
+            return self
+        }
+
+        // Casting to Int64 should be safe since we're using uptime since boot in both cases.
+        // Convert to milliseconds as stated in the sync ping format
+        took = (Int64(DispatchTime.now().uptimeNanoseconds) - Int64(startUptime)) / 1000000
+        return self
+    }
+}
+
+// Stats about a single engine's sync.
+public class SyncEngineStatsSession: StatsSession {
+    public var validationStats: ValidationStats?
+
+    private(set) var uploadStats: SyncUploadStats
+    private(set) var downloadStats: SyncDownloadStats
+
+    public init(collection: String) {
+        self.uploadStats = SyncUploadStats()
+        self.downloadStats = SyncDownloadStats()
+    }
+
+    public func recordDownload(stats: SyncDownloadStats) {
+        self.downloadStats.applied += stats.applied
+        self.downloadStats.succeeded += stats.succeeded
+        self.downloadStats.failed += stats.failed
+        self.downloadStats.newFailed += stats.newFailed
+        self.downloadStats.reconciled += stats.reconciled
+    }
+
+    public func recordUpload(stats: SyncUploadStats) {
+        self.uploadStats.sent += stats.sent
+        self.uploadStats.sentFailed += stats.sentFailed
+    }
+}
+
+extension SyncEngineStatsSession: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        var dict: [String: Any] = [
+            "took": took,
+        ]
+
+        if downloadStats.hasData() {
+            dict["incoming"] = downloadStats.asDictionary()
+        }
+
+        if uploadStats.hasData() {
+            dict["outgoing"] = uploadStats.asDictionary()
+        }
+
+        if let validation = self.validationStats, validation.hasData() {
+            dict["validation"] = validation.asDictionary()
+        }
+
+        return dict
+    }
+}
+
+// Stats and metadata for a sync operation.
+public class SyncOperationStatsSession: StatsSession {
+    public let why: SyncReason
+    public var uid: String?
+    public var deviceID: String?
+
+    fileprivate let didLogin: Bool
+
+    public init(why: SyncReason, uid: String, deviceID: String?) {
+        self.why = why
+        self.uid = uid
+        self.deviceID = deviceID
+        self.didLogin = (why == .didLogin)
+    }
+}
+
+extension SyncOperationStatsSession: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        let whenValue = when ?? 0
+        return [
+            "when": whenValue,
+            "took": took,
+            "didLogin": didLogin,
+            "why": why.rawValue
+        ]
+    }
+}
+
+public enum SyncPingError: MaybeErrorType {
+    case failedToRestoreScratchpad
+
+    public var description: String {
+        switch self {
+        case .failedToRestoreScratchpad: return "Failed to restore Scratchpad from prefs"
+        }
+    }
+}
+
+public enum SyncPingFailureReasonName: String {
+    case httpError = "httperror"
+    case unexpectedError = "unexpectederror"
+    case sqlError = "sqlerror"
+    case otherError = "othererror"
+}
+
+public protocol SyncPingFailureFormattable {
+    var failureReasonName: SyncPingFailureReasonName { get }
+}
+
+public struct SyncPing: SyncTelemetryPing {
+    public private(set) var payload: JSON
+
+    public static func from(result: SyncOperationResult,
+                            account: FirefoxAccount,
+                            remoteClientsAndTabs: RemoteClientsAndTabs,
+                            prefs: Prefs,
+                            why: SyncPingReason) -> Deferred<Maybe<SyncPing>> {
+        // Grab our token so we can use the hashed_fxa_uid and clientGUID from our scratchpad for
+        // our ping's identifiers
+        return account.syncAuthState.token(Date.now(), canBeExpired: false) >>== { (token, kB) in
+            let scratchpadPrefs = prefs.branch("sync.scratchpad")
+            guard let scratchpad = Scratchpad.restoreFromPrefs(scratchpadPrefs, syncKeyBundle: KeyBundle.fromKB(kB)) else {
+                return deferMaybe(SyncPingError.failedToRestoreScratchpad)
+            }
+
+            var ping: [String: Any] = [
+                "version": 1,
+                "why": why.rawValue,
+                "uid": token.hashedFxAUID,
+                "deviceID": (scratchpad.clientGUID + token.hashedFxAUID).sha256.hexEncodedString
+            ]
+
+            // TODO: We don't cache our sync pings so if it fails, it fails. Once we add
+            // some kind of caching we'll want to make sure we don't dump the events if
+            // the ping has failed.
+            let pickledEvents = prefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? []
+            let events = pickledEvents.flatMap(Event.unpickle).map { $0.toArray() }
+            ping["events"] = events
+            prefs.setObject(nil, forKey: PrefKeySyncEvents)
+
+            return dictionaryFrom(result: result, storage: remoteClientsAndTabs, token: token) >>== { syncDict in
+                // TODO: Split the sync ping metadata from storing a single sync.
+                ping["syncs"] = [syncDict]
+                return deferMaybe(SyncPing(payload: JSON(ping)))
+            }
+        }
+    }
+
+    // Generates a single sync ping payload that is stored in the 'syncs' list in the sync ping.
+    private static func dictionaryFrom(result: SyncOperationResult,
+                                       storage: RemoteClientsAndTabs,
+                                       token: TokenServerToken) -> Deferred<Maybe<[String: Any]>> {
+        return connectedDevices(fromStorage: storage, token: token) >>== { devices in
+            guard let stats = result.stats else {
+                return deferMaybe([String: Any]())
+            }
+
+            var dict = stats.asDictionary()
+            if let engineResults = result.engineResults.successValue {
+                dict["engines"] = SyncPing.enginePingDataFrom(engineResults: engineResults)
+            } else if let failure = result.engineResults.failureValue {
+                var errorName: SyncPingFailureReasonName
+                if let formattableFailure = failure as? SyncPingFailureFormattable {
+                    errorName = formattableFailure.failureReasonName
+                } else {
+                    errorName = .unexpectedError
+                }
+
+                dict["failureReason"] = [
+                    "name": errorName.rawValue,
+                    "error": "\(type(of: failure))",
+                ]
+            }
+
+            dict["devices"] = devices
+            return deferMaybe(dict)
+        }
+    }
+
+    // Returns a list of connected devices formatted for use in the 'devices' property in the sync ping.
+    private static func connectedDevices(fromStorage storage: RemoteClientsAndTabs,
+                                         token: TokenServerToken) -> Deferred<Maybe<[[String: Any]]>> {
+        func dictionaryFrom(client: RemoteClient) -> [String: Any]? {
+            var device = [String: Any]()
+            if let os = client.os {
+                device["os"] = os
+            }
+            if let version = client.version {
+                device["version"] = version
+            }
+            if let guid = client.guid {
+                device["id"] = (guid + token.hashedFxAUID).sha256.hexEncodedString
+            }
+            return device
+        }
+
+        return storage.getClients() >>== { deferMaybe($0.flatMap(dictionaryFrom)) }
+    }
+
+    private static func enginePingDataFrom(engineResults: EngineResults) -> [[String: Any]] {
+        return engineResults.map { result in
+            let (name, status) = result
+            var engine: [String: Any] = [
+                "name": name
+            ]
+
+            // For complete/partial results, extract out the collect stats
+            // and add it to engine information. For syncs that were not able to
+            // start, return why and a reason.
+            switch status {
+            case .completed(let stats):
+                engine.merge(with: stats.asDictionary())
+            case .partial(let stats):
+                engine.merge(with: stats.asDictionary())
+            case .notStarted(let reason):
+                engine.merge(with: [
+                    "status": reason.telemetryId
+                ])
+            }
+
+            return engine
+        }
+    }
+}

mobile/ios/Sync/Synchronizers/Bookmarks/BookmarksDownloader.swift

@@ -0,0 +1,161 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+
+private let log = Logger.syncLogger
+let BookmarksStorageVersion = 2
+
+/**
+ * This is like a synchronizer, but it downloads records bit by bit, eventually
+ * notifying that the local storage is up to date with the server contents.
+ *
+ * Because batches might be separated over time, it's possible for the server
+ * state to change between calls. These state changes might include:
+ *
+ * 1. New changes arriving. This is fairly routine, but it's worth noting that
+ *    changes might affect existing records that have been batched!
+ * 2. Wipes. The collection (or the server as a whole) might be deleted. This
+ *    should be accompanied by a change in syncID in meta/global; it's the caller's
+ *    responsibility to detect this.
+ * 3. A storage format change. This should be unfathomably rare, but if it happens
+ *    we must also be prepared to discard our existing batched data.
+ * 4. TTL expiry. We need to do better about TTL handling in general, but here
+ *    we might find that a downloaded record is no longer live by the time we
+ *    come to apply it! This doesn't apply to bookmark records, so we will ignore
+ *    it for the moment.
+ *
+ * Batch downloading without continuation tokens is achieved as follows:
+ *
+ * * A minimum timestamp is established. This starts as zero.
+ * * A fetch is issued against the server for records changed since that timestamp,
+ *   ordered by modified time ascending, and limited to the batch size.
+ * * If the batch is complete, we flush it to storage and advance the minimum
+ *   timestamp to just before the newest record in the batch. This ensures that
+ *   a divided set of records with the same modified times will be downloaded
+ *   entirely so long as the set is never larger than the batch size.
+ * * Iterate until we determine that there are no new records to fetch.
+ *
+ * Batch downloading with continuation tokens is much easier:
+ *
+ * * A minimum timestamp is established.
+ * * Make a request with limit=N.
+ * * Look for an X-Weave-Next-Offset header. Supply that in the next request.
+ *   Also supply X-If-Unmodified-Since to avoid missed modifications.
+ *
+ * We do the latter, because we only support Sync 1.5. The use of the offset
+ * allows us to efficiently process batches, particularly those that contain
+ * large sets of records with the same timestamp. We still maintain the last
+ * modified timestamp to allow for resuming a batch in the case of a conflicting
+ * write, detected via X-I-U-S.
+ */
+
+public class BookmarksMirrorer {
+    private let downloader: BatchingDownloader<BookmarkBasePayload>
+    private let storage: BookmarkBufferStorage
+    private let batchSize: Int
+    private let statsSession: SyncEngineStatsSession
+
+    public init(storage: BookmarkBufferStorage, client: Sync15CollectionClient<BookmarkBasePayload>, basePrefs: Prefs, collection: String, statsSession: SyncEngineStatsSession, batchSize: Int=100) {
+        self.storage = storage
+        self.downloader = BatchingDownloader(collectionClient: client, basePrefs: basePrefs, collection: collection)
+        self.batchSize = batchSize
+        self.statsSession = statsSession
+    }
+
+    var lastModified: Timestamp {
+        get {
+            return self.downloader.lastModified
+        }
+    }
+
+    // TODO
+    public func storageFormatDidChange() {
+    }
+
+    // TODO
+    public func onWipeWasAppliedToStorage() {
+    }
+
+    private func applyRecordsFromBatcher() -> Success {
+        let retrieved = self.downloader.retrieve()
+        let invalid = retrieved.filter { !$0.payload.isValid() }
+
+        if !invalid.isEmpty {
+            // There's nothing we can do with invalid input. There's also no point in
+            // tracking failing GUIDs here yet: if another client reuploads those records
+            // correctly, we'll encounter them routinely due to a newer timestamp.
+            // The only improvement we could make is to drop records from the buffer if we
+            // happen to see a new, invalid one before somehow syncing again, but that's
+            // unlikely enough that it's not worth doing.
+            //
+            // Bug 1258801 tracks recording telemetry for these invalid items, which is
+            // why we don't simply drop them on the ground at the download stage.
+            //
+            // We might also choose to perform certain simple recovery actions here: for example,
+            // bookmarks with null URIs are clearly invalid, and could be treated as if they
+            // weren't present on the server, or transparently deleted.
+            log.warning("Invalid records: \(invalid.map { $0.id }.joined(separator: ", ")).")
+        }
+
+        let mirrorItems = retrieved.flatMap { record -> BookmarkMirrorItem? in
+            guard record.payload.isValid() else {
+                return nil
+            }
+
+            return (record.payload as MirrorItemable).toMirrorItem(record.modified)
+        }
+
+        if mirrorItems.isEmpty {
+            log.debug("Got empty batch.")
+            return succeed()
+        }
+
+        log.debug("Applying \(mirrorItems.count) downloaded bookmarks.")
+        return self.storage.applyRecords(mirrorItems)
+    }
+
+    public func go(info: InfoCollections, greenLight: @escaping () -> Bool) -> SyncResult {
+        if !greenLight() {
+            log.info("Green light turned red. Stopping mirror operation.")
+            return deferMaybe(SyncStatus.notStarted(.redLight))
+        }
+
+        log.debug("Downloading up to \(self.batchSize) records.")
+        return self.downloader.go(info, limit: self.batchSize)
+                              .bind { result in
+            guard let end = result.successValue else {
+                log.warning("Got failure: \(result.failureValue!)")
+                return deferMaybe(result.failureValue!)
+            }
+            switch end {
+            case .complete:
+                log.info("Done with batched mirroring.")
+                return self.applyRecordsFromBatcher()
+                   >>> effect(self.downloader.advance)
+                   >>> self.storage.doneApplyingRecordsAfterDownload
+                   >>> always(SyncStatus.completed(self.statsSession.end()))
+            case .incomplete:
+                log.debug("Running another batch.")
+                // This recursion is fine because Deferred always pushes callbacks onto a queue.
+                return self.applyRecordsFromBatcher()
+                   >>> effect(self.downloader.advance)
+                   >>> { self.go(info: info, greenLight: greenLight) }
+            case .interrupted:
+                log.info("Interrupted. Aborting batching this time.")
+                return deferMaybe(SyncStatus.partial(self.statsSession))
+            case .noNewData:
+                log.info("No new data. No need to continue batching.")
+                self.downloader.advance()
+                return deferMaybe(SyncStatus.completed(self.statsSession.end()))
+            }
+        }
+    }
+
+    func advanceNextDownloadTimestampTo(timestamp: Timestamp) {
+        self.downloader.advanceTimestampTo(timestamp)
+    }
+}

mobile/ios/Sync/Synchronizers/Bookmarks/BookmarksRepairRequestor.swift

@@ -0,0 +1,571 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import Deferred
+import SwiftyJSON
+import SyncTelemetry
+
+private let log = Logger.syncLogger
+
+// How long should we wait after sending a repair request before we give up?
+private let ResponseIntervalTimeout = OneDayInMilliseconds * 3
+
+// The maximum number of IDs we will request to be repaired. Beyond this
+// number we assume that trying to repair may do more harm than good and may
+// ask another client to wipe the server and reupload everything. Bug 1341972
+// is tracking that work for Desktop.
+private let MaxRequestedIDs = 1000
+
+// If a repair is in progress, this is the generated GUID for the "flow ID".
+private let PrefFlowID = "flowID"
+// The IDs we are currently trying to obtain via the repair process.
+private let PrefMissingIDs = "ids"
+// The ID of the client we're currently trying to get the missing items from.
+private let PrefCurrentClient = "currentClient"
+// The IDs of the clients we've previously tried to get the missing items
+// from.
+private let PrefPreviousClients = "previousClients"
+// The time, in seconds, when we initiated the most recent client request.
+private let PrefLastRepair = "when"
+// Our current state.
+private let PrefCurrentState = "state"
+
+private enum RepairState: String {
+    // We have not started the repair process.
+    case notRepairing = ""
+
+    // We need to try to find another client to use.
+    case needNewClient = "repair.need-new-client"
+
+    // We've sent the first request to a client.
+    case sentRequest = "repair.sent"
+
+    // We've retried a request to a client.
+    case sentSecondRequest = "repair.sent-again"
+
+    // There were no problems, but we've gone as far as we can.
+    case finished = "repair.finished"
+
+    // We've found an error that forces us to abort this entire repair cycle.
+    case aborted = "repair.aborted"
+}
+
+struct RepairResponse {
+    let collection: String
+    let request: String
+    let flowID: String
+    let clientID: String
+    let ids: [String]
+
+    static func fromJSON(args: JSON) -> RepairResponse {
+        return RepairResponse(collection: args["collection"].stringValue, request: args["request"].stringValue,
+                              flowID: args["flowID"].stringValue, clientID: args["clientID"].stringValue,
+                              ids: args["ids"].arrayValue.map { $0.stringValue })
+    }
+}
+
+struct RepairRequest {
+    let collection: String
+    let request: String
+    let flowID: String
+    let requestor: String
+    let ids: [String]
+
+    func toSyncCommand() -> SyncCommand {
+        let jsonObj: [String: Any] = [
+            "command": "repairRequest",
+            "args": [
+                [
+                "collection": collection,
+                "request": request,
+                "flowID": flowID,
+                "requestor": requestor,
+                "ids": ids
+                ]
+            ]
+        ]
+        return SyncCommand(value: JSON(object: jsonObj).stringValue()!)
+    }
+}
+
+private class AbortRepairError: MaybeErrorType {
+    let description: String
+    init(_ description: String) {
+        self.description = description
+    }
+}
+
+private class UnknownClientError: MaybeErrorType {
+    let description: String
+    init(_ description: String) {
+        self.description = description
+    }
+}
+
+private class InvalidStateError: MaybeErrorType {
+    let description: String
+    init(_ description: String) {
+        self.description = description
+    }
+}
+
+class BookmarksRepairRequestor {
+    let prefs: Prefs
+    let basePrefs: Prefs
+    let remoteClients: RemoteClientsAndTabs
+    let scratchpad: Scratchpad
+
+    init(scratchpad: Scratchpad, basePrefs: Prefs, remoteClients: RemoteClientsAndTabs) {
+        self.scratchpad = scratchpad
+        self.basePrefs = basePrefs
+        self.prefs = basePrefs.branch("repairs.bookmark")
+        self.remoteClients = remoteClients
+    }
+
+    /**
+     * See if the repairer is willing and able to begin a repair process given
+     * the specified validation information.
+     *
+     * - returns: true if a repair was started and false otherwise.
+     */
+    func startRepairs(validationInfo: [BufferInconsistency: [GUID]], flowID: String = Bytes.generateGUID()) -> Deferred<Maybe<Bool>> {
+        guard self.currentState == .notRepairing else {
+            log.info("Can't start a repair - repair with ID \(self.flowID) is already in progress")
+            return deferMaybe(false)
+        }
+
+        let ids = self.getProblemIDs(validationInfo)
+
+        guard ids.count > 0 else {
+            log.info("Not starting a repair as there are no problems")
+            return deferMaybe(false)
+        }
+
+        guard ids.count <= MaxRequestedIDs else {
+            log.info("Not starting a repair as there are over \(MaxRequestedIDs) problems")
+            let extra = [
+                "flowID": flowID,
+                "reason": "too many problems: \(ids.count)"
+            ]
+
+            let event = Event(category: "sync", method: "repair", object: "aborted", extra: extra)
+            recordTelemetry(event: event)
+
+            return deferMaybe(false)
+        }
+
+        return self.anyClientsRepairing() >>== { clientsRepairing in
+            guard !clientsRepairing else {
+                log.info("Can't start repair, since other clients are already repairing bookmarks")
+                let extra = [
+                    "flowID": flowID,
+                    "reason": "other clients repairing"
+                ]
+                let event = Event(category: "sync", method: "repair", object: "aborted", extra: extra)
+                self.recordTelemetry(event: event)
+                return deferMaybe(false)
+            }
+
+            log.info("Starting a repair, looking for \(ids.count) missing item(s)")
+            // Setup our prefs to indicate we are on our way.
+            self.flowID = flowID
+            self.currentIDs = ids
+            self.currentState = .needNewClient
+
+            let extra = ["flowID": flowID, "numIDs": String(ids.count)]
+            let event = Event(category: "sync", method: "repair", object: "started", extra: extra)
+            self.recordTelemetry(event: event)
+
+            return self.continueRepairs()
+        }
+    }
+
+    /**
+     * Work out what state our current repair request is in, and whether it can
+     * proceed to a new state.
+     *
+     * - returns: true if we could continue the repair - even if the state didn't
+     *            actually move. Returns false if we aren't actually repairing.
+     */
+    func continueRepairs(response: RepairResponse? = nil) -> Deferred<Maybe<Bool>> {
+        // Note that "aborted" and "finished" should never be current when this
+        // function returns - this function resets to notRepairing in those cases.
+        guard self.currentState != .notRepairing else {
+            return deferMaybe(false)
+        }
+
+        var abortReason: String?
+
+        func runStateMachine(iteration: Int = 0) -> Deferred<Maybe<(state: RepairState, newState: RepairState)>> {
+            let state = self.currentState
+            log.info("continueRepairs starting with state \(state)")
+
+            return self.advanceRepairState(state: state, response: response).bind { result in
+                let newState: RepairState
+                if result.isSuccess {
+                    newState = result.successValue!
+                    log.info("continueRepairs has next state \(newState)")
+                } else {
+                    let failure = result.failureValue!
+                    if failure is AbortRepairError {
+                        let reason = failure.description
+                        log.info("Repair has been aborted: \(reason)")
+                        newState = .aborted
+                        abortReason = reason
+                    } else {
+                        return deferMaybe(failure)
+                    }
+                }
+                let done = deferMaybe((state: state, newState: newState))
+
+                if newState == .aborted {
+                    return done
+                }
+
+                self.currentState = newState
+                if state == newState {
+                    return done
+                }
+
+                // we loop until the state doesn't change - but enforce a max of 10 times
+                // to prevent errors causing infinite loops.
+                return (iteration < 10) ? runStateMachine(iteration: iteration + 1) : done
+            }
+        }
+
+        return runStateMachine() >>== { stateMachineResult in
+            let state = stateMachineResult.state
+            let newState = stateMachineResult.newState
+
+            if state != newState {
+                log.error("continueRepairs spun without getting a new state")
+            }
+
+            if newState == .finished || newState == .aborted {
+                let method = newState == .finished ? "finished" : "aborted"
+                var extra = [
+                    "flowID": self.flowID,
+                    "numIDs": String(self.currentIDs.count),
+                ]
+                if abortReason != nil {
+                    extra["reason"] = abortReason
+                }
+
+                let event = Event(category: "sync", method: "repair", object: method, extra: extra)
+                self.recordTelemetry(event: event)
+
+                self.prefs.clearAll()
+            }
+            return deferMaybe(true)
+        }
+    }
+
+    func recordTelemetry(event: Event) {
+        var events = self.basePrefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? []
+
+        if let data = event.pickle(), event.validate() {
+            events.append(data)
+            self.basePrefs.setObject(events, forKey: PrefKeySyncEvents)
+        } else {
+            log.info("Event not recorded due to validation failure or pickling error!")
+        }
+    }
+
+    private func advanceRepairState(state: RepairState, response: RepairResponse?) -> Deferred<Maybe<RepairState>> {
+        return self.anyClientsRepairing(flowID: self.flowID) >>== { anyClientsRepairing in
+            guard !anyClientsRepairing else {
+                return deferMaybe(AbortRepairError("other clients repairing"))
+            }
+
+            switch state {
+
+            case .sentRequest, .sentSecondRequest:
+                let flowID = self.flowID
+                guard let clientID = self.currentRemoteClient else {
+                    return deferMaybe(InvalidStateError("currentRemoteClient should be defined"))
+                }
+                if let response = response {
+                    // We got an explicit response - let's see how we went.
+                    return deferMaybe(self.handleResponse(state: state, response: response))
+                }
+                // So we've sent a request - and don't yet have a response. See if the
+                // client we sent it to has removed it from its list (ie, whether it
+                // has synced since we wrote the request.)
+                return self.remoteClients.getClientWithId(clientID) >>== { client in
+                    guard let client = client else {
+                        // hrmph - the client has disappeared.
+                        log.info("previously requested client \(clientID) has vanished - moving to next step")
+                        let extra = [
+                            "deviceID": self.scratchpad.hashedDeviceID ?? "unknown_deviceID",
+                            "flowID": flowID
+                        ]
+
+                        let event = Event(category: "sync", method: "repair", object: "abandon", value: "missing", extra: extra)
+                        self.recordTelemetry(event: event)
+
+                        return deferMaybe(.needNewClient)
+                    }
+                    return self.isCommandPending(clientID: clientID, flowID: flowID) >>== { isCommandPending in
+                        if isCommandPending {
+                            // So the command we previously sent is still queued for the client
+                            // (ie, that client is yet to have synced). Let's see if we should
+                            // give up on that client.
+                            if self.lastRepair + ResponseIntervalTimeout <= Date.now() {
+                                log.info("previous request to client \(clientID) is pending, but has taken too long")
+                                // XXX - should we remove the command?
+                                let extra = [
+                                    "deviceID": self.scratchpad.hashedDeviceID ?? "unknown_deviceID",
+                                    "flowID": flowID
+                                ]
+
+                                let event = Event(category: "sync", method: "repair", object: "abandon", value: "silent", extra: extra)
+                                self.recordTelemetry(event: event)
+
+                                return deferMaybe(.needNewClient)
+                            }
+                            // Let's continue to wait for that client to respond.
+                            // We are now sure that timeLeft > 0, so we can calculate it (Timestamp type is UInt64)
+                            let timeLeft = self.lastRepair + ResponseIntervalTimeout - Date.now()
+                            log.verbose("previous request to client \(clientID) has \(timeLeft) seconds before we give up on it")
+                            return deferMaybe(state)
+                        }
+                        // The command isn't pending - if this was the first request, we give
+                        // it another go (as that client may have cleared the command but is yet
+                        // to complete the sync)
+                        // XXX - note that this is no longer true - the responders don't remove
+                        // their command until they have written a response. This might mean
+                        // we could drop the entire STATE.SENT_SECOND_REQUEST concept???
+                        if state == .sentRequest {
+                            log.info("previous request to client \(clientID) was removed - trying a second time")
+                            return self.writeRequest(client: client) >>== { success in
+                                return deferMaybe(.sentSecondRequest)
+                            }
+                        } else {
+                            // this was the second time around, so give up on this client
+                            log.info("previous 2 requests to client \(clientID) were removed - need a new client")
+                            return deferMaybe(.needNewClient)
+                        }
+                    }
+                }
+
+            case .needNewClient:
+                // We need to find a new client to request.
+                return self.findNextClient() >>== { client in
+                    guard let nextClient = client else {
+                        return deferMaybe(.finished)
+                    }
+                    if let currentRemoteClient = self.currentRemoteClient {
+                        var previousRemoteClients = self.previousRemoteClients
+                        previousRemoteClients.append(currentRemoteClient)
+                        self.previousRemoteClients = previousRemoteClients
+                    }
+                    self.currentRemoteClient = nextClient.guid!
+                    return self.writeRequest(client: nextClient) >>== { success in
+                        return deferMaybe(.sentRequest)
+                    }
+                }
+
+            case .aborted:
+                break // our caller will take the abort action.
+
+            case .finished:
+                break
+
+            case .notRepairing:
+                // No repair is in progress. This is a common case, so only log trace.
+                log.verbose("continue repairs called but no repair in progress.")
+                break
+            }
+            return deferMaybe(state)
+        }
+    }
+
+    /**
+     * Handle being in the SENT_REQUEST or SENT_SECOND_REQUEST state with an
+     * explicit response.
+     */
+    private func handleResponse(state: RepairState, response: RepairResponse) -> RepairState {
+        guard let clientID = self.currentRemoteClient else {
+            log.error("Cannot handle the response of an unknown client")
+            return state
+        }
+        let flowID = self.flowID
+        guard response.flowID == flowID && response.clientID == clientID &&
+              response.request == "upload" else {
+            log.info("got a response to a different repair request: \(response)")
+            // hopefully just a stale request that finally came in (either from
+            // an entirely different repair flow, or from a client we've since
+            // given up on.) It doesn't mean we need to abort though...
+            return state
+        }
+        // Pull apart the response and see if it provided everything we asked for.
+        let remainingIDs = Array(Set(self.currentIDs).subtracting(Set(response.ids)))
+        log.info("repair response from \(clientID) provided '\(response.ids)', remaining now '\(remainingIDs)'")
+        self.currentIDs = remainingIDs
+        let newState: RepairState
+        if remainingIDs.count > 0 {
+            // try a new client for the remaining ones.
+            newState = .needNewClient
+        } else {
+            newState = .finished
+        }
+        // record telemetry about this
+        let extra = [
+            "deviceID": scratchpad.hashedDeviceID ?? "unknown_deviceID",
+            "flowID": flowID,
+            "numIDs": String(response.ids.count)
+        ]
+        let event = Event(category: "sync", method: "repair", object: "response", value: "upload", extra: extra)
+        recordTelemetry(event: event)
+        return newState
+    }
+
+    /**
+     * Issue a repair request to a specific client.
+     */
+    private func writeRequest(client: RemoteClient) -> Success {
+        log.verbose("writing repair request to client \(client.guid!)")
+        let ids = self.currentIDs
+        let flowID = self.flowID
+        // Post a command to that client.
+        let request = RepairRequest(collection: "bookmarks", request: "upload", flowID: flowID, requestor: self.scratchpad.clientGUID, ids: ids)
+
+        return self.remoteClients.insertCommand(request.toSyncCommand(), forClients: [client]) >>== { (_: Int) -> Success in
+            self.lastRepair = Date.now()
+            // record telemetry about this
+            let extra = [
+                "deviceID": self.scratchpad.hashedDeviceID ?? "unknown_deviceID",
+                "flowID": flowID,
+                "numIDs": String(ids.count),
+                ]
+
+            let event = Event(category: "sync", method: "repair", object: "request", value: "upload", extra: extra)
+            self.recordTelemetry(event: event)
+
+            return succeed()
+        }
+    }
+
+    private func findNextClient() -> Deferred<Maybe<RemoteClient?>> {
+        var alreadyDone = self.previousRemoteClients
+        if let currentRemoteClient = self.currentRemoteClient {
+            alreadyDone.append(currentRemoteClient)
+        }
+        return self.remoteClients.getClients() >>== { remoteClients in
+            // we want to consider the most-recently synced clients first.
+            let sortedClients = remoteClients.sorted(by: { (a, b) -> Bool in
+                return a.modified > b.modified
+            })
+            for client in sortedClients {
+                log.verbose("findNextClient considering \(client)")
+                guard let clientID = client.guid else {
+                    continue
+                }
+                if !alreadyDone.contains(clientID) && self.isSuitableClient(client) {
+                    return deferMaybe(client)
+                }
+            }
+            log.verbose("findNextClient found no client")
+            return deferMaybe(nil)
+        }
+    }
+
+    private func isSuitableClient(_ client: RemoteClient) -> Bool {
+        if let type = client.type,
+           let version = client.version,
+           let major = Int(version.components(separatedBy: ".")[0]) {
+            return type == "desktop" && major > 53
+        }
+        return false
+    }
+
+    private func getProblemIDs(_ validations: [BufferInconsistency: [GUID]]) -> [GUID] {
+        return validations.reduce([]) { acc, pair in acc + pair.value }
+    }
+
+    private func anyClientsRepairing(flowID: String? = nil) -> Deferred<Maybe<Bool>> {
+        return self.remoteClients.getCommands() >>== { allCommands in
+            return deferMaybe(allCommands.contains { (clientID: GUID, commands: [SyncCommand]) in
+                return commands.contains { (command: SyncCommand) in
+                    let json = JSON(parseJSON: command.value)
+                    guard let cmdName = json["command"].string,
+                          let argsArray = json["args"].array,
+                          let argObj = argsArray[0].dictionary,
+                          let argCol = argObj["collection"]?.string,
+                          let argFlowID = argObj["flowID"]?.string
+                    else {
+                        return false
+                    }
+                    return !((cmdName != "repairResponse" && cmdName != "repairRequest") ||
+                             argsArray.count != 1 ||
+                             argCol != "bookmarks" ||
+                             argFlowID == flowID
+                            )
+                }
+            })
+        }
+    }
+
+    private func isCommandPending(clientID: String, flowID: String) -> Deferred<Maybe<Bool>> {
+        return self.remoteClients.getCommands() >>== { allCommands in
+            guard let commands = allCommands[clientID] else {
+                return deferMaybe(false)
+            }
+            return deferMaybe(commands.contains { (command: SyncCommand) in
+                let json = JSON(parseJSON: command.value)
+                guard let cmdName = json["command"].string,
+                      let argsArray = json["args"].array,
+                      let argObj = argsArray[0].dictionary,
+                      let argCol = argObj["collection"]?.string,
+                      let argFlowID = argObj["flowID"]?.string,
+                      let argRequest = argObj["request"]?.string
+                else {
+                    return false
+                }
+                return cmdName == "repairRequest" && argsArray.count == 1 &&
+                    argCol == "bookmarks" && argRequest == "upload" &&
+                    argFlowID == flowID
+            })
+        }
+    }
+
+    private var flowID: String {
+        get { return self.prefs.stringForKey(PrefFlowID)! }
+        set { self.prefs.setString(newValue, forKey: PrefFlowID) }
+    }
+
+    private var currentIDs: [String] {
+        get { return self.prefs.stringArrayForKey(PrefMissingIDs) ?? [String]() }
+        set { self.prefs.setObject(newValue, forKey: PrefMissingIDs) }
+    }
+
+    private var currentRemoteClient: String? {
+        get { return self.prefs.stringForKey(PrefCurrentClient) }
+        set { self.prefs.setString(newValue!, forKey: PrefCurrentClient) }
+    }
+
+    private var previousRemoteClients: [String] {
+        get { return self.prefs.stringArrayForKey(PrefPreviousClients) ?? [String]() }
+        set { self.prefs.setObject(newValue, forKey: PrefPreviousClients) }
+    }
+
+    private var lastRepair: Timestamp {
+        get { return self.prefs.timestampForKey(PrefLastRepair)! }
+        set { self.prefs.setTimestamp(newValue, forKey: PrefLastRepair) }
+    }
+
+    private var currentState: RepairState {
+        get {
+            let Default = RepairState.notRepairing
+            guard let raw = self.prefs.stringForKey(PrefCurrentState) else {
+                return Default
+            }
+            return RepairState(rawValue: raw) ?? Default
+        }
+        set { self.prefs.setString(newValue.rawValue, forKey: PrefCurrentState) }
+    }
+}

mobile/ios/Sync/Synchronizers/Bookmarks/BookmarksSynchronizer.swift

@@ -0,0 +1,511 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Deferred
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+
+private let log = Logger.syncLogger
+
+typealias UploadFunction = ([Record<BookmarkBasePayload>], _ lastTimestamp: Timestamp?, _ onUpload: @escaping (POSTResult, Timestamp?) -> DeferredTimestamp) -> DeferredTimestamp
+
+class TrivialBookmarkStorer: BookmarkStorer {
+    let uploader: UploadFunction
+    init(uploader: @escaping UploadFunction) {
+        self.uploader = uploader
+    }
+
+    func applyUpstreamCompletionOp(_ op: UpstreamCompletionOp, itemSources: ItemSources, trackingTimesInto local: LocalOverrideCompletionOp) -> Deferred<Maybe<POSTResult>> {
+        log.debug("Uploading \(op.records.count) modified records.")
+        log.debug("Uploading \(op.amendChildrenFromBuffer.count) amended buffer records.")
+        log.debug("Uploading \(op.amendChildrenFromMirror.count) amended mirror records.")
+        log.debug("Uploading \(op.amendChildrenFromLocal.count) amended local records.")
+
+        var records: [Record<BookmarkBasePayload>] = []
+        records.reserveCapacity(op.records.count + op.amendChildrenFromBuffer.count + op.amendChildrenFromLocal.count + op.amendChildrenFromMirror.count)
+        records.append(contentsOf: op.records)
+
+        func accumulateFromAmendMap(_ itemsWithNewChildren: [GUID: [GUID]], fetch: ([GUID: [GUID]]) -> Maybe<[GUID: BookmarkMirrorItem]>) throws /* MaybeErrorType */ {
+            if itemsWithNewChildren.isEmpty {
+                return
+            }
+
+            let fetched = fetch(itemsWithNewChildren)
+            guard let items = fetched.successValue else {
+                log.warning("Couldn't fetch items to amend.")
+                throw fetched.failureValue!
+            }
+
+            items.forEach { (guid, item) in
+                let payload = item.asPayloadWithChildren(itemsWithNewChildren[guid])
+                let mappedGUID = payload["id"].string ?? guid
+                let record = Record<BookmarkBasePayload>(id: mappedGUID, payload: payload)
+                records.append(record)
+            }
+        }
+
+        do {
+            try accumulateFromAmendMap(op.amendChildrenFromBuffer, fetch: { itemSources.buffer.getBufferItemsWithGUIDs($0.keys).value })
+            try accumulateFromAmendMap(op.amendChildrenFromMirror, fetch: { itemSources.mirror.getMirrorItemsWithGUIDs($0.keys).value })
+            try accumulateFromAmendMap(op.amendChildrenFromLocal, fetch: { itemSources.local.getLocalItemsWithGUIDs($0.keys).value })
+        } catch {
+            return deferMaybe(error as MaybeErrorType)
+        }
+
+        var success: [GUID] = []
+        var failed: [GUID: String] = [:]
+
+        func onUpload(_ result: POSTResult, lastModified: Timestamp?) -> DeferredTimestamp {
+            success.append(contentsOf: result.success)
+            result.failed.forEach { guid, message in
+                failed[guid] = message
+            }
+
+            log.debug("Uploaded records got timestamp \(lastModified ??? "nil").")
+            let modified = lastModified ?? 0
+            local.setModifiedTime(modified, guids: result.success)
+            return deferMaybe(modified)
+        }
+
+        // Chain the last upload timestamp right into our lastFetched timestamp.
+        // This is what Sync clients tend to do, but we can probably do better.
+        return uploader(records, op.ifUnmodifiedSince, onUpload)
+            // As if we uploaded everything in one go.
+            >>> { deferMaybe(POSTResult(success: success, failed: failed)) }
+    }
+}
+
+open class MalformedRecordError: MaybeErrorType, SyncPingFailureFormattable {
+    open var description: String {
+        return "Malformed record."
+    }
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+}
+
+// MARK: - External synchronizer interface.
+
+open class BufferingBookmarksSynchronizer: TimestampedSingleCollectionSynchronizer, Synchronizer {
+    public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
+        super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "bookmarks")
+    }
+
+    override var storageVersion: Int {
+        return BookmarksStorageVersion
+    }
+
+    fileprivate func buildMobileRootRecord(_ local: LocalItemSource, _ buffer: BufferItemSource, additionalChildren: [BookmarkMirrorItem], deletedChildren: [GUID]) -> Deferred<Maybe<Record<BookmarkBasePayload>>> {
+        let newBookmarkGUIDs = additionalChildren.map { $0.guid }
+        return buffer.getBufferItemWithGUID(BookmarkRoots.MobileFolderGUID).bind { maybeMobileRoot in
+            // Update (or create!) the Mobile Root folder with its new children.
+            if let mobileRoot = maybeMobileRoot.successValue {
+                return buffer.getBufferChildrenGUIDsForParent(mobileRoot.guid)
+                    .map { $0.map({ (mobileRoot: mobileRoot, children: $0.filter { !deletedChildren.contains($0) } + newBookmarkGUIDs) }) }
+            } else {
+                return local.getLocalItemWithGUID(BookmarkRoots.MobileFolderGUID)
+                    .map { $0.map({ (mobileRoot: $0, children: newBookmarkGUIDs) }) }
+            }
+        } >>== { (mobileRoot: BookmarkMirrorItem, children: [GUID]) in
+            let payload = mobileRoot.asPayloadWithChildren(children)
+            guard let mappedGUID = payload["id"].string else {
+                return deferMaybe(MalformedRecordError())
+            }
+            return deferMaybe(Record<BookmarkBasePayload>(id: mappedGUID, payload: payload))
+        }
+    }
+
+    func buildMobileRootAndChildrenRecords(_ local: LocalItemSource, _ buffer: BufferItemSource, additionalChildren: [BookmarkMirrorItem], deletedChildren: [GUID]) -> Deferred<Maybe<(mobileRootRecord: Record<BookmarkBasePayload>, childrenRecords: [Record<BookmarkBasePayload>])>> {
+        let childrenRecords =
+        additionalChildren.map { bkm -> Record<BookmarkBasePayload> in
+            let payload = bkm.asPayload()
+            let mappedGUID = payload["id"].string ?? bkm.guid
+            return Record<BookmarkBasePayload>(id: mappedGUID, payload: payload)
+        } +
+        deletedChildren.map { guid -> Record<BookmarkBasePayload> in
+            let payload = BookmarkBasePayload.deletedPayload(guid)
+            let mappedGUID = payload["id"].string ?? guid
+            return Record<BookmarkBasePayload>(id: mappedGUID, payload: payload)
+        }
+
+        return self.buildMobileRootRecord(local, buffer, additionalChildren: additionalChildren, deletedChildren: deletedChildren) >>== { mobileRootRecord in
+            return deferMaybe((mobileRootRecord: mobileRootRecord, childrenRecords: childrenRecords))
+        }
+    }
+
+    func uploadSomeLocalRecords(_ storage: SyncableBookmarks & LocalItemSource & MirrorItemSource, _ mirrorer: BookmarksMirrorer, _ bookmarksClient: Sync15CollectionClient<BookmarkBasePayload>, mobileRootRecord: Record<BookmarkBasePayload>, childrenRecords: [Record<BookmarkBasePayload>]) -> Success {
+        var newBookmarkGUIDs: [GUID] = []
+        var deletedBookmarksGUIDs: [GUID] = []
+        for record in childrenRecords {
+            // No mutable l-values in Swift :(
+            if record.payload.deleted {
+                deletedBookmarksGUIDs.append(record.id)
+            } else {
+                newBookmarkGUIDs.append(record.id)
+            }
+        }
+        let records = [mobileRootRecord] + childrenRecords
+        return self.uploadRecordsSingleBatch(records, lastTimestamp: mirrorer.lastModified, storageClient: bookmarksClient) >>== { (timestamp: Timestamp, succeeded: [GUID]) -> Success in
+
+            let bufferValuesToMoveFromLocal = Set(newBookmarkGUIDs).intersection(Set(succeeded))
+            let deletedValues = Set(deletedBookmarksGUIDs).intersection(Set(succeeded))
+            let mobileRoot = (mobileRootRecord.payload as MirrorItemable).toMirrorItem(timestamp)
+            let bufferOP = BufferUpdatedCompletionOp(bufferValuesToMoveFromLocal: bufferValuesToMoveFromLocal, deletedValues: deletedValues, mobileRoot: mobileRoot, modifiedTime: timestamp)
+            return storage.applyBufferUpdatedCompletionOp(bufferOP) >>> {
+                mirrorer.advanceNextDownloadTimestampTo(timestamp: timestamp) // We need to advance our batching downloader timestamp to match. See Bug 1253458.
+                return succeed()
+            }
+        }
+    }
+
+    open func synchronizeBookmarksToStorage(_ storage: SyncableBookmarks & LocalItemSource & MirrorItemSource, usingBuffer buffer: BookmarkBufferStorage & BufferItemSource, withServer storageClient: Sync15StorageClient, info: InfoCollections, greenLight: @escaping () -> Bool, remoteClientsAndTabs: RemoteClientsAndTabs) -> SyncResult {
+        if self.prefs.boolForKey("dateAddedMigrationDone") != true {
+            self.lastFetched = 0
+            self.prefs.setBool(true, forKey: "dateAddedMigrationDone")
+        }
+
+        if let reason = self.reasonToNotSync(storageClient) {
+            return deferMaybe(.notStarted(reason))
+        }
+
+        let encoder = RecordEncoder<BookmarkBasePayload>(decode: BookmarkType.somePayloadFromJSON, encode: { $0.json })
+
+        guard let bookmarksClient = self.collectionClient(encoder, storageClient: storageClient) else {
+            log.error("Couldn't make bookmarks factory.")
+            return deferMaybe(FatalError(message: "Couldn't make bookmarks factory."))
+        }
+
+        let start = Date.nowMicroseconds()
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: bookmarksClient, basePrefs: self.prefs, collection: "bookmarks", statsSession: self.statsSession)
+        let storer = TrivialBookmarkStorer(uploader: { records, lastTimestamp, onUpload in
+            let timestamp = lastTimestamp ?? self.lastFetched
+            return self.uploadRecords(records, lastTimestamp: timestamp, storageClient: bookmarksClient, onUpload: onUpload)
+              >>== effect { timestamp in
+                // We need to advance our batching downloader timestamp to match. See Bug 1253458.
+                self.setTimestamp(timestamp)
+                mirrorer.advanceNextDownloadTimestampTo(timestamp: timestamp)
+            }
+        })
+
+        statsSession.start()
+        
+        let doMirror = mirrorer.go(info: info, greenLight: greenLight)
+        let run: SyncResult
+
+        if !AppConstants.shouldMergeBookmarks {
+            run = doMirror >>== { result -> SyncResult in
+                // Validate the buffer to report statistics.
+                if case .completed = result {
+                    log.debug("Validating completed buffer download.")
+                    return buffer.validate().bind { validationResult in
+                        guard let invalidError = validationResult.failureValue as? BufferInvalidError else {
+                            return deferMaybe(result)
+                        }
+                        return buffer.getUpstreamRecordCount().bind { checked -> Success in
+                            self.statsSession.validationStats = self.validationStatsFrom(error: invalidError, checked: checked)
+                            return self.maybeStartRepairProcedure(greenLight: greenLight, error: invalidError, remoteClientsAndTabs: remoteClientsAndTabs)
+                        } >>> {
+                            return deferMaybe(result)
+                        }
+                    }
+                }
+                return deferMaybe(result)
+            } >>== { result in
+                guard AppConstants.MOZ_SIMPLE_BOOKMARKS_SYNCING else {
+                    return deferMaybe(result)
+                }
+                guard case .completed = result else {
+                    return deferMaybe(result)
+                }
+
+                // -1 because we also need to upload the mobile root.
+                return (storage.getLocalBookmarksModifications(limit: bookmarksClient.maxBatchPostRecords - 1) >>== { (deletedGUIDs, newBookmarks) -> Success in
+                    guard newBookmarks.count > 0 || deletedGUIDs.count > 0 else {
+                        return succeed()
+                    }
+                    return self.buildMobileRootAndChildrenRecords(storage, buffer, additionalChildren: newBookmarks, deletedChildren: deletedGUIDs) >>== { (mobileRootRecord, childrenRecords) in
+                        return self.uploadSomeLocalRecords(storage, mirrorer, bookmarksClient, mobileRootRecord: mobileRootRecord, childrenRecords: childrenRecords)
+                    }
+                }).bind { simpleSyncingResult in
+                    if let failure = simpleSyncingResult.failureValue {
+                        let description = failure is RecordTooLargeError ? "Record too large" : failure.description
+                        Sentry.shared.send(message: "Failed to simple sync bookmarks", tag: SentryTag.bookmarks, severity: .error, description: description)
+                    }
+                    return deferMaybe(result)
+                }
+            }
+        } else {
+            run = doMirror >>== { result in
+                // Only bother trying to sync if the mirror operation wasn't interrupted or partial.
+                if case .completed = result {
+                    return buffer.validate().bind { result in
+                        if let invalidError = result.failureValue as? BufferInvalidError {
+                            return buffer.getUpstreamRecordCount().bind { checked in
+                                self.statsSession.validationStats = self.validationStatsFrom(error: invalidError, checked: checked)
+                                return self.maybeStartRepairProcedure(greenLight: greenLight, error: invalidError, remoteClientsAndTabs: remoteClientsAndTabs) >>> {
+                                    return deferMaybe(invalidError)
+                                }
+                            }
+                        }
+                        
+                        let applier = MergeApplier(buffer: buffer, storage: storage, client: storer, statsSession: self.statsSession, greenLight: greenLight)
+                        return applier.go()
+                    }
+                }
+                return deferMaybe(result)
+            }
+        }
+
+        run.upon { result in
+            let end = Date.nowMicroseconds()
+            let duration = end - start
+            log.info("Bookmark \(AppConstants.shouldMergeBookmarks ? "sync" : "mirroring") took \(duration)µs. Result was \(result.successValue?.description ?? result.failureValue?.description ?? "failure")")
+        }
+
+        return run
+    }
+
+    private func validationStatsFrom(error: BufferInvalidError, checked: Int?) -> ValidationStats {
+        let problems = error.inconsistencies.map { ValidationProblem(name: $0.trackingEvent, count: $1.count) }
+        return ValidationStats(problems: problems, took: error.validationDuration, checked: checked)
+    }
+
+    private func maybeStartRepairProcedure(greenLight: () -> Bool, error: BufferInvalidError, remoteClientsAndTabs: RemoteClientsAndTabs) -> Success {
+        guard AppConstants.MOZ_BOOKMARKS_REPAIR_REQUEST && greenLight() else {
+            return succeed()
+        }
+        log.warning("Buffer inconsistent, starting repair procedure")
+        let repairer = BookmarksRepairRequestor(scratchpad: self.scratchpad, basePrefs: self.basePrefs, remoteClients: remoteClientsAndTabs)
+        return repairer.startRepairs(validationInfo: error.inconsistencies).bind { result in
+            if let repairFailure = result.failureValue {
+                Sentry.shared.send(message: "Bookmarks repair failure", tag: SentryTag.bookmarks, severity: .error, description: repairFailure.description)
+            } else {
+                Sentry.shared.send(message: "Bookmarks repair succeeded", tag: SentryTag.bookmarks, severity: .debug)
+            }
+            return succeed()
+        }
+    }
+}
+
+class MergeApplier {
+    let greenLight: () -> Bool
+    let buffer: BookmarkBufferStorage
+    let storage: SyncableBookmarks
+    let client: BookmarkStorer
+    let merger: BookmarksStorageMerger
+    let statsSession: SyncEngineStatsSession
+
+    init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource, client: BookmarkStorer, statsSession: SyncEngineStatsSession, greenLight: @escaping () -> Bool) {
+        self.greenLight = greenLight
+        self.buffer = buffer
+        self.storage = storage
+        self.merger = ThreeWayBookmarksStorageMerger(buffer: buffer, storage: storage)
+        self.client = client
+        self.statsSession = statsSession
+    }
+
+    // Exposed for use from tests.
+    func applyResult(_ result: BookmarksMergeResult) -> Success {
+        return result.applyToClient(self.client, storage: self.storage, buffer: self.buffer)
+    }
+
+    func go() -> SyncResult {
+        guard self.greenLight() else {
+            log.info("Green light turned red; not merging bookmarks.")
+            return deferMaybe(SyncStatus.completed(statsSession.end()))
+        }
+
+        return self.merger.merge()
+          >>== self.applyResult
+           >>> always(SyncStatus.completed(statsSession.end()))
+    }
+}
+
+/**
+ * The merger takes as input an existing storage state (mirror and local override),
+ * a buffer of new incoming records that relate to the mirror, and performs a three-way
+ * merge.
+ *
+ * The merge itself does not mutate storage. The result of the merge is conceptually a
+ * tuple: a new mirror state, a set of reconciled + locally changed records to upload,
+ * and two checklists of buffer and local state to discard.
+ *
+ * Typically the merge will be complete, resulting in a new mirror state, records to
+ * upload, and completely emptied buffer and local. In the case of partial inconsistency
+ * this will not be the case; incomplete subtrees will remain in the buffer. (We don't
+ * expect local subtrees to ever be incomplete.)
+ *
+ * It is expected that the caller will immediately apply the result in this order:
+ *
+ * 1. Upload the remote changes, if any. If this fails we can retry the entire process.
+ *
+ * 2(a). Apply the local changes, if any. If this fails we will re-download the records
+ *       we just uploaded, and should reach the same end state.
+ *       This step takes a timestamp key from (1), because pushing a record into the mirror
+ *       requires a server timestamp.
+ *
+ * 2(b). Switch to the new mirror state. If this fails, we should find that our reconciled
+ *       server contents apply neatly to our mirror and empty local, and we'll reach the
+ *       same end state.
+ *
+ *       Mirror state is applied in a sane order to respect relational constraints, even though
+ *       we configure sqlite to defer constraint validation until the transaction is committed.
+ *       That means:
+ *
+ *       - Add any new records in the value table.
+ *       - Change any existing records in the value table.
+ *       - Update structure.
+ *       - Remove records from the value table.
+ *
+ * 3. Apply buffer changes. We only do this after the mirror has advanced; if we fail to
+ *    clean up the buffer, it'll reconcile neatly with the mirror on a subsequent try.
+ *
+ * 4. Update bookkeeping timestamps. If this fails we will download uploaded records,
+ *    find they match, and have no repeat merging work to do.
+ *
+ * The goal of merging is that the buffer is empty (because we reconciled conflicts and
+ * updated the server), our local overlay is empty (because we reconciled conflicts and
+ * applied our changes to the server), and the mirror matches the server.
+ *
+ * Note that upstream application is robust: we can use XIUS to ensure that writes don't
+ * race. Buffer application is similarly robust, because this code owns all writes to the
+ * buffer. Local and mirror application, however, is not: the user's actions can cause
+ * changes to write to the database before we're done applying the results of a sync.
+ * We mitigate this a little by being explicit about the local changes that we're flushing
+ * (rather than, say, `DELETE FROM local`), but to do better we'd need change detection
+ * (e.g., an in-memory monotonic counter) or locking to prevent bookmark operations from
+ * racing. Later!
+ */
+protocol BookmarksStorageMerger: class {
+    init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource)
+    func merge() -> Deferred<Maybe<BookmarksMergeResult>>
+}
+
+class NoOpBookmarksMerger: BookmarksStorageMerger {
+    let buffer: BookmarkBufferStorage & BufferItemSource
+    let storage: SyncableBookmarks & LocalItemSource & MirrorItemSource
+
+    required init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource) {
+        self.buffer = buffer
+        self.storage = storage
+    }
+
+    func merge() -> Deferred<Maybe<BookmarksMergeResult>> {
+        return deferMaybe(BookmarksMergeResult.NoOp(ItemSources(local: self.storage, mirror: self.storage, buffer: self.buffer)))
+    }
+}
+
+class ThreeWayBookmarksStorageMerger: BookmarksStorageMerger {
+    let buffer: BookmarkBufferStorage & BufferItemSource
+    let storage: SyncableBookmarks & LocalItemSource & MirrorItemSource
+
+    required init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource) {
+        self.buffer = buffer
+        self.storage = storage
+    }
+
+    // MARK: - BookmarksStorageMerger.
+
+    // Trivial one-way sync.
+    fileprivate func applyLocalDirectlyToMirror() -> Deferred<Maybe<BookmarksMergeResult>> {
+        // Theoretically, we do the following:
+        // * Construct a virtual bookmark tree overlaying local on the mirror.
+        // * Walk the tree to produce Sync records.
+        // * Upload those records.
+        // * Flatten that tree into the mirror, clearing local.
+        //
+        // This is simpler than a full three-way merge: it's tree delta then flatten.
+        //
+        // But we are confident that our local changes, when overlaid on the mirror, are
+        // consistent. So we can take a little bit of a shortcut: process records
+        // directly, rather than building a tree.
+        //
+        // So do the following:
+        // * Take everything in `local` and turn it into a Sync record. This means pulling
+        //   folder hierarchies out of localStructure, values out of local, and turning
+        //   them into records. Do so in hierarchical order if we can, and set sortindex
+        //   attributes to put folders first.
+        // * Upload those records in as few batches as possible. Ensure that each batch
+        //   is consistent, if at all possible, though we're hoping for server support for
+        //   atomic writes.
+        // * Take everything in local that was successfully uploaded and move it into the
+        //   mirror, using the timestamps we tracked from the upload.
+        //
+        // Optionally, set 'again' to true in our response, and do this work only for a
+        // particular subtree (e.g., a single root, or a single branch of changes). This
+        // allows us to make incremental progress.
+
+        // TODO
+        log.debug("No special-case local-only merging yet. Falling back to three-way merge.")
+        return self.threeWayMerge()
+    }
+
+    fileprivate func applyIncomingDirectlyToMirror() -> Deferred<Maybe<BookmarksMergeResult>> {
+        // If the incoming buffer is consistent -- and the result of the mirrorer
+        // gives us a hint about that! -- then we can move the buffer records into
+        // the mirror directly.
+        //
+        // Note that this is also true for entire subtrees: if none of the children
+        // of, say, 'menu________' are modified locally, then we can apply it without
+        // merging.
+        //
+        // TODO
+        log.debug("No special-case remote-only merging yet. Falling back to three-way merge.")
+        return self.threeWayMerge()
+    }
+
+    // This is exposed for testing.
+    func getMerger() -> Deferred<Maybe<ThreeWayTreeMerger>> {
+        return self.storage.treesForEdges() >>== { (local, remote) in
+            // At this point *might* have two empty trees. This should only be the case if
+            // there are value-only changes (e.g., a renamed bookmark).
+            // We don't fail in that case, but we could optimize here.
+
+            // Find the mirror tree so we can compare.
+            return self.storage.treeForMirror() >>== { mirror in
+                // At this point we know that there have been changes both locally and remotely.
+                // (Or, in the general case, changes either locally or remotely.)
+
+                let itemSources = ItemSources(local: CachingLocalItemSource(source: self.storage), mirror: CachingMirrorItemSource(source: self.storage), buffer: CachingBufferItemSource(source: self.buffer))
+                return deferMaybe(ThreeWayTreeMerger(local: local, mirror: mirror, remote: remote, itemSources: itemSources))
+            }
+        }
+    }
+
+    func getMergedTree() -> Deferred<Maybe<MergedTree>> {
+        return self.getMerger() >>== { $0.produceMergedTree() }
+    }
+
+    func threeWayMerge() -> Deferred<Maybe<BookmarksMergeResult>> {
+        return self.getMerger() >>== { $0.produceMergedTree() >>== $0.produceMergeResultFromMergedTree }
+    }
+
+    func merge() -> Deferred<Maybe<BookmarksMergeResult>> {
+        return self.buffer.isEmpty() >>== { noIncoming in
+
+            // TODO: the presence of empty desktop roots in local storage
+            // isn't something we really need to worry about. Can we skip it here?
+
+            return self.storage.isUnchanged() >>== { noOutgoing in
+                switch (noIncoming, noOutgoing) {
+                case (true, true):
+                    // Nothing to do!
+                    log.debug("No incoming and no outgoing records: no-op.")
+                    return deferMaybe(BookmarksMergeResult.NoOp(ItemSources(local: self.storage, mirror: self.storage, buffer: self.buffer)))
+                case (true, false):
+                    // No incoming records to apply. Unilaterally apply local changes.
+                    return self.applyLocalDirectlyToMirror()
+                case (false, true):
+                    // No outgoing changes. Unilaterally apply remote changes if they're consistent.
+                    return self.buffer.validate() >>> self.applyIncomingDirectlyToMirror
+                default:
+                    // Changes on both sides. Merge.
+                    return self.buffer.validate() >>> self.threeWayMerge
+                }
+            }
+        }
+    }
+}

mobile/ios/Sync/Synchronizers/Bookmarks/Merging.swift

@@ -0,0 +1,336 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Deferred
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+
+private let log = Logger.syncLogger
+
+// Because generic protocols in Swift are a pain in the ass.
+public protocol BookmarkStorer: class {
+    // TODO: this should probably return a timestamp.
+    func applyUpstreamCompletionOp(_ op: UpstreamCompletionOp, itemSources: ItemSources, trackingTimesInto local: LocalOverrideCompletionOp) -> Deferred<Maybe<POSTResult>>
+}
+
+open class UpstreamCompletionOp: PerhapsNoOp {
+    // Upload these records from the buffer, but with these child lists.
+    open var amendChildrenFromBuffer: [GUID: [GUID]] = [:]
+
+    // Upload these records from the mirror, but with these child lists.
+    open var amendChildrenFromMirror: [GUID: [GUID]] = [:]
+
+    // Upload these records from local, but with these child lists.
+    open var amendChildrenFromLocal: [GUID: [GUID]] = [:]
+
+    // Upload these records as-is.
+    open var records: [Record<BookmarkBasePayload>] = []
+
+    open let ifUnmodifiedSince: Timestamp?
+
+    open var isNoOp: Bool {
+        return records.isEmpty
+    }
+
+    public init(ifUnmodifiedSince: Timestamp?=nil) {
+        self.ifUnmodifiedSince = ifUnmodifiedSince
+    }
+}
+
+open class BookmarksMergeResult: PerhapsNoOp {
+    let uploadCompletion: UpstreamCompletionOp
+    let overrideCompletion: LocalOverrideCompletionOp
+    let bufferCompletion: BufferCompletionOp
+    let itemSources: ItemSources
+
+    open var isNoOp: Bool {
+        return self.uploadCompletion.isNoOp &&
+               self.overrideCompletion.isNoOp &&
+               self.bufferCompletion.isNoOp
+    }
+
+    func applyToClient(_ client: BookmarkStorer, storage: SyncableBookmarks, buffer: BookmarkBufferStorage) -> Success {
+        return client.applyUpstreamCompletionOp(self.uploadCompletion, itemSources: self.itemSources, trackingTimesInto: self.overrideCompletion)
+         >>> { storage.applyLocalOverrideCompletionOp(self.overrideCompletion, itemSources: self.itemSources) }
+         >>> { buffer.applyBufferCompletionOp(self.bufferCompletion, itemSources: self.itemSources) }
+    }
+
+    init(uploadCompletion: UpstreamCompletionOp, overrideCompletion: LocalOverrideCompletionOp, bufferCompletion: BufferCompletionOp, itemSources: ItemSources) {
+        self.uploadCompletion = uploadCompletion
+        self.overrideCompletion = overrideCompletion
+        self.bufferCompletion = bufferCompletion
+        self.itemSources = itemSources
+    }
+
+    static func NoOp(_ itemSources: ItemSources) -> BookmarksMergeResult {
+        return BookmarksMergeResult(uploadCompletion: UpstreamCompletionOp(), overrideCompletion: LocalOverrideCompletionOp(), bufferCompletion: BufferCompletionOp(), itemSources: itemSources)
+    }
+}
+
+// MARK: - Errors.
+
+open class BookmarksMergeError: MaybeErrorType, SyncPingFailureFormattable {
+    fileprivate let error: Error?
+
+    init(error: Error?=nil) {
+        self.error = error
+    }
+
+    open var description: String {
+        return "Merge error: \(self.error ??? "nil")"
+    }
+
+    open var failureReasonName: SyncPingFailureReasonName {
+        return .otherError
+    }
+}
+
+open class BookmarksMergeConsistencyError: BookmarksMergeError {
+    override open var description: String {
+        return "Merge consistency error"
+    }
+}
+
+open class BookmarksMergeErrorTreeIsUnrooted: BookmarksMergeConsistencyError {
+    open let roots: Set<GUID>
+
+    public init(roots: Set<GUID>) {
+        self.roots = roots
+    }
+
+    override open var description: String {
+        return "Tree is unrooted: roots are \(self.roots)"
+    }
+}
+
+enum MergeState<T> {
+    case unknown              // Default state.
+    case unchanged            // Nothing changed: no work needed.
+    case remote               // Take the associated remote value.
+    case local                // Take the associated local value.
+    case new(value: T)        // Take this synthesized value.
+
+    var isUnchanged: Bool {
+        if case .unchanged = self {
+            return true
+        }
+        return false
+    }
+
+    var isUnknown: Bool {
+        if case .unknown = self {
+            return true
+        }
+        return false
+    }
+
+    var label: String {
+        switch self {
+        case .unknown:
+            return "Unknown"
+        case .unchanged:
+            return "Unchanged"
+        case .remote:
+            return "Remote"
+        case .local:
+            return "Local"
+        case .new:
+            return "New"
+        }
+    }
+}
+
+func ==<T: Equatable>(lhs: MergeState<T>, rhs: MergeState<T>) -> Bool {
+    switch (lhs, rhs) {
+    case (.unknown, .unknown):
+        return true
+    case (.unchanged, .unchanged):
+        return true
+    case (.remote, .remote):
+        return true
+    case (.local, .local):
+        return true
+    case let (.new(lh), .new(rh)):
+        return lh == rh
+    default:
+        return false
+    }
+}
+
+/**
+ * Using this:
+ *
+ * You get one for the root. Then you give it children for the roots
+ * from the mirror.
+ *
+ * Then you walk those, populating the remote and local nodes by looking
+ * at the left/right trees.
+ *
+ * By comparing left and right, and doing value-based comparisons if necessary,
+ * a merge state is decided and assigned for both value and structure.
+ *
+ * One then walks both left and right child structures (both to ensure that
+ * all nodes on both left and right will be visited!) recursively.
+ */
+class MergedTreeNode {
+    let guid: GUID
+    let mirror: BookmarkTreeNode?
+    var remote: BookmarkTreeNode?
+    var local: BookmarkTreeNode?
+
+    var hasLocal: Bool { return self.local != nil }
+    var hasMirror: Bool { return self.mirror != nil }
+    var hasRemote: Bool { return self.remote != nil }
+
+    var valueState: MergeState<BookmarkMirrorItem> = MergeState.unknown
+    var structureState: MergeState<BookmarkTreeNode> = MergeState.unknown
+
+    var hasDecidedChildren: Bool {
+        return !self.structureState.isUnknown
+    }
+
+    var mergedChildren: [MergedTreeNode]?
+
+    // One-sided constructors.
+    static func forRemote(_ remote: BookmarkTreeNode, mirror: BookmarkTreeNode?=nil) -> MergedTreeNode {
+        let n = MergedTreeNode(guid: remote.recordGUID, mirror: mirror, structureState: MergeState.remote)
+        n.remote = remote
+        n.valueState = MergeState.remote
+        return n
+    }
+
+    static func forLocal(_ local: BookmarkTreeNode, mirror: BookmarkTreeNode?=nil) -> MergedTreeNode {
+        let n = MergedTreeNode(guid: local.recordGUID, mirror: mirror, structureState: MergeState.local)
+        n.local = local
+        n.valueState = MergeState.local
+        return n
+    }
+
+    static func forUnchanged(_ mirror: BookmarkTreeNode) -> MergedTreeNode {
+        let n = MergedTreeNode(guid: mirror.recordGUID, mirror: mirror, structureState: MergeState.unchanged)
+        n.valueState = MergeState.unchanged
+        return n
+    }
+
+    init(guid: GUID, mirror: BookmarkTreeNode?, structureState: MergeState<BookmarkTreeNode>) {
+        self.guid = guid
+        self.mirror = mirror
+        self.structureState = structureState
+    }
+
+    init(guid: GUID, mirror: BookmarkTreeNode?) {
+        self.guid = guid
+        self.mirror = mirror
+    }
+
+    // N.B., you cannot recurse down `decidedStructure`: you'll depart from the
+    // merged tree. You need to use `mergedChildren` instead.
+    fileprivate var decidedStructure: BookmarkTreeNode? {
+        switch self.structureState {
+        case .unknown:
+            return nil
+        case .unchanged:
+            return self.mirror
+        case .remote:
+            return self.remote
+        case .local:
+            return self.local
+        case let .new(node):
+            return node
+        }
+    }
+
+    func asUnmergedTreeNode() -> BookmarkTreeNode {
+        return self.decidedStructure ?? BookmarkTreeNode.unknown(guid: self.guid)
+    }
+
+    // Recursive. Starts returning Unknown when nodes haven't been processed.
+    func asMergedTreeNode() -> BookmarkTreeNode {
+        guard let decided = self.decidedStructure,
+              let merged = self.mergedChildren else {
+            return BookmarkTreeNode.unknown(guid: self.guid)
+        }
+
+        if case .folder = decided {
+            let children = merged.map { $0.asMergedTreeNode() }
+            return BookmarkTreeNode.folder(guid: self.guid, children: children)
+        }
+
+        return decided
+    }
+
+    var isFolder: Bool {
+        return self.mergedChildren != nil
+    }
+
+    func dump(_ indent: Int) {
+        precondition(indent < 200)
+        let r: Character = "R"
+        let l: Character = "L"
+        let m: Character = "M"
+        let ind = indenting(indent)
+        print(ind, "[V: ", box(self.remote, r), box(self.mirror, m), box(self.local, l), self.guid, self.valueState.label, "]")
+        guard self.isFolder else {
+            return
+        }
+
+        print(ind, "[S: ", self.structureState.label, "]")
+        if let children = self.mergedChildren {
+            print(ind, "  ..")
+            for child in children {
+                child.dump(indent + 2)
+            }
+        }
+    }
+}
+
+private func box<T>(_ x: T?, _ c: Character) -> Character {
+    if x == nil {
+        return "□"
+    }
+    return c
+}
+
+private func indenting(_ by: Int) -> String {
+    return String(repeating: " ", count: by)
+}
+
+class MergedTree {
+    var root: MergedTreeNode
+    var deleteLocally: Set<GUID> = Set()
+    var deleteRemotely: Set<GUID> = Set()
+    var deleteFromMirror: Set<GUID> = Set()
+    var acceptLocalDeletion: Set<GUID> = Set()
+    var acceptRemoteDeletion: Set<GUID> = Set()
+
+    var allGUIDs: Set<GUID> {
+        var out = Set<GUID>([self.root.guid])
+        func acc(_ node: MergedTreeNode) {
+            guard let children = node.mergedChildren else {
+                return
+            }
+            out.formUnion(Set(children.map { $0.guid }))
+            children.forEach(acc)
+        }
+        acc(self.root)
+        return out
+    }
+
+    init(mirrorRoot: BookmarkTreeNode) {
+        self.root = MergedTreeNode(guid: mirrorRoot.recordGUID, mirror: mirrorRoot, structureState: MergeState.unchanged)
+        self.root.valueState = MergeState.unchanged
+    }
+
+    func dump() {
+        print("Deleted locally: \(self.deleteLocally.joined(separator: ", "))")
+        print("Deleted remotely: \(self.deleteRemotely.joined(separator: ", "))")
+        print("Deleted from mirror: \(self.deleteFromMirror.joined(separator: ", "))")
+        print("Accepted local deletions: \(self.acceptLocalDeletion.joined(separator: ", "))")
+        print("Accepted remote deletions: \(self.acceptRemoteDeletion.joined(separator: ", "))")
+        print("Root: ")
+        self.root.dump(0)
+    }
+}

mobile/ios/Sync/Synchronizers/ClientsSynchronizer.swift

@@ -0,0 +1,432 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+let ClientsStorageVersion = 1
+
+// TODO
+public protocol Command {
+    static func fromName(_ command: String, args: [JSON]) -> Command?
+    func run(_ synchronizer: ClientsSynchronizer) -> Success
+    static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command?
+}
+
+// Shit.
+// We need a way to wipe or reset engines.
+// We need a way to log out the account.
+// So when we sync commands, we're gonna need a delegate of some kind.
+open class WipeCommand: Command {
+
+    public init?(command: String, args: [JSON]) {
+        return nil
+    }
+
+    open class func fromName(_ command: String, args: [JSON]) -> Command? {
+        return WipeCommand(command: command, args: args)
+    }
+
+    open func run(_ synchronizer: ClientsSynchronizer) -> Success {
+        return succeed()
+    }
+
+    open static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command? {
+        let json = JSON(parseJSON: syncCommand.value)
+        if let name = json["command"].string,
+            let args = json["args"].array {
+                return WipeCommand.fromName(name, args: args)
+        }
+        return nil
+    }
+}
+
+open class DisplayURICommand: Command {
+    let uri: URL
+    let title: String
+    let sender: String
+
+    public init?(command: String, args: [JSON]) {
+        if let uri = args[0].string?.asURL,
+            let sender = args[1].string,
+            let title = args[2].string {
+            self.uri = uri
+            self.sender = sender
+            self.title = title
+        } else {
+            // Oh, Swift.
+            self.uri = "http://localhost/".asURL!
+            self.title = ""
+            return nil
+        }
+    }
+
+    open class func fromName(_ command: String, args: [JSON]) -> Command? {
+        return DisplayURICommand(command: command, args: args)
+    }
+
+    open func run(_ synchronizer: ClientsSynchronizer) -> Success {
+        func display(_ deviceName: String? = nil) -> Success {
+            synchronizer.delegate.displaySentTab(for: uri, title: title, from: deviceName)
+            return succeed()
+        }
+
+        guard let getClientWithId = synchronizer.localClients?.getClientWithId(sender) else {
+            return display()
+        }
+
+        return getClientWithId >>== { client in
+            return display(client?.name)
+        }
+    }
+
+    open static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command? {
+        let json = JSON(parseJSON: syncCommand.value)
+        if let name = json["command"].string,
+            let args = json["args"].array {
+                return DisplayURICommand.fromName(name, args: args)
+        }
+        return nil
+    }
+}
+
+open class RepairResponseCommand: Command {
+    let repairResponse: RepairResponse
+
+    public init(command: String, args: [JSON]) {
+        self.repairResponse = RepairResponse.fromJSON(args: args[0])
+    }
+
+    open class func fromName(_ command: String, args: [JSON]) -> Command? {
+        return RepairResponseCommand(command: command, args: args)
+    }
+
+    open func run(_ synchronizer: ClientsSynchronizer) -> Success {
+        let repairer = BookmarksRepairRequestor(scratchpad: synchronizer.scratchpad, basePrefs: synchronizer.basePrefs, remoteClients: synchronizer.localClients!)
+        return repairer.continueRepairs(response: self.repairResponse) >>> succeed
+    }
+
+    open static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command? {
+        let json = JSON(parseJSON: syncCommand.value)
+        if let name = json["command"].string,
+            let args = json["args"].array {
+            return RepairResponseCommand.fromName(name, args: args)
+        }
+        return nil
+    }
+}
+
+let Commands: [String: (String, [JSON]) -> Command?] = [
+    "wipeAll": WipeCommand.fromName,
+    "wipeEngine": WipeCommand.fromName,
+    // resetEngine
+    // resetAll
+    // logout
+    "displayURI": DisplayURICommand.fromName,
+    "repairResponse": RepairResponseCommand.fromName
+]
+
+open class ClientsSynchronizer: TimestampedSingleCollectionSynchronizer, Synchronizer {
+    public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
+        super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "clients")
+    }
+
+    var localClients: RemoteClientsAndTabs?
+
+    override var storageVersion: Int {
+        return ClientsStorageVersion
+    }
+
+    var clientRecordLastUpload: Timestamp {
+        set(value) {
+            self.prefs.setLong(value, forKey: "lastClientUpload")
+        }
+
+        get {
+            return self.prefs.unsignedLongForKey("lastClientUpload") ?? 0
+        }
+    }
+
+    // Sync Object Format (Version 1) for Form Factors: http://docs.services.mozilla.com/sync/objectformats.html#id2
+    fileprivate enum SyncFormFactorFormat: String {
+        case phone = "phone"
+        case tablet = "tablet"
+    }
+
+    open func getOurClientRecord() -> Record<ClientPayload> {
+        let guid = self.scratchpad.clientGUID
+        let formfactor = formFactorString()
+        
+        let json = JSON(object: [
+            "id": guid,
+            "fxaDeviceId": self.scratchpad.fxaDeviceId,
+            "version": AppInfo.appVersion,
+            "protocols": ["1.5"],
+            "name": self.scratchpad.clientName,
+            "os": "iOS",
+            "commands": [JSON](),
+            "type": "mobile",
+            "appPackage": AppInfo.baseBundleIdentifier,
+            "application": AppInfo.displayName,
+            "device": DeviceInfo.deviceModel(),
+            "formfactor": formfactor])
+
+        let payload = ClientPayload(json)
+        return Record(id: guid, payload: payload, ttl: ThreeWeeksInSeconds)
+    }
+
+    fileprivate func formFactorString() -> String {
+        let userInterfaceIdiom = UIDevice.current.userInterfaceIdiom
+        var formfactor: String
+        
+        switch userInterfaceIdiom {
+        case .phone:
+            formfactor = SyncFormFactorFormat.phone.rawValue
+        case .pad:
+            formfactor = SyncFormFactorFormat.tablet.rawValue
+        default:
+            formfactor = SyncFormFactorFormat.phone.rawValue
+        }
+        
+        return formfactor
+    }
+
+    fileprivate func clientRecordToLocalClientEntry(_ record: Record<ClientPayload>) -> RemoteClient {
+        let modified = record.modified
+        let payload = record.payload
+        return RemoteClient(json: payload.json, modified: modified)
+    }
+
+    // If this is a fresh start, do a wipe.
+    // N.B., we don't wipe outgoing commands! (TODO: check this when we implement commands!)
+    // N.B., but perhaps we should discard outgoing wipe/reset commands!
+    fileprivate func wipeIfNecessary(_ localClients: RemoteClientsAndTabs) -> Success {
+        if self.lastFetched == 0 {
+            return localClients.wipeClients()
+        }
+        return succeed()
+    }
+
+    /**
+     * Returns whether any commands were found (and thus a replacement record
+     * needs to be uploaded). Also returns the commands: we run them after we
+     * upload a replacement record.
+     */
+    fileprivate func processCommandsFromRecord(_ record: Record<ClientPayload>?, withServer storageClient: Sync15CollectionClient<ClientPayload>) -> Deferred<Maybe<(Bool, [Command])>> {
+        log.debug("Processing commands from downloaded record.")
+
+        // TODO: short-circuit based on the modified time of the record we uploaded, so we don't need to skip ahead.
+        if let record = record {
+            let commands = record.payload.commands
+            if !commands.isEmpty {
+                func parse(_ json: JSON) -> Command? {
+                    if let name = json["command"].string,
+                        let args = json["args"].array,
+                        let constructor = Commands[name] {
+                            return constructor(name, args)
+                    }
+                    return nil
+                }
+
+                // TODO: can we do anything better if a command fails?
+                return deferMaybe((true, optFilter(commands.map(parse))))
+            }
+        }
+
+        return deferMaybe((false, []))
+    }
+
+    fileprivate func uploadClientCommands(toLocalClients localClients: RemoteClientsAndTabs, withServer storageClient: Sync15CollectionClient<ClientPayload>) -> Success {
+        return localClients.getCommands() >>== { clientCommands in
+            return clientCommands.map { (clientGUID, commands) -> Success in
+                self.syncClientCommands(clientGUID, commands: commands, clientsAndTabs: localClients, withServer: storageClient)
+            }.allSucceed()
+        }
+    }
+
+    fileprivate func syncClientCommands(_ clientGUID: GUID, commands: [SyncCommand], clientsAndTabs: RemoteClientsAndTabs, withServer storageClient: Sync15CollectionClient<ClientPayload>) -> Success {
+
+        let deleteCommands: () -> Success = {
+            return clientsAndTabs.deleteCommands(clientGUID).bind({ x in return succeed() })
+        }
+
+        log.debug("Fetching current client record for client \(clientGUID).")
+        let fetch = storageClient.get(clientGUID)
+        return fetch.bind() { result in
+            if let response = result.successValue, response.value.payload.isValid() {
+                let record = response.value
+                if var clientRecord = record.payload.json.dictionary {
+                    clientRecord["commands"] = JSON(record.payload.commands + commands.map { JSON(parseJSON: $0.value) })
+                    let uploadRecord = Record(id: clientGUID, payload: ClientPayload(JSON(clientRecord)), ttl: ThreeWeeksInSeconds)
+                    return storageClient.put(uploadRecord, ifUnmodifiedSince: record.modified)
+                        >>== { resp in
+                            log.debug("Client \(clientGUID) commands upload succeeded.")
+
+                            // Always succeed, even if we couldn't delete the commands.
+                            return deleteCommands()
+                    }
+                }
+            } else {
+                if let failure = result.failureValue {
+                    log.warning("Failed to fetch record with GUID \(clientGUID).")
+                    if failure is NotFound<HTTPURLResponse> {
+                        log.debug("Not waiting to see if the client comes back.")
+
+                        // TODO: keep these around and retry, expiring after a while.
+                        // For now we just throw them away so we don't fail every time.
+                        return deleteCommands()
+                    }
+
+                    if failure is BadRequestError<HTTPURLResponse> {
+                        log.debug("We made a bad request. Throwing away queued commands.")
+                        return deleteCommands()
+                    }
+                }
+            }
+
+            log.error("Client \(clientGUID) commands upload failed: No remote client for GUID")
+            return deferMaybe(UnknownError())
+        }
+    }
+
+    /**
+     * Upload our record if either (a) we know we should upload, or (b)
+     * our own notes tell us we're due to reupload.
+     */
+    fileprivate func maybeUploadOurRecord(_ should: Bool, ifUnmodifiedSince: Timestamp?, toServer storageClient: Sync15CollectionClient<ClientPayload>) -> Success {
+
+        let lastUpload = self.clientRecordLastUpload
+        let expired = lastUpload < (Date.now() - (2 * OneDayInMilliseconds))
+        log.debug("Should we upload our client record? Caller = \(should), expired = \(expired).")
+        if !should && !expired {
+            return succeed()
+        }
+
+        let iUS: Timestamp? = ifUnmodifiedSince ?? ((lastUpload == 0) ? nil : lastUpload)
+
+        var uploadStats = SyncUploadStats()
+        return storageClient.put(getOurClientRecord(), ifUnmodifiedSince: iUS)
+            >>== { resp in
+                if let ts = resp.metadata.lastModifiedMilliseconds {
+                    // Protocol says this should always be present for success responses.
+                    log.debug("Client record upload succeeded. New timestamp: \(ts).")
+                    self.clientRecordLastUpload = ts
+                    uploadStats.sent += 1
+                } else {
+                    uploadStats.sentFailed += 1
+                }
+                self.statsSession.recordUpload(stats: uploadStats)
+                return succeed()
+        }
+    }
+
+    fileprivate func applyStorageResponse(_ response: StorageResponse<[Record<ClientPayload>]>, toLocalClients localClients: RemoteClientsAndTabs, withServer storageClient: Sync15CollectionClient<ClientPayload>, notifier: CollectionChangedNotifier?) -> Success {
+        log.debug("Applying clients response.")
+
+        var downloadStats = SyncDownloadStats()
+
+        let records = response.value
+        let responseTimestamp = response.metadata.lastModifiedMilliseconds
+
+        log.debug("Got \(records.count) client records.")
+
+        let ourGUID = self.scratchpad.clientGUID
+        var toInsert = [RemoteClient]()
+        var ours: Record<ClientPayload>? = nil
+
+        for (rec) in records {
+            guard rec.payload.isValid() else {
+                log.warning("Client record \(rec.id) is invalid. Skipping.")
+                continue
+            }
+
+            if rec.id == ourGUID {
+                if rec.modified == self.clientRecordLastUpload {
+                    log.debug("Skipping our own unmodified record.")
+                } else {
+                    log.debug("Saw our own record in response.")
+                    ours = rec
+                }
+            } else {
+                toInsert.append(self.clientRecordToLocalClientEntry(rec))
+            }
+        }
+
+        downloadStats.applied += toInsert.count
+
+        // Apply remote changes.
+        // Collect commands from our own record and reupload if necessary.
+        // Then run the commands and return.
+        return localClients.insertOrUpdateClients(toInsert)
+            >>== { succeeded in
+                downloadStats.succeeded += succeeded
+                downloadStats.failed += (toInsert.count - succeeded)
+                self.statsSession.recordDownload(stats: downloadStats)
+                return succeed()
+            }
+            >>== { self.processCommandsFromRecord(ours, withServer: storageClient) }
+            >>== { (shouldUpload, commands) in
+                let isFirstSync = self.lastFetched == 0
+                return self.maybeUploadOurRecord(shouldUpload || self.why == .didLogin, ifUnmodifiedSince: ours?.modified, toServer: storageClient)
+                    >>> { self.uploadClientCommands(toLocalClients: localClients, withServer: storageClient) }
+                    >>> {
+                        log.debug("Running \(commands.count) commands.")
+                        for command in commands {
+                            _ = command.run(self)
+                        }
+                        self.lastFetched = responseTimestamp!
+                        if isFirstSync,
+                           let notifier = notifier {
+                            DispatchQueue.global(qos: DispatchQoS.background.qosClass).async { _ = notifier.notifyAll(collectionsChanged: ["clients"], reason: "firstsync") }
+                        }
+                        return succeed()
+                }
+        }
+    }
+
+    open func synchronizeLocalClients(_ localClients: RemoteClientsAndTabs, withServer storageClient: Sync15StorageClient, info: InfoCollections, notifier: CollectionChangedNotifier?) -> SyncResult {
+        log.debug("Synchronizing clients.")
+        self.localClients = localClients // Store for later when we process a repairResponse command
+
+        if let reason = self.reasonToNotSync(storageClient) {
+            switch reason {
+            case .engineRemotelyNotEnabled:
+                // This is a hard error for us.
+                return deferMaybe(FatalError(message: "clients not mentioned in meta/global. Server wiped?"))
+            default:
+                return deferMaybe(SyncStatus.notStarted(reason))
+            }
+        }
+
+        let keys = self.scratchpad.keys?.value
+        let encoder = RecordEncoder<ClientPayload>(decode: { ClientPayload($0) }, encode: { $0.json })
+        let encrypter = keys?.encrypter(self.collection, encoder: encoder)
+        if encrypter == nil {
+            log.error("Couldn't make clients encrypter.")
+            return deferMaybe(FatalError(message: "Couldn't make clients encrypter."))
+        }
+
+        let clientsClient = storageClient.clientForCollection(self.collection, encrypter: encrypter!)
+
+        // TODO: some of the commands we process might involve wiping collections or the
+        // entire profile. We should model this as an explicit status, and return it here
+        // instead of .completed.
+        statsSession.start()
+        // XXX: This is terrible. We always force a re-sync of the clients to work around
+        // the fact that `fxaDeviceId` may not have been populated if the list of clients
+        // hadn't changed since before the update to v8.0. To force a re-sync, we get all
+        // clients since the beginning of time instead of looking at `self.lastFetched`.
+        return clientsClient.getSince(0)
+            >>== { response in
+                return self.wipeIfNecessary(localClients)
+                    >>> { self.applyStorageResponse(response, toLocalClients: localClients, withServer: clientsClient, notifier: notifier) }
+            }
+            >>> { deferMaybe(self.completedWithStats) }
+    }
+}

mobile/ios/Sync/Synchronizers/Downloader.swift

@@ -0,0 +1,231 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+class BatchingDownloader<T: CleartextPayloadJSON> {
+    let client: Sync15CollectionClient<T>
+    let collection: String
+    let prefs: Prefs
+
+    var batch: [Record<T>] = []
+
+    func store(_ records: [Record<T>]) {
+        self.batch += records
+    }
+
+    func retrieve() -> [Record<T>] {
+        let ret = self.batch
+        self.batch = []
+        return ret
+    }
+
+    var _advance: (() -> Void)?
+    func advance() {
+        guard let f = self._advance else {
+            return
+        }
+        self._advance = nil
+        f()
+    }
+
+    init(collectionClient: Sync15CollectionClient<T>, basePrefs: Prefs, collection: String) {
+        self.client = collectionClient
+        self.collection = collection
+        let branchName = "downloader." + collection + "."
+        self.prefs = basePrefs.branch(branchName)
+
+        log.info("Downloader configured with prefs '\(self.prefs.getBranchPrefix())'.")
+    }
+
+    static func resetDownloaderWithPrefs(_ basePrefs: Prefs, collection: String) {
+        // This leads to stupid paths like 'profile.sync.synchronizer.history..downloader.history..'.
+        // Sorry, but it's out in the world now...
+        let branchName = "downloader." + collection + "."
+        let prefs = basePrefs.branch(branchName)
+
+        let lm = prefs.timestampForKey("lastModified")
+        let bt = prefs.timestampForKey("baseTimestamp")
+        log.debug("Resetting downloader prefs \(prefs.getBranchPrefix()). Previous values: \(lm ??? "nil"), \(bt ??? "nil").")
+
+        prefs.removeObjectForKey("nextOffset")
+        prefs.removeObjectForKey("offsetNewer")
+        prefs.removeObjectForKey("baseTimestamp")
+        prefs.removeObjectForKey("lastModified")
+    }
+
+    /**
+     * Clients should provide the same set of parameters alongside an `offset` as was
+     * provided with the initial request. The only thing that varies in our batch fetches
+     * is `newer`, so we track the original value alongside.
+     */
+    var nextFetchParameters: (String, Timestamp)? {
+        get {
+            let o = self.prefs.stringForKey("nextOffset")
+            let n = self.prefs.timestampForKey("offsetNewer")
+            guard let offset = o, let newer = n else {
+                return nil
+            }
+            return (offset, newer)
+        }
+        set (value) {
+            if let (offset, newer) = value {
+                self.prefs.setString(offset, forKey: "nextOffset")
+                self.prefs.setTimestamp(newer, forKey: "offsetNewer")
+            } else {
+                self.prefs.removeObjectForKey("nextOffset")
+                self.prefs.removeObjectForKey("offsetNewer")
+            }
+        }
+    }
+
+    // Set after each batch, from record timestamps.
+    var baseTimestamp: Timestamp {
+        get {
+            return self.prefs.timestampForKey("baseTimestamp") ?? 0
+        }
+        set (value) {
+            self.prefs.setTimestamp(value, forKey: "baseTimestamp")
+        }
+    }
+
+    // Only set at the end of a batch, from headers.
+    var lastModified: Timestamp {
+        get {
+            return self.prefs.timestampForKey("lastModified") ?? 0
+        }
+        set (value) {
+            self.prefs.setTimestamp(value, forKey: "lastModified")
+        }
+    }
+
+    /**
+     * Call this when a significant structural server change has been detected.
+     */
+    func reset() -> Success {
+        self.baseTimestamp = 0
+        self.lastModified = 0
+        self.nextFetchParameters = nil
+        self.batch = []
+        self._advance = nil
+        return succeed()
+    }
+
+    func go(_ info: InfoCollections, limit: Int) -> Deferred<Maybe<DownloadEndState>> {
+        guard let modified = info.modified(self.collection) else {
+            log.debug("No server modified time for collection \(self.collection).")
+            return deferMaybe(.noNewData)
+        }
+
+        log.debug("Modified: \(modified); last \(self.lastModified).")
+        if modified == self.lastModified {
+            log.debug("No more data to batch-download.")
+            return deferMaybe(.noNewData)
+        }
+
+        // If the caller hasn't advanced after the last batch, strange things will happen --
+        // potentially looping indefinitely. Warn.
+        if self._advance != nil && !self.batch.isEmpty {
+            log.warning("Downloading another batch without having advanced. This might be a bug.")
+        }
+        return self.downloadNextBatchWithLimit(limit, infoModified: modified)
+    }
+
+    func advanceTimestampTo(_ timestamp: Timestamp) {
+        log.debug("Advancing downloader lastModified from \(self.lastModified) to \(timestamp).")
+        self.lastModified = timestamp
+    }
+
+    // We're either fetching from our current base timestamp with no offset,
+    // or the timestamp we were using when we last saved an offset.
+    func fetchParameters() -> (String?, Timestamp) {
+        if let (offset, since) = self.nextFetchParameters {
+            return (offset, since)
+        }
+        return (nil, max(self.lastModified, self.baseTimestamp))
+    }
+
+    func downloadNextBatchWithLimit(_ limit: Int, infoModified: Timestamp) -> Deferred<Maybe<DownloadEndState>> {
+        let (offset, since) = self.fetchParameters()
+        log.debug("Fetching newer=\(since), offset=\(offset ?? "nil").")
+
+        let fetch = self.client.getSince(since, sort: SortOption.OldestFirst, limit: limit, offset: offset)
+
+        func handleFailure(_ err: MaybeErrorType) -> Deferred<Maybe<DownloadEndState>> {
+            log.debug("Handling failure.")
+            guard let badRequest = err as? BadRequestError<[Record<T>]>, badRequest.response.metadata.status == 412 else {
+                // Just pass through the failure.
+                return deferMaybe(err)
+            }
+
+            // Conflict. Start again.
+            log.warning("Server contents changed during offset-based batching. Stepping back.")
+            self.nextFetchParameters = nil
+            return deferMaybe(.interrupted)
+        }
+
+        func handleSuccess(_ response: StorageResponse<[Record<T>]>) -> Deferred<Maybe<DownloadEndState>> {
+            log.debug("Handling success.")
+            let nextOffset = response.metadata.nextOffset
+            let responseModified = response.value.last?.modified
+
+            // Queue up our metadata advance. We wait until the consumer has fetched
+            // and processed this batch; they'll call .advance() on success.
+            self._advance = {
+                // Shift to the next offset. This might be nil, in which case… fine!
+                // Note that we preserve the previous 'newer' value from the offset or the original fetch,
+                // even as we update baseTimestamp.
+                self.nextFetchParameters = nextOffset == nil ? nil : (nextOffset!, since)
+
+                // If there are records, advance to just before the timestamp of the last.
+                // If our next fetch with X-Weave-Next-Offset fails, at least we'll start here.
+                //
+                // This approach is only valid if we're fetching oldest-first.
+                if let newBase = responseModified {
+                    log.debug("Advancing baseTimestamp to \(newBase) - 1")
+                    self.baseTimestamp = newBase - 1
+                }
+
+                if nextOffset == nil {
+                    // If we can't get a timestamp from the header -- and we should always be able to --
+                    // we fall back on the collection modified time in i/c, as supplied by the caller.
+                    // In any case where there is no racing writer these two values should be the same.
+                    // If they differ, the header should be later. If it's missing, and we use the i/c
+                    // value, we'll simply redownload some records.
+                    // All bets are off if we hit this case and are filtering somehow… don't do that.
+                    let lm = response.metadata.lastModifiedMilliseconds
+                    log.debug("Advancing lastModified to \(String(describing: lm)) ?? \(infoModified).")
+                    self.lastModified = lm ?? infoModified
+                }
+            }
+
+            log.debug("Got success response with \(response.metadata.records ?? 0) records.")
+
+            // Store the incoming records for collection.
+            self.store(response.value)
+
+            return deferMaybe(nextOffset == nil ? .complete : .incomplete)
+        }
+
+        return fetch.bind { result in
+            guard let response = result.successValue else {
+                return handleFailure(result.failureValue!)
+            }
+            return handleSuccess(response)
+        }
+    }
+}
+
+public enum DownloadEndState: String {
+    case complete                         // We're done. Records are waiting for you.
+    case incomplete                       // applyBatch was called, and we think there are more records.
+    case noNewData                        // There were no records.
+    case interrupted                      // We got a 412 conflict when fetching the next batch.
+}

mobile/ios/Sync/Synchronizers/HistorySynchronizer.swift

@@ -0,0 +1,287 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+private let HistoryTTLInSeconds = 5184000                   // 60 days.
+let HistoryStorageVersion = 1
+
+func makeDeletedHistoryRecord(_ guid: GUID) -> Record<HistoryPayload> {
+    // Local modified time is ignored in upload serialization.
+    let modified: Timestamp = 0
+
+    // Sortindex for history is frecency. Make deleted items more frecent than almost
+    // anything.
+    let sortindex = 5_000_000
+
+    let ttl = HistoryTTLInSeconds
+
+    let json: JSON = JSON([
+        "id": guid,
+        "deleted": true,
+        ])
+    let payload = HistoryPayload(json)
+    return Record<HistoryPayload>(id: guid, payload: payload, modified: modified, sortindex: sortindex, ttl: ttl)
+}
+
+func makeHistoryRecord(_ place: Place, visits: [Visit]) -> Record<HistoryPayload> {
+    let id = place.guid
+    let modified: Timestamp = 0    // Ignored in upload serialization.
+    let sortindex = 1              // TODO: frecency!
+    let ttl = HistoryTTLInSeconds
+    let json: JSON = JSON([
+        "id": id,
+        "visits": visits.map { $0.toJSON() },
+        "histUri": place.url,
+        "title": place.title,
+        ])
+    let payload = HistoryPayload(json)
+    return Record<HistoryPayload>(id: id, payload: payload, modified: modified, sortindex: sortindex, ttl: ttl)
+}
+
+open class HistorySynchronizer: IndependentRecordSynchronizer, Synchronizer {
+    public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
+        super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "history")
+    }
+
+    override var storageVersion: Int {
+        return HistoryStorageVersion
+    }
+
+    fileprivate let batchSize: Int = 1000  // A balance between number of requests and per-request size.
+
+    fileprivate func mask(_ maxFailures: Int) -> (Maybe<()>) -> Success {
+        var failures = 0
+        return { result in
+            if result.isSuccess {
+                return Deferred(value: result)
+            }
+
+            failures += 1
+            if failures > maxFailures {
+                return Deferred(value: result)
+            }
+
+            log.debug("Masking failure \(failures).")
+            return succeed()
+        }
+    }
+
+    // TODO: this function should establish a transaction at suitable points.
+    // TODO: a much more efficient way to do this is to:
+    // 1. Start a transaction.
+    // 2. Try to update each place. Note failures.
+    // 3. bulkInsert all failed updates in one go.
+    // 4. Store all remote visits for all places in one go, constructing a single sequence of visits.
+    func applyIncomingToStorage(_ storage: SyncableHistory, records: [Record<HistoryPayload>]) -> Success {
+        // Skip over at most this many failing records before aborting the sync.
+        let maskSomeFailures = self.mask(3)
+
+        // TODO: it'd be nice to put this in an extension on SyncableHistory. Waiting for Swift 2.0...
+        func applyRecord(_ rec: Record<HistoryPayload>) -> Success {
+            let guid = rec.id
+            let payload = rec.payload
+            let modified = rec.modified
+
+            // We apply deletions immediately. Yes, this will throw away local visits
+            // that haven't yet been synced. That's how Sync works, alas.
+            if payload.deleted {
+                return storage.deleteByGUID(guid, deletedAt: modified).bind(maskSomeFailures)
+            }
+
+            // It's safe to apply other remote records, too -- even if we re-download, we know
+            // from our local cached server timestamp on each record that we've already seen it.
+            // We have to reconcile on-the-fly: we're about to overwrite the server record, which
+            // is our shared parent.
+            let place = rec.payload.asPlace()
+
+            if isIgnoredURL(place.url) {
+                log.debug("Ignoring incoming record \(guid) because its URL is one we wish to ignore.")
+                return succeed()
+            }
+
+            let placeThenVisits = storage.insertOrUpdatePlace(place, modified: modified)
+                              >>> { storage.storeRemoteVisits(payload.visits, forGUID: guid) }
+            return placeThenVisits.map({ result in
+                if result.isFailure {
+                    let reason = result.failureValue?.description ?? "unknown reason"
+                    log.error("Record application failed: \(reason)")
+                }
+                return result
+            }).bind(maskSomeFailures)
+        }
+
+        return self.applyIncomingRecords(records, apply: applyRecord)
+    }
+
+    fileprivate func uploadModifiedPlaces(_ places: [(Place, [Visit])], lastTimestamp: Timestamp, fromStorage storage: SyncableHistory, withServer storageClient: Sync15CollectionClient<HistoryPayload>) -> DeferredTimestamp {
+        log.info("Preparing upload…")
+
+        // Build sequences of 1000 history items, sequence by sequence
+        // These will be uploaded in smaller batches by the upload batcher, but we chunk here
+        // in order to bound peak memory usage when we call makeHistoryRecord below.
+        let toUpload = chunk(places, by: 1000)
+        let perChunk: (ArraySlice<(Place, [Visit])>, Timestamp) -> DeferredTimestamp = { (records, timestamp) in
+            let recs = records.map(makeHistoryRecord)
+            log.info("Uploading \(recs.count) history items…")
+            return self.uploadRecords(recs, lastTimestamp: timestamp, storageClient: storageClient) { result, lastModified in
+                // We don't do anything with failed.
+                return storage.markAsSynchronized(result.success, modified: lastModified ?? timestamp)
+            }
+        }
+
+        let start = deferMaybe(lastTimestamp)
+        return walk(toUpload, start: start, f: perChunk)
+    }
+
+    fileprivate func uploadDeletedPlaces(_ guids: [GUID], lastTimestamp: Timestamp, fromStorage storage: SyncableHistory, withServer storageClient: Sync15CollectionClient<HistoryPayload>) -> DeferredTimestamp {
+
+        let records = guids.map(makeDeletedHistoryRecord)
+
+        // Deletions are smaller, so upload 100 at a time.
+        return self.uploadRecords(records, lastTimestamp: lastTimestamp, storageClient: storageClient) { result, lastModified in
+            storage.markAsDeleted(result.success) >>> always(lastModified ?? lastTimestamp)
+        }
+    }
+
+    fileprivate func uploadOutgoingFromStorage(_ storage: SyncableHistory, lastTimestamp: Timestamp, withServer storageClient: Sync15CollectionClient<HistoryPayload>) -> Success {
+        var workWasDone = false
+
+        let uploadDeleted: (Timestamp) -> DeferredTimestamp = { timestamp in
+            storage.getDeletedHistoryToUpload()
+            >>== { guids in
+                if !guids.isEmpty {
+                    workWasDone = true
+                }
+                log.info("Uploading \(guids.count) deleted places.")
+                return self.uploadDeletedPlaces(guids, lastTimestamp: timestamp, fromStorage: storage, withServer: storageClient)
+            }
+        }
+
+        let uploadModified: (Timestamp) -> DeferredTimestamp = { timestamp in
+            storage.getModifiedHistoryToUpload()
+                >>== { places in
+                    if !places.isEmpty {
+                        workWasDone = true
+                    }
+                    log.info("Uploading \(places.count) modified places.")
+                    return self.uploadModifiedPlaces(places, lastTimestamp: timestamp, fromStorage: storage, withServer: storageClient)
+            }
+        }
+
+        // The last clause will checkpoint the DB. But we just checkpointed the DB after downloading records!
+        // Yes, that's true. Either there will be lots of work to do (e.g., having just marked
+        // thousands of records as uploaded, or dropping lots of deleted rows), and so it's
+        // worthwhile… or there won't be much work to do, and the checkpoint will be cheap.
+        // If we did nothing -- uploaded no deletions, uploaded no modified records -- then we
+        // don't checkpoint at all.
+        return deferMaybe(lastTimestamp)
+          >>== uploadDeleted
+          >>== uploadModified
+           >>> effect({ log.debug("Done syncing. Work was done? \(workWasDone)") })
+           >>> { workWasDone ? storage.doneUpdatingMetadataAfterUpload() : succeed() }    // A closure so we eval workWasDone after it's set!
+           >>> effect({ log.debug("Done.") })
+    }
+
+    /**
+     * If the green light turns red, we don't want to continue to upload -- doing
+     * so would cause us to fast-forward our last sync timestamp and skip whatever
+     * we hadn't yet downloaded.
+     */
+    fileprivate func go(_ info: InfoCollections, greenLight: @escaping () -> Bool, downloader: BatchingDownloader<HistoryPayload>, history: SyncableHistory) -> SyncResult {
+
+        if !greenLight() {
+            log.info("Green light turned red. Stopping history download.")
+            return deferMaybe(.partial(self.statsSession))
+        }
+
+        func applyBatched() -> Success {
+            return self.applyIncomingToStorage(history, records: downloader.retrieve())
+               >>> effect(downloader.advance)
+        }
+
+        func onBatchResult(_ result: Maybe<DownloadEndState>) -> SyncResult {
+            guard let end = result.successValue else {
+                log.warning("Got failure: \(result.failureValue!)")
+                return deferMaybe(completedWithStats)
+            }
+
+            switch end {
+            case .complete:
+                log.info("Done with batched mirroring.")
+                return applyBatched()
+                   >>> history.doneApplyingRecordsAfterDownload
+                   >>> { deferMaybe(self.completedWithStats) }
+            case .incomplete:
+                log.debug("Running another batch.")
+                // This recursion is fine because Deferred always pushes callbacks onto a queue.
+                return applyBatched()
+                   >>> { self.go(info, greenLight: greenLight, downloader: downloader, history: history) }
+            case .interrupted:
+                log.info("Interrupted. Aborting batching this time.")
+                return deferMaybe(.partial(self.statsSession))
+            case .noNewData:
+                log.info("No new data. No need to continue batching.")
+                downloader.advance()
+                return deferMaybe(completedWithStats)
+            }
+        }
+
+        return downloader.go(info, limit: self.batchSize)
+                         .bind(onBatchResult)
+    }
+
+    open func synchronizeLocalHistory(_ history: SyncableHistory, withServer storageClient: Sync15StorageClient, info: InfoCollections, greenLight: @escaping () -> Bool) -> SyncResult {
+        if let reason = self.reasonToNotSync(storageClient) {
+            return deferMaybe(.notStarted(reason))
+        }
+
+        let encoder = RecordEncoder<HistoryPayload>(decode: { HistoryPayload($0) }, encode: { $0.json })
+
+        guard let historyClient = self.collectionClient(encoder, storageClient: storageClient) else {
+            log.error("Couldn't make history factory.")
+            return deferMaybe(FatalError(message: "Couldn't make history factory."))
+        }
+
+        let downloader = BatchingDownloader(collectionClient: historyClient, basePrefs: self.prefs, collection: "history")
+
+        // The original version of the history synchronizer tracked its
+        // own last fetched time. We need to migrate this into the
+        // batching downloader.
+        let since: Timestamp = self.lastFetched
+        if since > downloader.lastModified {
+            log.debug("Advancing downloader lastModified to synchronizer lastFetched \(since).")
+            downloader.lastModified = since
+            self.lastFetched = 0
+        }
+
+        statsSession.start()
+        return self.go(info, greenLight: greenLight, downloader: downloader, history: history)
+            >>== { syncResult in
+                switch syncResult {
+                case .completed:
+                    // When we're done downloading, we can upload.
+                    return self.uploadOutgoingFromStorage(history,
+                                                          lastTimestamp: 0,
+                                                          withServer: historyClient)
+                       >>> { deferMaybe(self.completedWithStats) }
+
+                // If we didn't finish downloading, do nothing further -- just pass
+                // through the download result.
+                case .notStarted(_):
+                    return deferMaybe(syncResult)
+
+                case .partial:
+                    log.debug("Didn't finish downloading history; not uploading yet.")
+                    return deferMaybe(syncResult)
+                }
+        }
+    }
+}

mobile/ios/Sync/Synchronizers/IndependentRecordSynchronizer.swift

@@ -0,0 +1,147 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+class Uploader {
+    /**
+     * Upload just about anything that can be turned into something we can upload.
+     */
+    func sequentialPosts<T>(_ items: [T], by: Int, lastTimestamp: Timestamp, storageOp: @escaping ([T], Timestamp) -> DeferredTimestamp) -> DeferredTimestamp {
+
+        // This needs to be a real Array, not an ArraySlice,
+        // for the types to line up.
+        let chunks = chunk(items, by: by).map { Array($0) }
+
+        let start = deferMaybe(lastTimestamp)
+
+        let perChunk: ([T], Timestamp) -> DeferredTimestamp = { (records, timestamp) in
+            // TODO: detect interruptions -- clients uploading records during our sync --
+            // by using ifUnmodifiedSince. We can detect uploaded records since our download
+            // (chain the download timestamp into this function), and we can detect uploads
+            // that race with our own (chain download timestamps across 'walk' steps).
+            // If we do that, we can also advance our last fetch timestamp after each chunk.
+            log.debug("Uploading \(records.count) records.")
+            return storageOp(records, timestamp)
+        }
+
+        return walk(chunks, start: start, f: perChunk)
+    }
+}
+
+open class IndependentRecordSynchronizer: TimestampedSingleCollectionSynchronizer {
+    private func reportApplyStatsWrap<T>(apply: @escaping (T) -> Success) -> (T) -> Success {
+        return { record in
+            return apply(record).bind({ result in
+                var stats = SyncDownloadStats()
+                stats.applied = 1
+                if result.isSuccess {
+                    stats.succeeded = 1
+                } else {
+                    stats.failed = 1
+                }
+                self.statsSession.recordDownload(stats: stats)
+                return Deferred(value: result)
+            })
+        }
+    }
+
+    /**
+     * Just like the usual applyIncomingToStorage, but doesn't fast-forward the timestamp.
+     */
+    func applyIncomingRecords<T>(_ records: [T], apply: @escaping (T) -> Success) -> Success {
+        if records.isEmpty {
+            log.debug("No records; done applying.")
+            return succeed()
+        }
+
+        return walk(records, f: reportApplyStatsWrap(apply: apply))
+    }
+
+    func applyIncomingToStorage<T>(_ records: [T], fetched: Timestamp, apply: @escaping (T) -> Success) -> Success {
+        func done() -> Success {
+            log.debug("Bumping fetch timestamp to \(fetched).")
+            self.lastFetched = fetched
+            return succeed()
+        }
+
+        if records.isEmpty {
+            log.debug("No records; done applying.")
+            return done()
+        }
+
+        return walk(records, f: reportApplyStatsWrap(apply: apply)) >>> done
+    }
+}
+
+extension TimestampedSingleCollectionSynchronizer {
+    /**
+     * On each chunk that we upload, we pass along the server modified timestamp to the next,
+     * chained through the provided `onUpload` function.
+     *
+     * The last chunk passes this modified timestamp out, and we assign it to lastFetched.
+     *
+     * The idea of this is twofold:
+     *
+     * 1. It does the fast-forwarding that every other Sync client does.
+     *
+     * 2. It allows us to (eventually) pass the last collection modified time as If-Unmodified-Since
+     *    on each upload batch, as we do between the download and the upload phase.
+     *    This alone allows us to detect conflicts from racing clients.
+     *
+     * In order to implement the latter, we'd need to chain the date from getSince in place of the
+     * 0 in the call to uploadOutgoingFromStorage in each synchronizer.
+     */
+    func uploadRecords<T>(_ records: [Record<T>], lastTimestamp: Timestamp, storageClient: Sync15CollectionClient<T>, onUpload: @escaping (POSTResult, Timestamp?) -> DeferredTimestamp) -> DeferredTimestamp {
+        if records.isEmpty {
+            log.debug("No modified records to upload.")
+            return deferMaybe(lastTimestamp)
+        }
+
+        func reportUploadStatsWrap(result: POSTResult, timestamp: Timestamp?) -> DeferredTimestamp {
+            let stats = SyncUploadStats(sent: result.success.count, sentFailed: result.failed.count)
+            self.statsSession.recordUpload(stats: stats)
+            return onUpload(result, timestamp)
+        }
+
+        let batch = storageClient.newBatch(ifUnmodifiedSince: (lastTimestamp == 0) ? nil : lastTimestamp, onCollectionUploaded: reportUploadStatsWrap)
+        return batch.addRecords(records)
+            >>> batch.endBatch
+            >>> {
+                let timestamp = batch.ifUnmodifiedSince ?? lastTimestamp
+                self.setTimestamp(timestamp)
+                return deferMaybe(timestamp)
+            }
+    }
+
+    func uploadRecordsSingleBatch<T>(_ records: [Record<T>], lastTimestamp: Timestamp, storageClient: Sync15CollectionClient<T>) -> Deferred<Maybe<(timestamp: Timestamp, succeeded: [GUID])>> {
+        if records.isEmpty {
+            log.debug("No modified records to upload.")
+            return deferMaybe((timestamp: lastTimestamp, succeeded: []))
+        }
+
+        func reportUploadStatsWrap(result: POSTResult, timestamp: Timestamp?) -> DeferredTimestamp {
+            let stats = SyncUploadStats(sent: result.success.count, sentFailed: result.failed.count)
+            self.statsSession.recordUpload(stats: stats)
+            return deferMaybe(timestamp ?? lastTimestamp)
+        }
+
+        let batch = storageClient.newBatch(ifUnmodifiedSince: (lastTimestamp == 0) ? nil : lastTimestamp, onCollectionUploaded: reportUploadStatsWrap)
+        return batch.addRecords(records, singleBatch: true)
+            >>== batch.endSingleBatch
+            >>== { (succeeded, lastModified) in
+                guard let timestamp = lastModified else {
+                    return deferMaybe(FatalError(message: "Could not retrieve lastModified from the server response."))
+                }
+                self.setTimestamp(timestamp)
+                return deferMaybe((timestamp: timestamp, succeeded: succeeded))
+        }
+    }
+}

mobile/ios/Sync/Synchronizers/LoginsSynchronizer.swift

@@ -0,0 +1,190 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+let PasswordsStorageVersion = 1
+
+private func makeDeletedLoginRecord(_ guid: GUID) -> Record<LoginPayload> {
+    // Local modified time is ignored in upload serialization.
+    let modified: Timestamp = 0
+
+    // Arbitrary large number: deletions sync down first.
+    let sortindex = 5_000_000
+
+    let json: JSON = JSON([
+        "id": guid,
+        "deleted": true,
+        ])
+    let payload = LoginPayload(json)
+    return Record<LoginPayload>(id: guid, payload: payload, modified: modified, sortindex: sortindex)
+}
+
+func makeLoginRecord(_ login: Login) -> Record<LoginPayload> {
+    let id = login.guid
+    let modified: Timestamp = 0    // Ignored in upload serialization.
+    let sortindex = 1
+
+    let tLU = NSNumber(value: login.timeLastUsed / 1000)
+    let tPC = NSNumber(value: login.timePasswordChanged / 1000)
+    let tC = NSNumber(value: login.timeCreated / 1000)
+
+    let dict: [String: Any] = [
+        "id": id,
+        "hostname": login.hostname,
+        "httpRealm": login.httpRealm as Any,
+        "formSubmitURL": login.formSubmitURL as Any,
+        "username": login.username ?? "",
+        "password": login.password ,
+        "usernameField": login.usernameField ?? "",
+        "passwordField": login.passwordField ?? "",
+        "timesUsed": login.timesUsed,
+        "timeLastUsed": tLU,
+        "timePasswordChanged": tPC,
+        "timeCreated": tC,
+    ]
+
+    let payload = LoginPayload(JSON(dict))
+    return Record<LoginPayload>(id: id, payload: payload, modified: modified, sortindex: sortindex)
+}
+
+/**
+ * Our current local terminology ("logins") has diverged from the terminology in
+ * use when Sync was built ("passwords"). I've done my best to draw a reasonable line
+ * between the server collection/record format/etc. and local stuff: local storage
+ * works with logins, server records and collection are passwords.
+ */
+open class LoginsSynchronizer: IndependentRecordSynchronizer, Synchronizer {
+    public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
+        super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "passwords")
+    }
+
+    override var storageVersion: Int {
+        return PasswordsStorageVersion
+    }
+
+    func getLogin(_ record: Record<LoginPayload>) -> ServerLogin {
+        let guid = record.id
+        let payload = record.payload
+        let modified = record.modified
+
+        let login = ServerLogin(guid: guid, hostname: payload.hostname, username: payload.username, password: payload.password, modified: modified)
+        login.formSubmitURL = payload.formSubmitURL
+        login.httpRealm = payload.httpRealm
+        login.usernameField = payload.usernameField
+        login.passwordField = payload.passwordField
+
+        // Microseconds locally, milliseconds remotely. We should clean this up.
+        login.timeCreated = 1000 * (payload.timeCreated ?? 0)
+        login.timeLastUsed = 1000 * (payload.timeLastUsed ?? 0)
+        login.timePasswordChanged = 1000 * (payload.timePasswordChanged ?? 0)
+        login.timesUsed = payload.timesUsed ?? 0
+        return login
+    }
+
+    func applyIncomingToStorage(_ storage: SyncableLogins, records: [Record<LoginPayload>], fetched: Timestamp) -> Success {
+        return self.applyIncomingToStorage(records, fetched: fetched) { rec in
+            let guid = rec.id
+            let payload = rec.payload
+
+            guard payload.isValid() else {
+                log.warning("Login record \(guid) is invalid. Skipping.")
+                return succeed()
+            }
+
+            // We apply deletions immediately. That might not be exactly what we want -- perhaps you changed
+            // a password locally after deleting it remotely -- but it's expedient.
+            if payload.deleted {
+                return storage.deleteByGUID(guid, deletedAt: rec.modified)
+            }
+
+            return storage.applyChangedLogin(self.getLogin(rec))
+        }
+    }
+
+    fileprivate func uploadChangedRecords<T>(_ deleted: Set<GUID>, modified: Set<GUID>, records: [Record<T>], lastTimestamp: Timestamp, storage: SyncableLogins, withServer storageClient: Sync15CollectionClient<T>) -> Success {
+
+        let onUpload: (POSTResult, Timestamp?) -> DeferredTimestamp = { result, lastModified in
+            let uploaded = Set(result.success)
+            return storage.markAsDeleted(uploaded.intersection(deleted)) >>> { storage.markAsSynchronized(uploaded.intersection(modified), modified: lastModified ?? lastTimestamp) }
+        }
+
+        return uploadRecords(records,
+                             lastTimestamp: lastTimestamp,
+                             storageClient: storageClient,
+                             onUpload: onUpload) >>> succeed
+    }
+
+    // Find any records for which a local overlay exists. If we want to be really precise,
+    // we can find the original server modified time for each record and use it as
+    // If-Unmodified-Since on a PUT, or just use the last fetch timestamp, which should
+    // be equivalent.
+    // We will already have reconciled any conflicts on download, so this upload phase should
+    // be as simple as uploading any changed or deleted items.
+    fileprivate func uploadOutgoingFromStorage(_ storage: SyncableLogins, lastTimestamp: Timestamp, withServer storageClient: Sync15CollectionClient<LoginPayload>) -> Success {
+        let deleted: () -> Deferred<Maybe<(Set<GUID>, [Record<LoginPayload>])>> = {
+            return storage.getDeletedLoginsToUpload() >>== { guids in
+                let records = guids.map(makeDeletedLoginRecord)
+                return deferMaybe((Set(guids), records))
+            }
+        }
+
+        let modified: () -> Deferred<Maybe<(Set<GUID>, [Record<LoginPayload>])>> = {
+            return storage.getModifiedLoginsToUpload() >>== { logins in
+                let guids = Set(logins.map { $0.guid })
+                let records = logins.map(makeLoginRecord)
+                return deferMaybe((guids, records))
+            }
+        }
+
+        return accumulate([deleted, modified]) >>== { result in
+            let (deletedGUIDs, deletedRecords) = result[0]
+            let (modifiedGUIDs, modifiedRecords) = result[1]
+            let allRecords = deletedRecords + modifiedRecords
+
+            return self.uploadChangedRecords(deletedGUIDs, modified: modifiedGUIDs, records: allRecords,
+                                             lastTimestamp: lastTimestamp, storage: storage, withServer: storageClient)
+        }
+    }
+
+    open func synchronizeLocalLogins(_ logins: SyncableLogins, withServer storageClient: Sync15StorageClient, info: InfoCollections) -> SyncResult {
+        if let reason = self.reasonToNotSync(storageClient) {
+            return deferMaybe(.notStarted(reason))
+        }
+
+        let encoder = RecordEncoder<LoginPayload>(decode: { LoginPayload($0) }, encode: { $0.json })
+        guard let passwordsClient = self.collectionClient(encoder, storageClient: storageClient) else {
+            log.error("Couldn't make logins factory.")
+            return deferMaybe(FatalError(message: "Couldn't make logins factory."))
+        }
+
+        let since: Timestamp = self.lastFetched
+        log.debug("Synchronizing \(self.collection). Last fetched: \(since).")
+
+        let applyIncomingToStorage: (StorageResponse<[Record<LoginPayload>]>) -> Success = { response in
+            let ts = response.metadata.timestampMilliseconds
+            let lm = response.metadata.lastModifiedMilliseconds!
+            log.debug("Applying incoming password records from response timestamped \(ts), last modified \(lm).")
+            log.debug("Records header hint: \(response.metadata.records ??? "nil")")
+            return self.applyIncomingToStorage(logins, records: response.value, fetched: lm) >>> effect {
+                NotificationCenter.default.post(name: NotificationDataRemoteLoginChangesWereApplied, object: nil)
+            }
+        }
+
+        statsSession.start()
+        return passwordsClient.getSince(since)
+            >>== applyIncomingToStorage
+            // TODO: If we fetch sorted by date, we can bump the lastFetched timestamp
+            // to the last successfully applied record timestamp, no matter where we fail.
+            // There's no need to do the upload before bumping -- the storage of local changes is stable.
+            >>> { self.uploadOutgoingFromStorage(logins, lastTimestamp: 0, withServer: passwordsClient) }
+            >>> { return deferMaybe(self.completedWithStats) }
+    }
+}

mobile/ios/Sync/Synchronizers/Synchronizer.swift

@@ -0,0 +1,293 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+
+private let log = Logger.syncLogger
+
+/**
+ * This exists to pass in external context: e.g., the UIApplication can
+ * expose notification functionality in this way.
+ */
+public protocol SyncDelegate {
+    func displaySentTab(for url: URL, title: String, from deviceName: String?)
+    // TODO: storage.
+}
+
+/**
+ * We sometimes want to make a synchronizer start from scratch: to throw away any
+ * metadata and reset storage to match, allowing us to respond to significant server
+ * changes.
+ *
+ * But instantiating a Synchronizer is a lot of work if we simply want to change some
+ * persistent state. This protocol describes a static func that fits most synchronizers.
+ *
+ * When the returned `Deferred` is filled with a success value, the supplied prefs and
+ * storage are ready to sync from scratch.
+ *
+ * Persisted long-term/local data is kept, and will later be reconciled as appropriate.
+ */
+public protocol ResettableSynchronizer {
+    static func resetSynchronizerWithStorage(_ storage: ResettableSyncStorage, basePrefs: Prefs, collection: String) -> Success
+}
+
+/**
+ * This is a delegate that allows synchronizers to notify other devices in the Sync account
+ * that a collection changed.
+ */
+public protocol CollectionChangedNotifier {
+    func notify(deviceIDs: [GUID], collectionsChanged collections: [String], reason: String) -> Success
+    func notifyAll(collectionsChanged collections: [String], reason: String) -> Success
+}
+
+// TODO: return values?
+/**
+ * A Synchronizer is (unavoidably) entirely in charge of what it does within a sync.
+ * For example, it might make incremental progress in building a local cache of remote records, never actually performing an upload or modifying local storage.
+ * It might only upload data. Etc.
+ *
+ * Eventually I envision an intent-like approach, or additional methods, to specify preferences and constraints
+ * (e.g., "do what you can in a few seconds", or "do a full sync, no matter how long it takes"), but that'll come in time.
+ *
+ * A Synchronizer is a two-stage beast. It needs to support synchronization, of course; that
+ * needs a completely configured client, which can only be obtained from Ready. But it also
+ * needs to be able to do certain things beforehand:
+ *
+ * * Wipe its collections from the server (presumably via a delegate from the state machine).
+ * * Prepare to sync from scratch ("reset") in response to a changed set of keys, syncID, or node assignment.
+ * * Wipe local storage ("wipeClient").
+ *
+ * Those imply that some kind of 'Synchronizer' exists throughout the state machine. We *could*
+ * pickle instructions for eventual delivery next time one is made and synchronized…
+ */
+public protocol Synchronizer {
+    init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason)
+
+    /**
+     * Return a reason if the current state of this synchronizer -- particularly prefs and scratchpad --
+     * prevent a routine sync from occurring.
+     */
+    func reasonToNotSync(_: Sync15StorageClient) -> SyncNotStartedReason?
+}
+
+/**
+ * We sometimes wish to return something more nuanced than simple success or failure.
+ * For example, refusing to sync because the engine was disabled isn't success (nothing was transferred!)
+ * but it also isn't an error.
+ *
+ * To do this we model real failures -- something went wrong -- as failures in the Result, and
+ * everything else in this status enum. This will grow as we return more details from a sync to allow
+ * for batch scheduling, success-case backoff and so on.
+ */
+public enum SyncStatus {
+    case completed(SyncEngineStatsSession)
+    case notStarted(SyncNotStartedReason)
+    case partial(SyncEngineStatsSession)
+
+    public var description: String {
+        switch self {
+        case .completed:
+            return "Completed"
+        case let .notStarted(reason):
+            return "Not started: \(reason.description)"
+        case .partial:
+            return "Partial"
+        }
+    }
+}
+
+public typealias DeferredTimestamp = Deferred<Maybe<Timestamp>>
+public typealias SyncResult = Deferred<Maybe<SyncStatus>>
+public typealias EngineIdentifier = String
+public typealias EngineStatus = (EngineIdentifier, SyncStatus)
+public typealias EngineResults = [EngineStatus]
+public typealias SyncOperationResult = (engineResults: Maybe<EngineResults>, stats: SyncOperationStatsSession?)
+
+public enum SyncNotStartedReason {
+    case noAccount
+    case offline
+    case backoff(remainingSeconds: Int)
+    case engineRemotelyNotEnabled(collection: String)
+    case engineFormatOutdated(needs: Int)
+    case engineFormatTooNew(expected: Int)   // This'll disappear eventually; we'll wipe the server and upload m/g.
+    case storageFormatOutdated(needs: Int)
+    case storageFormatTooNew(expected: Int)  // This'll disappear eventually; we'll wipe the server and upload m/g.
+    case stateMachineNotReady                // Because we're not done implementing.
+    case redLight
+    case unknown                             // Likely a programming error.
+
+    var telemetryId: String {
+        switch self {
+        case .noAccount:
+            return "sync.not_started.reason.no_account"
+        case .offline:
+            return "sync.not_started.reason.offline"
+        case .backoff(_):
+            return "sync.not_started.reason.backoff"
+        case .engineRemotelyNotEnabled(_):
+            return "sync.not_started.reason.remotely_not_enabled"
+        case .engineFormatOutdated(_):
+            return "sync.not_started.reason.format_outdated"
+        case .engineFormatTooNew(_):   // This'll disappear eventually; we'll wipe the server and upload m/g.
+            return "sync.not_started.reason.format_too_new"
+        case .storageFormatOutdated(_):
+            return "sync.not_started.reason.storage_format_outdated"
+        case .storageFormatTooNew(_):  // This'll disappear eventually; we'll wipe the server and upload m/g.
+            return "sync.not_started.reason.storage_format_too_new"
+        case .stateMachineNotReady:                // Because we're not done implementing.
+            return "sync.not_started.reason.state_machine_not_ready"
+        case .redLight:
+            return "sync.not_started.reason.red_light"
+        case .unknown:                             // Likely a programming error
+            return "sync.not_started.reason.unknown"
+        }
+    }
+
+    var description: String {
+        switch self {
+        case .noAccount:
+            return "no account"
+        case let .backoff(remaining):
+            return "in backoff: \(remaining) seconds remaining"
+        default:
+            return "undescribed reason"
+        }
+    }
+}
+
+open class FatalError: SyncError {
+    let message: String
+    init(message: String) {
+        self.message = message
+    }
+
+    open var description: String {
+        return self.message
+    }
+}
+
+public protocol SingleCollectionSynchronizer {
+    func remoteHasChanges(_ info: InfoCollections) -> Bool
+}
+
+open class BaseCollectionSynchronizer {
+    let collection: String
+
+    let scratchpad: Scratchpad
+    let delegate: SyncDelegate
+    let basePrefs: Prefs
+    let prefs: Prefs
+    let why: SyncReason
+
+    var statsSession: SyncEngineStatsSession
+
+    static func prefsForCollection(_ collection: String, withBasePrefs basePrefs: Prefs) -> Prefs {
+        let branchName = "synchronizer." + collection + "."
+        return basePrefs.branch(branchName)
+    }
+
+    init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason, collection: String) {
+        self.scratchpad = scratchpad
+        self.delegate = delegate
+        self.collection = collection
+        self.basePrefs = basePrefs
+        self.prefs = BaseCollectionSynchronizer.prefsForCollection(collection, withBasePrefs: basePrefs)
+        self.statsSession = SyncEngineStatsSession(collection: collection)
+        self.why = why
+
+        log.info("Synchronizer configured with prefs '\(self.prefs.getBranchPrefix()).'")
+    }
+
+    var storageVersion: Int {
+        assert(false, "Override me!")
+        return 0
+    }
+
+    // Short-hand for returning .Complete status + recorded stats
+    var completedWithStats: SyncStatus {
+        return .completed(statsSession.end())
+    }
+
+    open func reasonToNotSync(_ client: Sync15StorageClient) -> SyncNotStartedReason? {
+        let now = Date.now()
+        if let until = client.backoff.isInBackoff(now) {
+            let remaining = (until - now) / 1000
+            return .backoff(remainingSeconds: Int(remaining))
+        }
+
+        if let metaGlobal = self.scratchpad.global?.value {
+            // There's no need to check the global storage format here; the state machine will already have
+            // done so.
+            if let engineMeta = metaGlobal.engines[collection] {
+                if engineMeta.version > self.storageVersion {
+                    return .engineFormatOutdated(needs: engineMeta.version)
+                }
+                if engineMeta.version < self.storageVersion {
+                    return .engineFormatTooNew(expected: engineMeta.version)
+                }
+            } else {
+                return .engineRemotelyNotEnabled(collection: self.collection)
+            }
+        } else {
+            // But a missing meta/global is a real problem.
+            return .stateMachineNotReady
+        }
+
+        // Success!
+        return nil
+    }
+
+    func encrypter<T>(_ encoder: RecordEncoder<T>) -> RecordEncrypter<T>? {
+        return self.scratchpad.keys?.value.encrypter(self.collection, encoder: encoder)
+    }
+
+    func collectionClient<T>(_ encoder: RecordEncoder<T>, storageClient: Sync15StorageClient) -> Sync15CollectionClient<T>? {
+        if let encrypter = self.encrypter(encoder) {
+            return storageClient.clientForCollection(self.collection, encrypter: encrypter)
+        }
+        return nil
+    }
+}
+
+/**
+ * Tracks a lastFetched timestamp, uses it to decide if there are any
+ * remote changes, and exposes a method to fast-forward after upload.
+ */
+open class TimestampedSingleCollectionSynchronizer: BaseCollectionSynchronizer, SingleCollectionSynchronizer {
+
+    var lastFetched: Timestamp {
+        set(value) {
+            self.prefs.setLong(value, forKey: "lastFetched")
+        }
+
+        get {
+            return self.prefs.unsignedLongForKey("lastFetched") ?? 0
+        }
+    }
+
+    func setTimestamp(_ timestamp: Timestamp) {
+        log.debug("Setting post-upload lastFetched to \(timestamp).")
+        self.lastFetched = timestamp
+    }
+
+    open func remoteHasChanges(_ info: InfoCollections) -> Bool {
+        return info.modified(self.collection) ?? 0 > self.lastFetched
+    }
+}
+
+extension BaseCollectionSynchronizer: ResettableSynchronizer {
+    public static func resetSynchronizerWithStorage(_ storage: ResettableSyncStorage, basePrefs: Prefs, collection: String) -> Success {
+        let synchronizerPrefs = BaseCollectionSynchronizer.prefsForCollection(collection, withBasePrefs: basePrefs)
+        synchronizerPrefs.removeObjectForKey("lastFetched")
+
+        // Not all synchronizers use a batching downloader, but it's
+        // convenient to just always reset it here.
+        return storage.resetClient()
+           >>> effect({ BatchingDownloader.resetDownloaderWithPrefs(synchronizerPrefs, collection: collection) })
+    }
+}

mobile/ios/Sync/Synchronizers/TabsSynchronizer.swift

@@ -0,0 +1,206 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import Deferred
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+let TabsStorageVersion = 1
+
+open class TabsSynchronizer: TimestampedSingleCollectionSynchronizer, Synchronizer {
+    public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
+        super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "tabs")
+    }
+
+    override var storageVersion: Int {
+        return TabsStorageVersion
+    }
+
+    var tabsRecordLastUpload: Timestamp {
+        set(value) {
+            self.prefs.setLong(value, forKey: "lastTabsUpload")
+        }
+
+        get {
+            return self.prefs.unsignedLongForKey("lastTabsUpload") ?? 0
+        }
+    }
+
+    fileprivate func createOwnTabsRecord(_ tabs: [RemoteTab]) -> Record<TabsPayload> {
+        let guid = self.scratchpad.clientGUID
+
+        let tabsJSON = JSON([
+            "id": guid,
+            "clientName": self.scratchpad.clientName,
+            "tabs": tabs.flatMap { $0.toDictionary() }
+        ])
+        if Logger.logPII {
+            log.verbose("Sending tabs JSON \(tabsJSON.stringValue() ?? "nil")")
+        }
+        let payload = TabsPayload(tabsJSON)
+        return Record(id: guid, payload: payload, ttl: ThreeWeeksInSeconds)
+    }
+
+    fileprivate func uploadOurTabs(_ localTabs: RemoteClientsAndTabs, toServer tabsClient: Sync15CollectionClient<TabsPayload>) -> Success {
+        // check to see if our tabs have changed or we're in a fresh start
+        let lastUploadTime: Timestamp? = (self.tabsRecordLastUpload == 0) ? nil : self.tabsRecordLastUpload
+        if let lastUploadTime = lastUploadTime,
+            lastUploadTime >= (Date.now() - (OneMinuteInMilliseconds)) {
+            log.debug("Not uploading tabs: already did so at \(lastUploadTime).")
+            return succeed()
+        }
+
+        return localTabs.getTabsForClientWithGUID(nil) >>== { tabs in
+            if let lastUploadTime = lastUploadTime {
+                // TODO: track this in memory so we don't have to hit the disk to figure out when our tabs have
+                // changed and need to be uploaded.
+                if tabs.every({ $0.lastUsed < lastUploadTime }) {
+                    return succeed()
+                }
+            }
+
+            let tabsRecord = self.createOwnTabsRecord(tabs)
+            log.debug("Uploading our tabs: \(tabs.count).")
+
+            var uploadStats = SyncUploadStats()
+            uploadStats.sent += 1
+
+            // We explicitly don't send If-Unmodified-Since, because we always
+            // want our upload to succeed -- we own the record.
+            return tabsClient.put(tabsRecord, ifUnmodifiedSince: nil) >>== { resp in
+                if let ts = resp.metadata.lastModifiedMilliseconds {
+                    // Protocol says this should always be present for success responses.
+                    log.debug("Tabs record upload succeeded. New timestamp: \(ts).")
+                    self.tabsRecordLastUpload = ts
+                } else {
+                    uploadStats.sentFailed += 1
+                }
+                return succeed()
+            } >>== effect({ self.statsSession.recordUpload(stats: uploadStats) })
+        }
+    }
+
+    open func synchronizeLocalTabs(_ localTabs: RemoteClientsAndTabs, withServer storageClient: Sync15StorageClient, info: InfoCollections) -> SyncResult {
+        func onResponseReceived(_ response: StorageResponse<[Record<TabsPayload>]>) -> Success {
+
+            func afterWipe() -> Success {
+                var downloadStats = SyncDownloadStats()
+
+                let doInsert: (Record<TabsPayload>) -> Deferred<Maybe<(Int)>> = { record in
+                    let remotes = record.payload.isValid() ? record.payload.remoteTabs : []
+                    let ins = localTabs.insertOrUpdateTabsForClientGUID(record.id, tabs: remotes)
+
+                    // Since tabs are all sent within a single record, we don't count number of tabs applied
+                    // but number of records. In this case it's just one.
+                    downloadStats.applied += 1
+                    ins.upon() { res in
+                        if let inserted = res.successValue {
+                            if inserted != remotes.count {
+                                log.warning("Only inserted \(inserted) tabs, not \(remotes.count). Malformed or missing client?")
+                            }
+                            downloadStats.applied += 1
+                        } else {
+                            downloadStats.failed += 1
+                        }
+                    }
+                    return ins
+                }
+
+                let ourGUID = self.scratchpad.clientGUID
+
+                let records = response.value
+                let responseTimestamp = response.metadata.lastModifiedMilliseconds
+
+                log.debug("Got \(records.count) tab records.")
+
+                // We can only insert tabs for clients that we know locally, so
+                // first we fetch the list of IDs and intersect the two.
+                // TODO: there's a much more efficient way of doing this.
+                return localTabs.getClientGUIDs()
+                    >>== { clientGUIDs in
+                        let filtered = records.filter({ $0.id != ourGUID && clientGUIDs.contains($0.id) })
+                        if filtered.count != records.count {
+                            log.debug("Filtered \(records.count) records down to \(filtered.count).")
+                        }
+
+                        let allDone = all(filtered.map(doInsert))
+                        return allDone.bind { (results) -> Success in
+                            if let failure = results.find({ $0.isFailure }) {
+                                return deferMaybe(failure.failureValue!)
+                            }
+
+                            self.lastFetched = responseTimestamp!
+                            return succeed()
+                        }
+                } >>== effect({ self.statsSession.downloadStats })
+            }
+
+            // If this is a fresh start, do a wipe.
+            if self.lastFetched == 0 {
+                log.info("Last fetch was 0. Wiping tabs.")
+                return localTabs.wipeRemoteTabs()
+                    >>== afterWipe
+            }
+
+            return afterWipe()
+        }
+
+        if let reason = self.reasonToNotSync(storageClient) {
+            return deferMaybe(SyncStatus.notStarted(reason))
+        }
+
+        let keys = self.scratchpad.keys?.value
+        let encoder = RecordEncoder<TabsPayload>(decode: { TabsPayload($0) }, encode: { $0.json })
+        if let encrypter = keys?.encrypter(self.collection, encoder: encoder) {
+            let tabsClient = storageClient.clientForCollection(self.collection, encrypter: encrypter)
+
+            statsSession.start()
+
+            if !self.remoteHasChanges(info) {
+                // upload local tabs if they've changed or we're in a fresh start.
+                return uploadOurTabs(localTabs, toServer: tabsClient)
+                    >>> { deferMaybe(self.completedWithStats) }
+            }
+
+            return tabsClient.getSince(self.lastFetched)
+                >>== onResponseReceived
+                >>> { self.uploadOurTabs(localTabs, toServer: tabsClient) }
+                >>> { deferMaybe(self.completedWithStats) }
+        }
+
+        log.error("Couldn't make tabs factory.")
+        return deferMaybe(FatalError(message: "Couldn't make tabs factory."))
+    }
+
+    /**
+     * This is a dedicated resetting interface that does both tabs and clients at the
+     * same time.
+     */
+    open static func resetClientsAndTabsWithStorage(_ storage: ResettableSyncStorage, basePrefs: Prefs) -> Success {
+        let clientPrefs = BaseCollectionSynchronizer.prefsForCollection("clients", withBasePrefs: basePrefs)
+        let tabsPrefs = BaseCollectionSynchronizer.prefsForCollection("tabs", withBasePrefs: basePrefs)
+        clientPrefs.removeObjectForKey("lastFetched")
+        tabsPrefs.removeObjectForKey("lastFetched")
+        return storage.resetClient()
+    }
+}
+
+extension RemoteTab {
+    public func toDictionary() -> Dictionary<String, Any>? {
+        let tabHistory = history.flatMap { $0.absoluteString }
+        if tabHistory.isEmpty {
+            return nil
+        }
+        return [
+            "title": title,
+            "icon": icon?.absoluteString as Any? ?? NSNull(),
+            "urlHistory": tabHistory,
+            "lastUsed": millisecondsToDecimalSeconds(lastUsed)
+        ]
+    }
+}

mobile/ios/Sync/TabsPayload.swift

@@ -0,0 +1,155 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Storage
+import XCGLogger
+import SwiftyJSON
+
+// Int64.max / 1000.
+private let MaxSecondsToConvertInt64: Int64 = 9223372036854775
+private let MaxSecondsToConvertDouble: Double = Double(9223372036854775 as Int64)
+
+private let log = Logger.browserLogger
+
+open class TabsPayload: CleartextPayloadJSON {
+    open class Tab {
+        let title: String
+        let urlHistory: [String]
+        let lastUsed: Timestamp
+        let icon: String?
+
+        fileprivate init(title: String, urlHistory: [String], lastUsed: Timestamp, icon: String?) {
+            self.title = title
+            self.urlHistory = urlHistory
+            self.lastUsed = lastUsed
+            self.icon = icon
+        }
+
+        func toRemoteTabForClient(_ guid: GUID) -> RemoteTab? {
+            let urls = urlHistory.flatMap({ $0.asURL })
+            if urls.isEmpty {
+                log.debug("Bug 1201875 - Discarding tab as history has no conforming URLs.")
+                return nil
+            }
+
+            return RemoteTab(clientGUID: guid, URL: urls[0], title: self.title, history: urls, lastUsed: self.lastUsed, icon: self.icon?.asURL)
+        }
+
+        class func remoteTabFromJSON(_ json: JSON, clientGUID: GUID) -> RemoteTab? {
+            return fromJSON(json)?.toRemoteTabForClient(clientGUID)
+        }
+
+        class func fromJSON(_ json: JSON) -> Tab? {
+            func getLastUsed(_ json: JSON) -> Timestamp? {
+                let lastUsed = json["lastUsed"]
+                if lastUsed.isBool() {
+                    return nil
+                }
+                // This might be a string or a number.
+                if let num = lastUsed.int64 {
+                    if num < 0 {
+                        // Timestamps are unsigned.
+                        return nil
+                    }
+                    if num > MaxSecondsToConvertInt64 {
+                        // This will overflow when multiplied.
+                        return nil
+                    }
+                    return Timestamp(num * 1000)
+                }
+
+                if let num = lastUsed.double {
+                    if num < 0 {
+                        // Timestamps are unsigned.
+                        return nil
+                    }
+                    if num > MaxSecondsToConvertDouble {
+                        // This will overflow when multiplied.
+                        return nil
+                    }
+                    return Timestamp(num * 1000)
+                }
+
+                if let num = lastUsed.string {
+                    // Try parsing.
+                    return someKindOfTimestampStringToTimestamp(num)
+                }
+
+                return nil
+            }
+
+            if let title = json["title"].string,
+               let urlHistory = jsonsToStrings(json["urlHistory"].array),
+               let lastUsed = getLastUsed(json) {
+                return Tab(title: title, urlHistory: urlHistory, lastUsed: lastUsed, icon: json["icon"].string)
+            }
+
+            return nil
+        }
+    }
+
+    override open func isValid() -> Bool {
+        if !super.isValid() {
+            return false
+        }
+
+        if self["deleted"].bool ?? false {
+            return true
+        }
+
+        return self["clientName"].isString() &&
+               self["tabs"].isArray()
+    }
+
+    // Eventually it'd be nice to unify RemoteTab and Tab. We want to kill the GUID in RemoteTab,
+    // at which point the only distinction between the two is that RemoteTab is "simple" and
+    // lives in Storage, and Tab is more closely tied to TabsPayload.
+
+    var remoteTabs: [RemoteTab] {
+        if let clientGUID = self["id"].string {
+            let payloadTabs = self["tabs"].arrayValue
+            let remoteTabs = payloadTabs.flatMap({ Tab.remoteTabFromJSON($0, clientGUID: clientGUID) })
+            if payloadTabs.count != remoteTabs.count {
+                log.debug("Bug 1201875 - Missing remote tabs from sync")
+            }
+            return remoteTabs
+        }
+        log.debug("no client ID for remote tabs")
+        return []
+    }
+
+    var tabs: [Tab] {
+        return self["tabs"].arrayValue.flatMap(Tab.fromJSON)
+    }
+
+    var clientName: String {
+        return self["clientName"].string!
+    }
+
+    override open func equalPayloads(_ obj: CleartextPayloadJSON) -> Bool {
+        if !(obj is TabsPayload) {
+            return false
+        }
+
+        if !super.equalPayloads(obj) {
+            return false
+        }
+
+        let p = obj as! TabsPayload
+        if p.clientName != self.clientName {
+            return false
+        }
+
+        // TODO: compare tabs.
+        /*
+        if p.tabs != self.tabs {
+            return false;
+        }
+        */
+
+        return true
+    }
+}