Dactyloidae iOS initial commit

mobile/ios/SyncTests/BatchingClientTests.swift

@@ -0,0 +1,608 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Deferred
+@testable import Sync
+
+import XCTest
+import SwiftyJSON
+
+// Always return a gigantic encoded payload.
+private func massivify<T>(_ record: Record<T>) -> JSON? {
+    return JSON([
+        "id": record.id,
+        "foo": String(repeating: "X", count: Sync15StorageClient.maxRecordSizeBytes + 1)
+    ])
+}
+
+private func basicSerializer<T>(record: Record<T>) -> String {
+    return JSON(object: [
+        "id": record.id,
+        "payload": record.payload.json.dictionaryObject as Any
+        ]).stringValue()!
+}
+
+// Create a basic record with an ID and a title that is the `Site$ID`.
+private func createRecordWithID(id: String) -> Record<CleartextPayloadJSON> {
+    let jsonString = "{\"id\":\"\(id)\",\"title\": \"\(id)\"}"
+    return Record<CleartextPayloadJSON>(id: id,
+                                        payload: CleartextPayloadJSON(JSON(parseJSON: jsonString)),
+                                        modified: 10_000,
+                                        sortindex: 123,
+                                        ttl: 1_000_000)
+}
+
+private func assertLinesMatchRecords<T>(lines: [String], records: [Record<T>], serializer: (Record<T>) -> String) {
+    guard lines.count == records.count else {
+        XCTFail("Number of lines mismatch number of records")
+        return
+    }
+
+    lines.enumerated().forEach { index, line in
+        let record = records[index]
+        XCTAssertEqual(line, serializer(record))
+    }
+}
+
+private func deferEmptyResponse(token batchToken: BatchToken? = nil, lastModified: Timestamp? = nil) -> Deferred<Maybe<StorageResponse<POSTResult>>> {
+    var headers = [String: Any]()
+    if let lastModified = lastModified {
+        headers["X-Last-Modified"] = lastModified
+    }
+
+    return deferMaybe(
+        StorageResponse(value: POSTResult(success: [], failed: [:], batchToken: batchToken), metadata: ResponseMetadata(status: 200, headers: headers))
+    )
+}
+
+// Small helper operator for comparing query parameters below
+private func ==(param1: NSURLQueryItem, param2: NSURLQueryItem) -> Bool {
+    return param1.name == param2.name && param1.value == param2.value
+}
+
+private let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 2, maxPostBytes: 1_048_576, maxTotalRecords: 10, maxTotalBytes: 104_857_600)
+
+class Sync15BatchClientTests: XCTestCase {
+
+    func testAddLargeRecordFails() {
+        let uploader: BatchUploadFunction = { _ in deferEmptyResponse(lastModified: 10_000) }
+        let serializeRecord = { massivify($0)?.stringValue() }
+
+        let batch = Sync15BatchClient(config: miniConfig,
+                                      ifUnmodifiedSince: nil,
+                                      serializeRecord: serializeRecord,
+                                      uploader: uploader,
+                                      onCollectionUploaded: { _ in deferMaybe(Date() as! MaybeErrorType)})
+
+        let record = createRecordWithID(id: "A")
+        let result = batch.addRecords([record]).value
+        XCTAssertTrue(result.isFailure)
+        XCTAssertTrue(result.failureValue! is RecordTooLargeError)
+    }
+
+    func testFailToSerializeRecord() {
+        let uploader: BatchUploadFunction = { _ in deferEmptyResponse(lastModified: 10_000) }
+        let batch = Sync15BatchClient(config: miniConfig,
+                                      ifUnmodifiedSince: nil,
+                                      serializeRecord: { _ in nil },
+                                      uploader: uploader,
+                                      onCollectionUploaded: { _ in deferMaybe(Date.now())})
+
+        let record = createRecordWithID(id: "A")
+        let result = batch.addRecords([record]).value
+        XCTAssertTrue(result.isFailure)
+        XCTAssertTrue(result.failureValue! is SerializeRecordFailure)
+    }
+
+    func testBackoffDuringBatchUploading() {
+        let uploader: BatchUploadFunction = { lines, ius, queryParams in
+            deferMaybe(ServerInBackoffError(until: 10_000))
+        }
+
+        // Setup a configuration so each batch supports two payloads of two records each
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 1,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 4,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: { _ in deferMaybe(Date.now())})
+
+        let record = createRecordWithID(id: "A")
+        batch.addRecords([record]).succeeded()
+        let result = batch.endBatch().value
+
+        // Verify that when the server goes into backoff, we get those bubbled up through the batching client
+        XCTAssertTrue(result.isFailure)
+        XCTAssertTrue(result.failureValue! is ServerInBackoffError)
+    }
+
+    func testIfUnmodifiedSinceUpdatesForSinglePOSTs() {
+        var requestCount = 0
+        var linesSent = [String]()
+        var lastIUS: Timestamp? = 0
+
+        // Since we never return a batch token, the batch client won't batch upload and default to single POSTs
+        let uploader: BatchUploadFunction = { lines, ius, _ in
+            linesSent += lines
+            requestCount += 1
+            switch requestCount {
+            case 1:
+                XCTAssertEqual(ius, 10_000_000)
+                return deferEmptyResponse(lastModified: 20_000)
+            case 2:
+                XCTAssertEqual(ius, 20_000_000)
+                return deferEmptyResponse(lastModified: 30_000)
+            case 3:
+                XCTAssertEqual(ius, 30_000_000)
+                lastIUS = ius
+                return deferEmptyResponse(lastModified: 30_000)
+            default:
+                XCTFail()
+                return deferEmptyResponse(lastModified: 0)
+            }
+        }
+
+        let singleRecordConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 1,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 10,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: singleRecordConfig,
+                                      ifUnmodifiedSince: 10_000_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: { _ in deferMaybe(Date.now())})
+
+        let recordA = createRecordWithID(id: "A")
+        let recordB = createRecordWithID(id: "B")
+        let recordC = createRecordWithID(id: "C")
+        let allRecords = [recordA, recordB, recordC]
+
+        batch.addRecords([recordA, recordB, recordC]).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent
+        XCTAssertEqual(requestCount, 3)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: allRecords, serializer: basicSerializer)
+
+        // Validate the last IUS we got is the last request
+        XCTAssertEqual(lastIUS, 30_000_000)
+    }
+
+    func testUploadBatchUnsupportedBatching() {
+        var requestCount = 0
+        var uploadedCollectionCount = 0
+        var linesSent = [String]()
+
+        // Since we never return a batch token, the batch client won't batch upload and default to single POSTs
+        let uploader: BatchUploadFunction = { lines, ius, _ in
+            linesSent += lines
+            requestCount += 1
+            return deferEmptyResponse(lastModified: 10_000)
+        }
+
+        let collectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp = { _ in
+            uploadedCollectionCount += 1
+            return deferMaybe(Date.now())
+        }
+
+        // Setup a configuration so we each payload would be one record
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 1,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 10,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: collectionUploaded)
+
+        let recordA = createRecordWithID(id: "A")
+        let recordB = createRecordWithID(id: "B")
+        let allRecords = [recordA, recordB]
+
+        batch.addRecords([recordA, recordB]).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent. One for the start post, and one for the committing
+        XCTAssertEqual(requestCount, 2)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: allRecords, serializer: basicSerializer)
+
+        // Validate we only made 2 calls to collection uploaded since we're doing single POSTs
+        XCTAssertEqual(uploadedCollectionCount, 2)
+    }
+
+    /**
+     Tests sending a batch consisting of 3 full payloads. This batch is regular in the sense that it should
+     contain a batch=true call, an upload within the batch, and finish with a commit=true upload.
+     */
+    func testUploadRegularSingleBatch() {
+        var requestCount = 0
+        var uploadedCollectionCount = 0
+        var linesSent = [String]()
+
+        let allRecords: [Record<CleartextPayloadJSON>] = "ABCDEF".characters.reduce([]) { list, char in
+            return list + [createRecordWithID(id: String(char))]
+        }
+
+        // Since we never return a batch token, the batch client won't batch upload and default to single POSTs
+        let uploader: BatchUploadFunction = { lines, ius, queryParams in
+            linesSent += lines
+            requestCount += 1
+            switch requestCount {
+            case 1:
+                let expected = URLQueryItem(name: "batch", value: "true")
+                XCTAssertEqual(expected, queryParams![0])
+                XCTAssertEqual(ius, 10_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[0..<2]), serializer: basicSerializer)
+                return deferEmptyResponse(token: "1", lastModified: 10_000)
+            case 2:
+                let expected = URLQueryItem(name: "batch", value: "1")
+                XCTAssertEqual(expected, queryParams![0])
+                XCTAssertEqual(ius, 10_000_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[2..<4]), serializer: basicSerializer)
+                return deferEmptyResponse(token: "1", lastModified: 10_000)
+            case 3:
+                let expectedBatch = URLQueryItem(name: "batch", value: "1")
+                let expectedCommit = URLQueryItem(name: "commit", value: "true")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                XCTAssertEqual(expectedCommit, queryParams![1])
+                XCTAssertEqual(ius, 10_000_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[4..<6]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 20_000)
+            default:
+                XCTFail()
+                return deferEmptyResponse(lastModified: 0)
+            }
+        }
+
+        let collectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp = { _ in
+            uploadedCollectionCount += 1
+            return deferMaybe(Date.now())
+        }
+
+        // Setup a configuration so we send 2 records per each payload
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 2,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 10,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: collectionUploaded)
+
+        batch.addRecords(allRecords).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent. One for the start post, and one for the committing
+        XCTAssertEqual(requestCount, 3)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: allRecords, serializer: basicSerializer)
+
+        // Validate we only made one call to the collection upload callback
+        XCTAssertEqual(uploadedCollectionCount, 2)
+        XCTAssertEqual(batch.ifUnmodifiedSince!, 20_000_000)
+    }
+
+    /**
+     Tests pushing a batch where one of the payloads is not at limit.
+     */
+    func testBatchUploadWithPartialPayload() {
+        var requestCount = 0
+        var uploadedCollectionCount = 0
+        var linesSent = [String]()
+
+        let allRecords: [Record<CleartextPayloadJSON>] = "ABC".characters.reduce([]) { list, char in
+            return list + [createRecordWithID(id: String(char))]
+        }
+
+        // Since we never return a batch token, the batch client won't batch upload and default to single POSTs
+        let uploader: BatchUploadFunction = { lines, ius, queryParams in
+            linesSent += lines
+            requestCount += 1
+            switch requestCount {
+            case 1:
+                let expected = URLQueryItem(name: "batch", value: "true")
+                XCTAssertEqual(expected, queryParams![0])
+                XCTAssertEqual(ius, 10_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[0..<2]), serializer: basicSerializer)
+                return deferEmptyResponse(token: "1", lastModified: 10_000)
+            case 2:
+                let expectedBatch = URLQueryItem(name: "batch", value: "1")
+                let expectedCommit = URLQueryItem(name: "commit", value: "true")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                XCTAssertEqual(expectedCommit, queryParams![1])
+                XCTAssertEqual(ius, 10_000_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[2..<3]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 20_000)
+            default:
+                XCTFail()
+                return deferEmptyResponse(lastModified: 0)
+            }
+        }
+
+        let collectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp = { _ in
+            uploadedCollectionCount += 1
+            return deferMaybe(Date.now())
+        }
+
+        // Setup a configuration so we send 2 records per each payload
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 2,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 10,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: collectionUploaded)
+
+        batch.addRecords(allRecords).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent. One for the start post, and one for the committing
+        XCTAssertEqual(requestCount, 2)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: allRecords, serializer: basicSerializer)
+
+        // Validate we only made one call to the collection upload callback
+        XCTAssertEqual(uploadedCollectionCount, 2)
+        XCTAssertEqual(batch.ifUnmodifiedSince!, 20_000_000)
+    }
+
+    /**
+     Attempt to send 3 payloads: 2 which are full, 1 that is partial, within a batch that only supports 5 records.
+     */
+    func testBatchUploadWithUnevenPayloadsInBatch() {
+        var requestCount = 0
+        var uploadedCollectionCount = 0
+        var linesSent = [String]()
+
+        let allRecords: [Record<CleartextPayloadJSON>] = "ABCDE".characters.reduce([]) { list, char in
+            return list + [createRecordWithID(id: String(char))]
+        }
+
+        // Since we never return a batch token, the batch client won't batch upload and default to single POSTs
+        let uploader: BatchUploadFunction = { lines, ius, queryParams in
+            linesSent += lines
+            requestCount += 1
+            switch requestCount {
+            case 1:
+                let expected = URLQueryItem(name: "batch", value: "true")
+                XCTAssertEqual(expected, queryParams![0])
+                XCTAssertEqual(ius, 10_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[0..<2]), serializer: basicSerializer)
+                return deferEmptyResponse(token: "1", lastModified: 10_000)
+            case 2:
+                let expectedBatch = URLQueryItem(name: "batch", value: "1")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                XCTAssertEqual(ius, 10_000_000)
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[2..<4]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 10_000)
+            case 3:
+                let expectedBatch = URLQueryItem(name: "batch", value: "1")
+                let expectedCommit = URLQueryItem(name: "commit", value: "true")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                XCTAssertEqual(expectedCommit, queryParams![1])
+                XCTAssertEqual(ius, 10_000_000)
+                assertLinesMatchRecords(lines: lines, records: [allRecords[4]], serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 20_000)
+            default:
+                XCTFail()
+                return deferEmptyResponse(lastModified: 0)
+            }
+        }
+
+        let collectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp = { _ in
+            uploadedCollectionCount += 1
+            return deferMaybe(Date.now())
+        }
+
+        // Setup a configuration so we send 2 records per each payload
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 2,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 5,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: collectionUploaded)
+
+        batch.addRecords(allRecords).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent. One for the start post, and one for the committing
+        XCTAssertEqual(requestCount, 3)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: allRecords, serializer: basicSerializer)
+
+        // Validate we only made one call to the collection upload callback
+        XCTAssertEqual(uploadedCollectionCount, 2)
+        XCTAssertEqual(batch.ifUnmodifiedSince!, 20_000_000)
+    }
+
+    /**
+     Tests pushing up a single payload as part of a batch.
+     */
+    func testBatchUploadWithSinglePayload() {
+        var requestCount = 0
+        var uploadedCollectionCount = 0
+        var linesSent = [String]()
+
+        let recordA = createRecordWithID(id: "A")
+
+        // Since we never return a batch token, the batch client won't batch upload and default to single POSTs
+        let uploader: BatchUploadFunction = { lines, ius, queryParams in
+            linesSent += lines
+            requestCount += 1
+            switch requestCount {
+            case 1:
+                XCTAssertEqual(ius, 10_000)
+                assertLinesMatchRecords(lines: lines, records: [recordA], serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 20_000)
+            default:
+                XCTFail()
+                return deferEmptyResponse(lastModified: 0)
+            }
+        }
+
+        let collectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp = { _ in
+            uploadedCollectionCount += 1
+            return deferMaybe(Date.now())
+        }
+
+        // Setup a configuration so we send 2 records per each payload
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 2,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 10,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: collectionUploaded)
+
+        batch.addRecords([recordA]).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent. One for the start post, and one for the committing
+        XCTAssertEqual(requestCount, 1)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: [recordA], serializer: basicSerializer)
+
+        // Validate we only made one call to the collection upload callback
+        XCTAssertEqual(uploadedCollectionCount, 1)
+        XCTAssertEqual(batch.ifUnmodifiedSince!, 20_000_000)
+    }
+
+    func testMultipleBatchUpload() {
+        var requestCount = 0
+        var uploadedCollectionCount = 0
+        var linesSent = [String]()
+
+        let allRecords: [Record<CleartextPayloadJSON>] = "ABCDEFGHIJKL".characters.reduce([]) { list, char in
+            return list + [createRecordWithID(id: String(char))]
+        }
+
+        // For each upload, verify that we are getting the correct queryParams and records to be sent.
+        let uploader: BatchUploadFunction = { lines, ius, queryParams in
+            linesSent += lines
+            requestCount += 1
+            switch requestCount {
+            case 1:
+                let expected = URLQueryItem(name: "batch", value: "true")
+                XCTAssertEqual(expected, queryParams![0])
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[0..<2]), serializer: basicSerializer)
+                return deferEmptyResponse(token: "1", lastModified: 20_000)
+            case 2:
+                let expectedBatch = URLQueryItem(name: "batch", value: "1")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[2..<4]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 20_000)
+            case 3:
+                let expectedBatch = URLQueryItem(name: "batch", value: "1")
+                let expectedCommit = URLQueryItem(name: "commit", value: "true")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                XCTAssertEqual(expectedCommit, queryParams![1])
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[4..<6]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 20_000)
+            case 4:
+                let expected = URLQueryItem(name: "batch", value: "true")
+                XCTAssertEqual(expected, queryParams![0])
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[6..<8]), serializer: basicSerializer)
+                return deferEmptyResponse(token: "2", lastModified: 30_000)
+            case 5:
+                let expectedBatch = URLQueryItem(name: "batch", value: "2")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[8..<10]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 30_000)
+            case 6:
+                let expectedBatch = URLQueryItem(name: "batch", value: "2")
+                let expectedCommit = URLQueryItem(name: "commit", value: "true")
+                XCTAssertEqual(expectedBatch, queryParams![0])
+                XCTAssertEqual(expectedCommit, queryParams![1])
+                assertLinesMatchRecords(lines: lines, records: Array(allRecords[10..<12]), serializer: basicSerializer)
+                return deferEmptyResponse(lastModified: 30_000)
+            default:
+                XCTFail()
+                return deferEmptyResponse(lastModified: 0)
+            }
+        }
+
+        let collectionUploaded: (POSTResult, Timestamp?) -> DeferredTimestamp = { _ in
+            uploadedCollectionCount += 1
+            return deferMaybe(Date.now())
+        }
+
+        // Setup a configuration so each batch supports two payloads of two records each
+        let twoRecordBatchesConfig = InfoConfiguration(
+            maxRequestBytes: 1_048_576,
+            maxPostRecords: 2,
+            maxPostBytes: 1_048_576,
+            maxTotalRecords: 6,
+            maxTotalBytes: 104_857_600
+        )
+
+        let batch = Sync15BatchClient(config: twoRecordBatchesConfig,
+                                      ifUnmodifiedSince: 10_000_000,
+                                      serializeRecord: basicSerializer,
+                                      uploader: uploader,
+                                      onCollectionUploaded: collectionUploaded)
+
+        batch.addRecords(allRecords).succeeded()
+        batch.endBatch().succeeded()
+
+        // Validate number of requests sent. One for the start post, and one for the committing
+        XCTAssertEqual(requestCount, 6)
+
+        // Validate contents sent to the server
+        assertLinesMatchRecords(lines: linesSent, records: allRecords, serializer: basicSerializer)
+
+        // Validate we only called collection uploaded when we start and finish a batch. The uploads inside
+        // a batch should not trigger the callback.
+        XCTAssertEqual(uploadedCollectionCount, 4)
+    }
+}

mobile/ios/SyncTests/CryptoTests.swift

@@ -0,0 +1,88 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import UIKit
+import Shared
+import Storage
+@testable import Sync
+
+import XCTest
+
+class CryptoTests: XCTestCase {
+    let hmacB16 = "b1e6c18ac30deb70236bc0d65a46f7a4dce3b8b0e02cf92182b914e3afa5eebc"
+    let ivB64 = "GX8L37AAb2FZJMzIoXlX8w=="
+
+    let hmacKey = Bytes.decodeBase64("MMntEfutgLTc8FlTLQFms8/xMPmCldqPlq/QQXEjx70=")!
+    let encKey = Bytes.decodeBase64("9K/wLdXdw+nrTtXo4ZpECyHFNr4d7aYHqeg3KW9+m6Q=")!
+
+    let invalidB64 = "NMsdnRulLwQsVcwxKW9XwaUe7ouJk5~~~~~~~~~~~~~~~"
+
+    let ciphertextB64 = "NMsdnRulLwQsVcwxKW9XwaUe7ouJk5Wn80QhbD80l0HEcZGCynh45qIbeYBik0lgcHbKmlIxTJNwU+OeqipN+/j7MqhjKOGIlvbpiPQQLC6/ffF2vbzL0nzMUuSyvaQzyGGkSYM2xUFt06aNivoQTvU2GgGmUK6MvadoY38hhW2LCMkoZcNfgCqJ26lO1O0sEO6zHsk3IVz6vsKiJ2Hq6VCo7hu123wNegmujHWQSGyf8JeudZjKzfi0OFRRvvm4QAKyBWf0MgrW1F8SFDnVfkq8amCB7NhdwhgLWbN+21NitNwWYknoEWe1m6hmGZDgDT32uxzWxCV8QqqrpH/ZggViEr9uMgoy4lYaWqP7G5WKvvechc62aqnsNEYhH26A5QgzmlNyvB+KPFvPsYzxDnSCjOoRSLx7GG86wT59QZw="
+
+    let cleartextB64 = "eyJpZCI6IjVxUnNnWFdSSlpYciIsImhpc3RVcmkiOiJmaWxlOi8vL1VzZXJzL2phc29uL0xpYnJhcnkvQXBwbGljYXRpb24lMjBTdXBwb3J0L0ZpcmVmb3gvUHJvZmlsZXMva3NnZDd3cGsuTG9jYWxTeW5jU2VydmVyL3dlYXZlL2xvZ3MvIiwidGl0bGUiOiJJbmRleCBvZiBmaWxlOi8vL1VzZXJzL2phc29uL0xpYnJhcnkvQXBwbGljYXRpb24gU3VwcG9ydC9GaXJlZm94L1Byb2ZpbGVzL2tzZ2Q3d3BrLkxvY2FsU3luY1NlcnZlci93ZWF2ZS9sb2dzLyIsInZpc2l0cyI6W3siZGF0ZSI6MTMxOTE0OTAxMjM3MjQyNSwidHlwZSI6MX1dfQ=="
+
+    override func setUp() {
+        super.setUp()
+        // Put setup code here. This method is called before the invocation of each test method in the class.
+    }
+
+    override func tearDown() {
+        // Put teardown code here. This method is called after the invocation of each test method in the class.
+        super.tearDown()
+    }
+
+    func testHMAC() {
+        let keyBundle = KeyBundle(encKey: encKey, hmacKey: hmacKey)
+        // HMAC is computed against the Base64 ciphertext.
+        let ciphertextRaw: Data = dataFromBase64(b64: ciphertextB64)
+        XCTAssertNotNil(ciphertextRaw)
+        XCTAssertEqual(hmacB16, keyBundle.hmacString(ciphertextRaw))
+    }
+
+    func dataFromBase64(b64: String) -> Data {
+        return Bytes.dataFromBase64(b64)!
+    }
+
+    func testDecrypt() {
+        let keyBundle = KeyBundle(encKey: encKey, hmacKey: hmacKey)
+        // Decryption is done against raw bytes.
+        let ciphertext = Bytes.decodeBase64(ciphertextB64)!
+        let iv = Bytes.decodeBase64(ivB64)!
+        let s = keyBundle.decrypt(ciphertext, iv: iv)
+        let cleartext = NSString(data: Bytes.decodeBase64(cleartextB64)!,
+                                 encoding: String.Encoding.utf8.rawValue)
+        XCTAssertTrue(cleartext!.isEqual(to: s!))
+    }
+
+    func testBadBase64() {
+        XCTAssertNil(Bytes.decodeBase64(invalidB64))
+    }
+
+    func testEncrypt() {
+        let keyBundle = KeyBundle(encKey: encKey, hmacKey: hmacKey)
+        let cleartext = Bytes.decodeBase64(cleartextB64)!
+
+        // With specified IV.
+        let iv = Bytes.decodeBase64(ivB64)!
+        if let (b, ivOut) = keyBundle.encrypt(cleartext, iv: iv) {
+            // The output IV should be the input.
+            XCTAssertEqual(ivOut, iv)
+            XCTAssertEqual(b, Bytes.decodeBase64(ciphertextB64)!)
+        } else {
+            XCTFail("Encrypt failed.")
+        }
+
+        // With a random IV.
+        if let (b, ivOut) = keyBundle.encrypt(cleartext) {
+            // The output IV should be different.
+            // TODO: check that it's not empty!
+            XCTAssertNotEqual(ivOut, iv)
+
+            // The result will not match the ciphertext for which a different IV was used.
+            XCTAssertNotEqual(b, Bytes.decodeBase64(ciphertextB64)!)
+        } else {
+            XCTFail("Encrypt failed.")
+        }
+    }
+}

mobile/ios/SyncTests/DownloadTests.swift

@@ -0,0 +1,96 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+@testable import Sync
+
+import XCTest
+import SwiftyJSON
+
+func identity<T>(x: T) -> T {
+    return x
+}
+
+class DownloadTests: XCTestCase {
+    func loadEmptyBookmarksIntoServer(server: MockSyncServer) {
+        server.storeRecords(records: [], inCollection: "bookmarks")
+    }
+
+    func testBasicDownload() {
+        let server = getServer(preStart: loadEmptyBookmarksIntoServer)
+        server.storeRecords(records: [], inCollection: "bookmarks")
+
+        let storageClient = getClient(server: server)
+        let bookmarksClient = storageClient.clientForCollection("bookmarks", encrypter: getEncrypter())
+
+        let expectation = self.expectation(description: "Waiting for result.")
+        let deferred = bookmarksClient.getSince(0)
+        deferred >>== { response in
+            XCTAssertEqual(response.metadata.status, 200)
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testDownloadBatches() {
+        let guid1: GUID = "abcdefghijkl"
+        let ts1: Timestamp = 1326254123650
+        let rec1 = MockSyncServer.makeValidEnvelope(guid: guid1, modified: ts1)
+
+        let guid2: GUID = "bbcdefghijkl"
+        let ts2: Timestamp = 1326254125650
+        let rec2 = MockSyncServer.makeValidEnvelope(guid: guid2, modified: ts2)
+
+        let server = getServer(preStart: loadEmptyBookmarksIntoServer)
+
+        server.storeRecords(records: [rec1], inCollection: "clients", now: ts1)
+
+        let storageClient = getClient(server: server)
+        let bookmarksClient = storageClient.clientForCollection("clients", encrypter: getEncrypter())
+        let prefs = MockProfilePrefs()
+
+        let batcher = BatchingDownloader(collectionClient: bookmarksClient, basePrefs: prefs, collection: "clients")
+
+        let ic1 = InfoCollections(collections: ["clients": ts1])
+        let fetch1 = batcher.go(ic1, limit: 1).value
+        XCTAssertEqual(fetch1.successValue, DownloadEndState.complete)
+        XCTAssertEqual(0, batcher.baseTimestamp)    // This isn't updated until after success.
+        let records1 = batcher.retrieve()
+        XCTAssertEqual(1, records1.count)
+        XCTAssertEqual(guid1, records1[0].id)
+        batcher.advance()
+        XCTAssertNotEqual(0, batcher.baseTimestamp)
+
+        // Fetching again yields nothing, because the collection hasn't
+        // changed.
+        XCTAssertEqual(batcher.go(ic1, limit: 1).value.successValue, DownloadEndState.noNewData)
+
+        // More records. Start again.
+        let _ = batcher.reset().value
+
+        let ic2 = InfoCollections(collections: ["clients": ts2])
+        server.storeRecords(records: [rec2], inCollection: "clients", now: ts2)
+
+        let fetch2 = batcher.go(ic2, limit: 1).value
+        XCTAssertEqual(fetch2.successValue, DownloadEndState.incomplete)
+        let records2 = batcher.retrieve()
+        XCTAssertEqual(1, records2.count)
+        XCTAssertEqual(guid1, records2[0].id)
+        batcher.advance()
+
+        let fetch3 = batcher.go(ic2, limit: 1).value
+        XCTAssertEqual(fetch3.successValue, DownloadEndState.complete)
+        let records3 = batcher.retrieve()
+        XCTAssertEqual(1, records3.count)
+        XCTAssertEqual(guid2, records3[0].id)
+        batcher.advance()
+
+        let fetch4 = batcher.go(ic2, limit: 1).value
+        XCTAssertEqual(fetch4.successValue, DownloadEndState.noNewData)
+        let records4 = batcher.retrieve()
+        XCTAssertEqual(0, records4.count)
+        batcher.advance()
+    }
+}

mobile/ios/SyncTests/HistorySynchronizerTests.swift

@@ -0,0 +1,269 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Shared
+import Storage
+@testable import Sync
+import XCGLogger
+import Deferred
+
+import XCTest
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+class MockSyncDelegate: SyncDelegate {
+    func displaySentTab(for url: URL, title: String, from deviceName: String?) {
+    }
+}
+
+class DBPlace: Place {
+    var isDeleted = false
+    var shouldUpload = false
+    var serverModified: Timestamp?
+    var localModified: Timestamp?
+}
+
+class MockSyncableHistory {
+    var wasReset: Bool = false
+    var places = [GUID: DBPlace]()
+    var remoteVisits = [GUID: Set<Visit>]()
+    var localVisits = [GUID: Set<Visit>]()
+
+    init() {
+    }
+
+    fileprivate func placeForURL(url: String) -> DBPlace? {
+        return findOneValue(places) { $0.url == url }
+    }
+}
+
+extension MockSyncableHistory: ResettableSyncStorage {
+    func resetClient() -> Success {
+        self.wasReset = true
+        return succeed()
+    }
+}
+
+extension MockSyncableHistory: SyncableHistory {
+    // TODO: consider comparing the timestamp to local visits, perhaps opting to
+    // not delete the local place (and instead to give it a new GUID) if the visits
+    // are newer than the deletion.
+    // Obviously this'll behave badly during reconciling on other devices:
+    // they might apply our new record first, renaming their local copy of
+    // the old record with that URL, and thus bring all the old visits back to life.
+    // Desktop just finds by GUID then deletes by URL.
+    func deleteByGUID(_ guid: GUID, deletedAt: Timestamp) -> Deferred<Maybe<()>> {
+        self.remoteVisits.removeValue(forKey: guid)
+        self.localVisits.removeValue(forKey: guid)
+        self.places.removeValue(forKey: guid)
+
+        return succeed()
+    }
+
+    func hasSyncedHistory() -> Deferred<Maybe<Bool>> {
+        let has = self.places.values.contains(where: { $0.serverModified != nil })
+        return deferMaybe(has)
+    }
+
+    /**
+     * This assumes that the provided GUID doesn't already map to a different URL!
+     */
+    func ensurePlaceWithURL(_ url: String, hasGUID guid: GUID) -> Success {
+        // Find by URL.
+        if let existing = self.placeForURL(url: url) {
+            let p = DBPlace(guid: guid, url: url, title: existing.title)
+            p.isDeleted = existing.isDeleted
+            p.serverModified = existing.serverModified
+            p.localModified = existing.localModified
+            self.places.removeValue(forKey: existing.guid)
+            self.places[guid] = p
+        }
+
+        return succeed()
+    }
+
+    func storeRemoteVisits(_ visits: [Visit], forGUID guid: GUID) -> Success {
+        // Strip out existing local visits.
+        // We trust that an identical timestamp and type implies an identical visit.
+        var remote = Set<Visit>(visits)
+        if let local = self.localVisits[guid] {
+            remote.subtract(local)
+        }
+
+        // Visits are only ever added.
+        if var r = self.remoteVisits[guid] {
+            r.formUnion(remote)
+        } else {
+            self.remoteVisits[guid] = remote
+        }
+        return succeed()
+    }
+
+    func insertOrUpdatePlace(_ place: Place, modified: Timestamp) -> Deferred<Maybe<GUID>> {
+        // See if we've already applied this one.
+        if let existingModified = self.places[place.guid]?.serverModified {
+            if existingModified == modified {
+                log.debug("Already seen unchanged record \(place.guid).")
+                return deferMaybe(place.guid)
+            }
+        }
+
+        // Make sure that we collide with any matching URLs -- whether locally
+        // modified or not. Then overwrite the upstream and merge any local changes.
+        return self.ensurePlaceWithURL(place.url, hasGUID: place.guid)
+            >>> {
+                if let existingLocal = self.places[place.guid] {
+                    if existingLocal.shouldUpload {
+                        log.debug("Record \(existingLocal.guid) modified locally and remotely.")
+                        log.debug("Local modified: \(existingLocal.localModified ??? "nil"); remote: \(modified).")
+
+                        // Should always be a value if marked as changed.
+                        if let localModified = existingLocal.localModified, localModified > modified {
+                            // Nothing to do: it's marked as changed.
+                            log.debug("Discarding remote non-visit changes!")
+                            self.places[place.guid]?.serverModified = modified
+                            return deferMaybe(place.guid)
+                        } else {
+                            log.debug("Discarding local non-visit changes!")
+                            self.places[place.guid]?.shouldUpload = false
+                        }
+                    } else {
+                        log.debug("Remote record exists, but has no local changes.")
+                    }
+                } else {
+                    log.debug("Remote record doesn't exist locally.")
+                }
+
+                // Apply the new remote record.
+                let p = DBPlace(guid: place.guid, url: place.url, title: place.title)
+                p.localModified = Date.now()
+                p.serverModified = modified
+                p.isDeleted = false
+                self.places[place.guid] = p
+                return deferMaybe(place.guid)
+        }
+    }
+
+    func getModifiedHistoryToUpload() -> Deferred<Maybe<[(Place, [Visit])]>> {
+        // TODO.
+        return deferMaybe([])
+    }
+
+    func getDeletedHistoryToUpload() -> Deferred<Maybe<[GUID]>> {
+        // TODO.
+        return deferMaybe([])
+    }
+
+    func markAsSynchronized(_: [GUID], modified: Timestamp) -> Deferred<Maybe<Timestamp>> {
+        // TODO
+        return deferMaybe(0)
+    }
+
+    func markAsDeleted(_: [GUID]) -> Success {
+        // TODO
+        return succeed()
+    }
+
+    func onRemovedAccount() -> Success {
+        // TODO
+        return succeed()
+    }
+
+    func doneApplyingRecordsAfterDownload() -> Success {
+        return succeed()
+    }
+
+    func doneUpdatingMetadataAfterUpload() -> Success {
+        return succeed()
+    }
+}
+
+class HistorySynchronizerTests: XCTestCase {
+    private func applyRecords(records: [Record<HistoryPayload>], toStorage storage: SyncableHistory & ResettableSyncStorage) -> (synchronizer: HistorySynchronizer, prefs: Prefs, scratchpad: Scratchpad) {
+        let delegate = MockSyncDelegate()
+
+        // We can use these useless values because we're directly injecting decrypted
+        // payloads; no need for real keys etc.
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let synchronizer = HistorySynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+
+        let expectation = self.expectation(description: "Waiting for application.")
+        var succeeded = false
+        synchronizer.applyIncomingToStorage(storage, records: records)
+                    .upon({ result in
+            succeeded = result.isSuccess
+            expectation.fulfill()
+        })
+
+        waitForExpectations(timeout: 10, handler: nil)
+        XCTAssertTrue(succeeded, "Application succeeded.")
+        return (synchronizer, prefs, scratchpad)
+    }
+
+    func testRecordSerialization() {
+        let id = "abcdefghi"
+        let modified: Timestamp = 0    // Ignored in upload serialization.
+        let sortindex = 1
+        let ttl = 12345
+        let json: JSON = JSON([
+            "id": id,
+            "visits": [],
+            "histUri": "http://www.slideshare.net/swadpasc/bpm-edu-netseminarscepwithreactionrulemlprova",
+            "title": "Semantic Complex Event Processing with \(Character(UnicodeScalar(11)))Reaction RuleML 1.0 and Prova",
+            ])
+        let payload = HistoryPayload(json)
+        let record = Record<HistoryPayload>(id: id, payload: payload, modified: modified, sortindex: sortindex, ttl: ttl)
+        let k = KeyBundle.random()
+        let s = k.serializer({ (x: HistoryPayload) -> JSON in x.json })
+        let converter = { (x: JSON) -> HistoryPayload in HistoryPayload(x) }
+        let f = k.factory(converter)
+        let serialized = s(record)!
+        let envelope = EnvelopeJSON(serialized)
+
+        // With a badly serialized payload, we get null JSON!
+        let p = f(envelope.payload)
+        XCTAssertFalse(p!.json.isNull())
+
+        // When we round-trip, the payload should be valid, and we'll get a record here.
+        let roundtripped = Record<HistoryPayload>.fromEnvelope(envelope, payloadFactory: f)
+        XCTAssertNotNil(roundtripped)
+    }
+
+    func testApplyRecords() {
+        let earliest = Date.now()
+
+        let empty = MockSyncableHistory()
+        let noRecords = [Record<HistoryPayload>]()
+
+        // Apply no records.
+        let _ = self.applyRecords(records: noRecords, toStorage: empty)
+
+        // Hey look! Nothing changed.
+        XCTAssertTrue(empty.places.isEmpty)
+        XCTAssertTrue(empty.remoteVisits.isEmpty)
+        XCTAssertTrue(empty.localVisits.isEmpty)
+
+        // Apply one remote record.
+        let jA = "{\"id\":\"aaaaaa\",\"histUri\":\"http://foo.com/\",\"title\": \"ñ\",\"visits\":[{\"date\":1222222222222222,\"type\":1}]}"
+        let pA = HistoryPayload.fromJSON(JSON(parseJSON: jA))!
+        let rA = Record<HistoryPayload>(id: "aaaaaa", payload: pA, modified: earliest + 10000, sortindex: 123, ttl: 1000000)
+
+        let (_, prefs, _) = self.applyRecords(records: [rA], toStorage: empty)
+
+        // The record was stored. This is checking our mock implementation, but real storage should work, too!
+
+        XCTAssertEqual(1, empty.places.count)
+        XCTAssertEqual(1, empty.remoteVisits.count)
+        XCTAssertEqual(1, empty.remoteVisits["aaaaaa"]!.count)
+        XCTAssertTrue(empty.localVisits.isEmpty)
+
+        // Test resetting now that we have a timestamp.
+        XCTAssertFalse(empty.wasReset)
+        XCTAssertTrue(HistorySynchronizer.resetSynchronizerWithStorage(empty, basePrefs: prefs, collection: "history").value.isSuccess)
+        XCTAssertTrue(empty.wasReset)
+    }
+}

mobile/ios/SyncTests/Info.plist

@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>en</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>BNDL</string>
+	<key>CFBundleShortVersionString</key>
+	<string>10.6</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+</dict>
+</plist>

mobile/ios/SyncTests/InfoTests.swift

@@ -0,0 +1,56 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+@testable import Sync
+
+import XCTest
+import SwiftyJSON
+
+class InfoTests: XCTestCase {
+    func testSame() {
+        let empty = JSON(parseJSON: "{}")
+
+        let oneA = JSON(parseJSON: "{\"foo\": 1234.0, \"bar\": 456.12}")
+        let oneB = JSON(parseJSON: "{\"bar\": 456.12, \"foo\": 1234.0}")
+        
+        let twoA = JSON(parseJSON: "{\"bar\": 456.12}")
+        let twoB = JSON(parseJSON: "{\"foo\": 1234.0}")
+
+        let iEmpty = InfoCollections.fromJSON(empty)!
+        let iOneA = InfoCollections.fromJSON(oneA)!
+        let iOneB = InfoCollections.fromJSON(oneB)!
+        let iTwoA = InfoCollections.fromJSON(twoA)!
+        let iTwoB = InfoCollections.fromJSON(twoB)!
+
+        XCTAssertTrue(iEmpty.same(iEmpty, collections: nil))
+        XCTAssertTrue(iEmpty.same(iEmpty, collections: []))
+        XCTAssertTrue(iEmpty.same(iEmpty, collections: ["anything"]))
+        
+        XCTAssertTrue(iEmpty.same(iOneA, collections: []))
+        XCTAssertTrue(iEmpty.same(iOneA, collections: ["anything"]))
+        XCTAssertTrue(iOneA.same(iEmpty, collections: []))
+        XCTAssertTrue(iOneA.same(iEmpty, collections: ["anything"]))
+
+        XCTAssertFalse(iEmpty.same(iOneA, collections: ["foo"]))
+        XCTAssertFalse(iOneA.same(iEmpty, collections: ["foo"]))
+        XCTAssertFalse(iEmpty.same(iOneA, collections: nil))
+        XCTAssertFalse(iOneA.same(iEmpty, collections: nil))
+
+        XCTAssertTrue(iOneA.same(iOneA, collections: nil))
+        XCTAssertTrue(iOneA.same(iOneA, collections: ["foo", "bar", "baz"]))
+        XCTAssertTrue(iOneA.same(iOneB, collections: ["foo", "bar", "baz"]))
+        XCTAssertTrue(iOneB.same(iOneA, collections: ["foo", "bar", "baz"]))
+        
+        XCTAssertFalse(iTwoA.same(iOneA, collections: nil))
+        XCTAssertTrue(iTwoA.same(iOneA, collections: ["bar", "baz"]))
+        XCTAssertTrue(iOneA.same(iTwoA, collections: ["bar", "baz"]))
+        XCTAssertTrue(iTwoB.same(iOneA, collections: ["foo", "baz"]))
+        
+        XCTAssertFalse(iTwoA.same(iTwoB, collections: nil))
+        XCTAssertFalse(iTwoA.same(iTwoB, collections: ["foo"]))
+        XCTAssertFalse(iTwoA.same(iTwoB, collections: ["bar"]))
+    }
+}

mobile/ios/SyncTests/LiveStorageClientTests.swift

@@ -0,0 +1,131 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Account
+import Foundation
+import FxA
+import Shared
+import Deferred
+@testable import Sync
+
+import XCTest
+import SwiftyJSON
+
+private class KeyFetchError: MaybeErrorType {
+    var description: String {
+        return "key fetch error"
+    }
+}
+
+class LiveStorageClientTests: LiveAccountTest {
+    func getKeys(kB: Data, token: TokenServerToken) -> Deferred<Maybe<Record<KeysPayload>>> {
+        let endpoint = token.api_endpoint
+        XCTAssertTrue(endpoint.range(of: "services.mozilla.com") != nil, "We got a Sync server.")
+
+        let cryptoURI = URL(string: endpoint)
+        let authorizer: Authorizer = {
+            (r: URLRequest) -> URLRequest in
+            var request = r
+            let helper = HawkHelper(id: token.id, key: token.key.data(using: String.Encoding.utf8, allowLossyConversion: false)!)
+            request.addValue(helper.getAuthorizationValueFor(r), forHTTPHeaderField: "Authorization")
+            return request
+        }
+
+        let keyBundle: KeyBundle = KeyBundle.fromKB(kB as Data)
+        let encoder = RecordEncoder<KeysPayload>(decode: { KeysPayload($0) }, encode: { $0.json })
+        let encrypter = Keys(defaultBundle: keyBundle).encrypter("crypto", encoder: encoder)
+
+        let workQueue = DispatchQueue.global(qos: DispatchQoS.default.qosClass)
+        let resultQueue = DispatchQueue.main
+        let backoff = MockBackoffStorage()
+
+        let storageClient = Sync15StorageClient(serverURI: cryptoURI!, authorizer: authorizer, workQueue: workQueue, resultQueue: resultQueue, backoff: backoff)
+        let keysFetcher = storageClient.clientForCollection("crypto", encrypter: encrypter)
+
+        return keysFetcher.get("keys").map({ res in
+            // Unwrap the response.
+            if let r = res.successValue {
+                return Maybe(success: r.value)
+            }
+            return Maybe(failure: KeyFetchError())
+        })
+    }
+
+    func getState(user: String, password: String) -> Deferred<Maybe<FxAState>> {
+        let err: NSError = NSError(domain: FxAClientErrorDomain, code: 0, userInfo: nil)
+        return Deferred(value: Maybe<FxAState>(failure: FxAClientError.local(err)))
+    }
+
+    func getTokenAndDefaultKeys() -> Deferred<Maybe<(TokenServerToken, KeyBundle)>> {
+        let authState = self.syncAuthState(Date.now())
+
+        let keysPayload: Deferred<Maybe<Record<KeysPayload>>> = authState.bind { tokenResult in
+            if let (token, forKey) = tokenResult.successValue {
+                return self.getKeys(kB: forKey, token: token)
+            }
+            XCTAssertEqual(tokenResult.failureValue!.description, "")
+            return Deferred(value: Maybe(failure: KeyFetchError()))
+        }
+
+        let result = Deferred<Maybe<(TokenServerToken, KeyBundle)>>()
+        keysPayload.upon { res in
+            if let rec = res.successValue {
+                XCTAssert(rec.id == "keys", "GUID is correct.")
+                XCTAssert(rec.modified > 1000, "modified is sane.")
+                let payload: KeysPayload = rec.payload as KeysPayload
+                print("Body: \(payload.json.stringValue() ?? "nil")", terminator: "\n")
+                XCTAssert(rec.id == "keys", "GUID inside is correct.")
+                if let keys = payload.defaultKeys {
+                    // Extracting the token like this is not great, but...
+                    result.fill(Maybe(success: (authState.value.successValue!.token, keys)))
+                    return
+                }
+            }
+
+            result.fill(Maybe(failure: KeyFetchError()))
+        }
+        return result
+    }
+
+    func testLive() {
+        let expctn = expectation(description: "Waiting on value.")
+        let deferred = getTokenAndDefaultKeys()
+        deferred.upon { res in
+            if let (_, _) = res.successValue {
+                print("Yay", terminator: "\n")
+            } else {
+                XCTAssertEqual(res.failureValue!.description, "")
+            }
+            expctn.fulfill()
+        }
+
+        // client: mgWl22CIzHiE
+        waitForExpectations(timeout: 20) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testStateMachine() {
+        let expctn = expectation(description: "Waiting on value.")
+        let authState = self.getAuthState(Date.now())
+
+        let d = chainDeferred(authState, f: { SyncStateMachine(prefs: MockProfilePrefs()).toReady($0) })
+
+        d.upon { result in
+            if let ready = result.successValue {
+                XCTAssertTrue(ready.collectionKeys.defaultBundle.encKey.count == 32)
+                XCTAssertTrue(ready.scratchpad.global != nil)
+                if let clients = ready.scratchpad.global?.value.engines["clients"] {
+                    XCTAssertTrue(clients.syncID.characters.count == 12)
+                }
+            }
+            XCTAssertTrue(result.isSuccess)
+            expctn.fulfill()
+        }
+
+        waitForExpectations(timeout: 20) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+}

mobile/ios/SyncTests/MetaGlobalTests.swift

@@ -0,0 +1,624 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+@testable import Account
+import Foundation
+import Shared
+import Storage
+@testable import Sync
+import XCGLogger
+import Deferred
+
+import XCTest
+import SwiftyJSON
+
+private let log = Logger.syncLogger
+
+class MockSyncAuthState: SyncAuthState {
+
+    let serverRoot: String
+    let kB: Data
+
+    var deviceID: String? {
+        return "mock_device_id"
+    }
+
+    init(serverRoot: String, kB: Data) {
+        self.serverRoot = serverRoot
+        self.kB = kB
+    }
+
+    func invalidate() {
+    }
+
+    func token(_ now: Timestamp, canBeExpired: Bool) -> Deferred<Maybe<(token: TokenServerToken, forKey: Data)>> {
+        let token = TokenServerToken(id: "id", key: "key", api_endpoint: serverRoot, uid: UInt64(0), hashedFxAUID: "",
+            durationInSeconds: UInt64(5 * 60), remoteTimestamp: Timestamp(now - 1))
+        return deferMaybe((token, self.kB))
+    }
+}
+
+class MetaGlobalTests: XCTestCase {
+    var server: MockSyncServer!
+    var serverRoot: String!
+    var kB: Data!
+    var syncPrefs: Prefs!
+    var authState: SyncAuthState!
+    var stateMachine: SyncStateMachine!
+
+    override func setUp() {
+        kB = Data.randomOfLength(32)!
+        server = MockSyncServer(username: "1234567")
+        server.start()
+        serverRoot = server.baseURL
+        syncPrefs = MockProfilePrefs()
+        authState = MockSyncAuthState(serverRoot: serverRoot, kB: kB)
+        stateMachine = SyncStateMachine(prefs: syncPrefs)
+    }
+
+    func storeMetaGlobal(metaGlobal: MetaGlobal) {
+        let envelope = EnvelopeJSON(JSON(object: [
+            "id": "global",
+            "collection": "meta",
+            "payload": metaGlobal.asPayload().json.stringValue()!,
+            "modified": Double(Date.now())/1000]))
+        server.storeRecords(records: [envelope], inCollection: "meta")
+    }
+
+    func storeCryptoKeys(keys: Keys) {
+        let keyBundle = KeyBundle.fromKB(kB)
+        let record = Record(id: "keys", payload: keys.asPayload())
+        let envelope = EnvelopeJSON(keyBundle.serializer({ $0.json })(record)!)
+        server.storeRecords(records: [envelope], inCollection: "crypto")
+    }
+
+    func assertFreshStart(ready: Ready?, after: Timestamp) {
+        XCTAssertNotNil(ready)
+        guard let ready = ready else {
+            return
+        }
+        // We should have wiped.
+        // We should have uploaded new meta/global and crypto/keys.
+        XCTAssertGreaterThan(server.collections["meta"]?.records["global"]?.modified ?? 0, after)
+        XCTAssertGreaterThan(server.collections["meta"]?.modified ?? 0, after)
+        XCTAssertGreaterThan(server.collections["crypto"]?.records["keys"]?.modified ?? 0, after)
+        XCTAssertGreaterThan(server.collections["crypto"]?.modified ?? 0, after)
+
+        // And we should have downloaded meta/global and crypto/keys.
+        XCTAssertNotNil(ready.scratchpad.global)
+        XCTAssertNotNil(ready.scratchpad.keys)
+
+        // We should have the default engine configuration.
+        XCTAssertNotNil(ready.scratchpad.engineConfiguration)
+        guard let engineConfiguration = ready.scratchpad.engineConfiguration else {
+            return
+        }
+        XCTAssertEqual(engineConfiguration.enabled.sorted(), ["addons", "bookmarks", "clients", "forms", "history", "passwords", "prefs", "tabs"])
+        XCTAssertEqual(engineConfiguration.declined, [])
+
+        // Basic verifications.
+        XCTAssertEqual(ready.collectionKeys.defaultBundle.encKey.count, 32)
+        if let clients = ready.scratchpad.global?.value.engines["clients"] {
+            XCTAssertTrue(clients.syncID.characters.count == 12)
+        }
+    }
+
+    func testMetaGlobalVersionTooNew() {
+        // There's no recovery from a meta/global version "in the future": just bail out with an UpgradeRequiredError.
+        storeMetaGlobal(metaGlobal: MetaGlobal(syncID: "id", storageVersion: 6, engines: [String: EngineMeta](), declined: []))
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "clientUpgradeRequired"])
+            XCTAssertNotNil(result.failureValue as? ClientUpgradeRequiredError)
+            XCTAssertNil(result.successValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testMetaGlobalVersionTooOld() {
+        // To recover from a meta/global version "in the past", fresh start.
+        storeMetaGlobal(metaGlobal: MetaGlobal(syncID: "id", storageVersion: 4, engines: [String: EngineMeta](), declined: []))
+
+        let afterStores = Date.now()
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "remoteUpgradeRequired",
+                "freshStartRequired", "serverConfigurationRequired", "initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            self.assertFreshStart(ready: result.successValue, after: afterStores)
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testMetaGlobalMissing() {
+        // To recover from a missing meta/global, fresh start.
+        let afterStores = Date.now()
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "missingMetaGlobal",
+                "freshStartRequired", "serverConfigurationRequired", "initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            self.assertFreshStart(ready: result.successValue, after: afterStores)
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testCryptoKeysMissing() {
+        // To recover from a missing crypto/keys, fresh start.
+        storeMetaGlobal(metaGlobal: createMetaGlobal())
+
+        let afterStores = Date.now()
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "missingCryptoKeys", "freshStartRequired", "serverConfigurationRequired", "initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            self.assertFreshStart(ready: result.successValue, after: afterStores)
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testMetaGlobalAndCryptoKeysFresh() {
+        // When encountering a valid meta/global and crypto/keys, advance smoothly.
+        let metaGlobal = MetaGlobal(syncID: "id", storageVersion: 5, engines: [String: EngineMeta](), declined: [])
+        let cryptoKeys = Keys.random()
+        storeMetaGlobal(metaGlobal: metaGlobal)
+        storeCryptoKeys(keys: cryptoKeys)
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+
+            // And we should have downloaded meta/global and crypto/keys.
+            XCTAssertEqual(ready.scratchpad.global?.value, metaGlobal)
+            XCTAssertEqual(ready.scratchpad.keys?.value, cryptoKeys)
+
+            // We should have marked all local engines for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["bookmarks", "clients", "history", "passwords", "tabs"])
+            ready.clearLocalCommands()
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        let afterFirstSync = Date.now()
+
+        // Now, run through the state machine again.  Nothing's changed remotely, so we should advance quickly.
+        let secondExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "hasMetaGlobal", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+            // And we should have not downloaded a fresh meta/global or crypto/keys.
+            XCTAssertLessThan(ready.scratchpad.global?.timestamp ?? Timestamp.max, afterFirstSync)
+            XCTAssertLessThan(ready.scratchpad.keys?.timestamp ?? Timestamp.max, afterFirstSync)
+
+            // We should not have marked any local engines for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), [])
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            secondExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testFailingOptimisticStateMachine() {
+        // We test only the optimistic state machine, knowing it will need to go through 
+        // needsFreshMetaGlobal, and fail.
+        let metaGlobal = MetaGlobal(syncID: "id", storageVersion: 5, engines: [String: EngineMeta](), declined: [])
+        let cryptoKeys = Keys.random()
+        storeMetaGlobal(metaGlobal: metaGlobal)
+        storeCryptoKeys(keys: cryptoKeys)
+
+        stateMachine = SyncStateMachine(prefs: syncPrefs, allowingStates: SyncStateMachine.OptimisticStates)
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal"])
+            XCTAssertNotNil(result.failureValue)
+            if let failure = result.failureValue as? DisallowedStateError {
+                XCTAssertEqual(failure.state, SyncStateLabel.NeedsFreshMetaGlobal)
+            } else {
+                XCTFail("SyncStatus failed, but with a different error")
+            }
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testHappyOptimisticStateMachine() {
+        // We should be able to quickly progress through a constrained (a.k.a. optimistic) state machine
+        let metaGlobal = MetaGlobal(syncID: "id", storageVersion: 5, engines: [String: EngineMeta](), declined: [])
+        let cryptoKeys = Keys.random()
+        storeMetaGlobal(metaGlobal: metaGlobal)
+        storeCryptoKeys(keys: cryptoKeys)
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        // Now, run through the state machine again.  Nothing's changed remotely, so we should advance quickly.
+        // We should be able to use this 'optimistic' path in an extension.
+        stateMachine = SyncStateMachine(prefs: syncPrefs, allowingStates: SyncStateMachine.OptimisticStates)
+
+        let secondExpectation = self.expectation(description: "Waiting on value.")
+
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "hasMetaGlobal", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            secondExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testUpdatedCryptoKeys() {
+        // When encountering a valid meta/global and crypto/keys, advance smoothly.
+        let metaGlobal = MetaGlobal(syncID: "id", storageVersion: 5, engines: [String: EngineMeta](), declined: [])
+        let cryptoKeys = Keys.random()
+        cryptoKeys.collectionKeys.updateValue(KeyBundle.random(), forKey: "bookmarks")
+        cryptoKeys.collectionKeys.updateValue(KeyBundle.random(), forKey: "clients")
+        storeMetaGlobal(metaGlobal: metaGlobal)
+        storeCryptoKeys(keys: cryptoKeys)
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+
+            // And we should have downloaded meta/global and crypto/keys.
+            XCTAssertEqual(ready.scratchpad.global?.value, metaGlobal)
+            XCTAssertEqual(ready.scratchpad.keys?.value, cryptoKeys)
+
+            // We should have marked all local engines for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["bookmarks", "clients", "history", "passwords", "tabs"])
+            ready.clearLocalCommands()
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        let afterFirstSync = Date.now()
+
+        // Store a fresh crypto/keys, with the same default key, one identical collection key, and one changed collection key.
+        let freshCryptoKeys = Keys(defaultBundle: cryptoKeys.defaultBundle)
+        freshCryptoKeys.collectionKeys.updateValue(cryptoKeys.forCollection("bookmarks"), forKey: "bookmarks")
+        freshCryptoKeys.collectionKeys.updateValue(KeyBundle.random(), forKey: "clients")
+        storeCryptoKeys(keys: freshCryptoKeys)
+
+        // Now, run through the state machine again.
+        let secondExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+            // And we should have not downloaded a fresh meta/global ...
+            XCTAssertLessThan(ready.scratchpad.global?.timestamp ?? Timestamp.max, afterFirstSync)
+            // ... but we should have downloaded a fresh crypto/keys.
+            XCTAssertGreaterThanOrEqual(ready.scratchpad.keys?.timestamp ?? Timestamp.min, afterFirstSync)
+
+            // We should have marked only the local engine with a changed key for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["clients"])
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            secondExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        let afterSecondSync = Date.now()
+
+        // Store a fresh crypto/keys, with a changed default key and one identical collection key, and one changed collection key.
+        let freshCryptoKeys2 = Keys.random()
+        freshCryptoKeys2.collectionKeys.updateValue(freshCryptoKeys.forCollection("bookmarks"), forKey: "bookmarks")
+        freshCryptoKeys2.collectionKeys.updateValue(KeyBundle.random(), forKey: "clients")
+        storeCryptoKeys(keys: freshCryptoKeys2)
+
+        // Now, run through the state machine again.
+        let thirdExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+            // And we should have not downloaded a fresh meta/global ...
+            XCTAssertLessThan(ready.scratchpad.global?.timestamp ?? Timestamp.max, afterSecondSync)
+            // ... but we should have downloaded a fresh crypto/keys.
+            XCTAssertGreaterThanOrEqual(ready.scratchpad.keys?.timestamp ?? Timestamp.min, afterSecondSync)
+
+            // We should have marked all local engines as needing reset, except for the engine whose key remained constant.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["clients", "history", "passwords", "tabs"])
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            thirdExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        let afterThirdSync = Date.now()
+
+        // Now store a random crypto/keys, with a different default key (and no bulk keys).
+        let randomCryptoKeys = Keys.random()
+        storeCryptoKeys(keys: randomCryptoKeys)
+
+        // Now, run through the state machine again.
+        let fourthExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+            // And we should have not downloaded a fresh meta/global ...
+            XCTAssertLessThan(ready.scratchpad.global?.timestamp ?? Timestamp.max, afterThirdSync)
+            // ... but we should have downloaded a fresh crypto/keys.
+            XCTAssertGreaterThanOrEqual(ready.scratchpad.keys?.timestamp ?? Timestamp.min, afterThirdSync)
+
+            // We should have marked all local engines for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["bookmarks", "clients", "history", "passwords", "tabs"])
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            fourthExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    private func createUnusualMetaGlobal() -> MetaGlobal {
+        let metaGlobal = MetaGlobal(syncID: "id", storageVersion: 5,
+            engines: ["bookmarks": EngineMeta(version: 1, syncID: "bookmarks"), "unknownEngine1": EngineMeta(version: 2, syncID: "engineId1")],
+            declined: ["clients", "forms", "unknownEngine2"])
+        return metaGlobal
+    }
+
+    func testEngineConfigurations() {
+        // When encountering a valid meta/global and crypto/keys, advance smoothly.  Keep the engine configuration for re-upload.
+        let metaGlobal = createUnusualMetaGlobal()
+        let cryptoKeys = Keys.random()
+        storeMetaGlobal(metaGlobal: metaGlobal)
+        storeCryptoKeys(keys: cryptoKeys)
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+
+            // We should have saved the engine configuration.
+            XCTAssertNotNil(ready.scratchpad.engineConfiguration)
+            guard let engineConfiguration = ready.scratchpad.engineConfiguration else {
+                return
+            }
+            XCTAssertEqual(engineConfiguration, metaGlobal.engineConfiguration())
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        // Wipe meta/global.
+        server.removeAllItemsFromCollection(collection: "meta", atTime: Date.now())
+
+        // Now, run through the state machine again.  We should produce and upload a meta/global reflecting our engine configuration.
+        let secondExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "missingMetaGlobal", "freshStartRequired", "serverConfigurationRequired", "initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+
+            // The downloaded meta/global should reflect our local engine configuration.
+            XCTAssertNotNil(ready.scratchpad.global)
+            guard let global = ready.scratchpad.global?.value else {
+                return
+            }
+            XCTAssertEqual(global.engineConfiguration(), metaGlobal.engineConfiguration())
+
+            // We should have the same cached engine configuration.
+            XCTAssertNotNil(ready.scratchpad.engineConfiguration)
+            guard let engineConfiguration = ready.scratchpad.engineConfiguration else {
+                return
+            }
+            XCTAssertEqual(engineConfiguration, metaGlobal.engineConfiguration())
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            secondExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+
+    func testMetaGlobalModified() {
+        // When encountering a valid meta/global and crypto/keys, advance smoothly.
+        let metaGlobal = createUnusualMetaGlobal()
+        let cryptoKeys = Keys.random()
+        storeMetaGlobal(metaGlobal: metaGlobal)
+        storeCryptoKeys(keys: cryptoKeys)
+
+        let expectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+
+            // And we should have downloaded meta/global and crypto/keys.
+            XCTAssertEqual(ready.scratchpad.global?.value, metaGlobal)
+            XCTAssertEqual(ready.scratchpad.keys?.value, cryptoKeys)
+
+            // We should have marked all local engines for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["bookmarks", "clients", "history", "passwords", "tabs"])
+            XCTAssertEqual(ready.enginesEnabled(), [])
+            XCTAssertEqual(ready.enginesDisabled(), [])
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            expectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        let afterFirstSync = Date.now()
+
+        // Store a meta/global with a new global syncID.
+        let newMetaGlobal = metaGlobal.withSyncID("newID")
+        storeMetaGlobal(metaGlobal: newMetaGlobal)
+
+        // Now, run through the state machine again.
+        let secondExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "needsFreshCryptoKeys", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+            // And we should have downloaded a fresh meta/global ...
+            XCTAssertGreaterThanOrEqual(ready.scratchpad.global?.timestamp ?? Timestamp.min, afterFirstSync)
+            // ... and we should have downloaded a fresh crypto/keys -- but its timestamp is identical to the old one!
+            // Therefore, the "needsFreshCryptoKeys" stage above is our test that we re-downloaded crypto/keys.
+
+            // We should have marked all local engines for reset.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["bookmarks", "clients", "history", "passwords", "tabs"])
+
+            // And our engine configuration should be unchanged.
+            XCTAssertNotNil(ready.scratchpad.global)
+            guard let global = ready.scratchpad.global?.value else {
+                return
+            }
+            XCTAssertEqual(global.engineConfiguration(), metaGlobal.engineConfiguration())
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            secondExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+
+        // Now store a meta/global with a changed engine syncID, a new engine, and a new declined entry.
+        var engines = newMetaGlobal.engines
+        engines.updateValue(EngineMeta(version: 1, syncID: Bytes.generateGUID()), forKey: "bookmarks")
+        engines.updateValue(EngineMeta(version: 1, syncID: Bytes.generateGUID()), forKey: "forms")
+        engines.removeValue(forKey: "unknownEngine1")
+        var declined = newMetaGlobal.declined.filter({ $0 != "forms" })
+        declined.append("unknownEngine1")
+        let secondMetaGlobal = MetaGlobal(syncID: newMetaGlobal.syncID, storageVersion: 5, engines: engines, declined: declined)
+        storeMetaGlobal(metaGlobal: secondMetaGlobal)
+        syncPrefs.removeObjectForKey("scratchpad.localCommands")
+
+        // Now, run through the state machine again.
+        let thirdExpectation = self.expectation(description: "Waiting on value.")
+        stateMachine.toReady(authState).upon { result in
+            XCTAssertEqual(self.stateMachine.stateLabelSequence.map { $0.rawValue }, ["initialWithLiveToken", "initialWithLiveTokenAndInfo", "needsFreshMetaGlobal", "resolveMetaGlobalVersion", "resolveMetaGlobalContent", "hasMetaGlobal", "hasFreshCryptoKeys", "ready"])
+            XCTAssertNotNil(result.successValue)
+            guard let ready = result.successValue else {
+                return
+            }
+            // And we should have downloaded a fresh meta/global ...
+            XCTAssertGreaterThanOrEqual(ready.scratchpad.global?.timestamp ?? Timestamp.min, afterFirstSync)
+            // ... and we should have downloaded a fresh crypto/keys -- but its timestamp is identical to the old one!
+            // Therefore, the "needsFreshCryptoKeys" stage above is our test that we re-downloaded crypto/keys.
+
+            // We should have marked the changed engine for local reset, and identified the enabled and disabled engines.
+            XCTAssertEqual(ready.collectionsThatNeedLocalReset(), ["bookmarks"])
+            XCTAssertEqual(ready.enginesEnabled(), ["forms"])
+            XCTAssertEqual(ready.enginesDisabled(), ["unknownEngine1"])
+
+            // And our engine configuration should reflect the new meta/global on the server.
+            XCTAssertNotNil(ready.scratchpad.global)
+            guard let global = ready.scratchpad.global?.value else {
+                return
+            }
+            XCTAssertEqual(global.engineConfiguration(), secondMetaGlobal.engineConfiguration())
+
+            XCTAssertTrue(result.isSuccess)
+            XCTAssertNil(result.failureValue)
+            thirdExpectation.fulfill()
+        }
+
+        waitForExpectations(timeout: 2000) { (error) in
+            XCTAssertNil(error, "Error: \(error ??? "nil")")
+        }
+    }
+}

mobile/ios/SyncTests/MockSyncServer.swift

@@ -0,0 +1,453 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import GCDWebServers
+import SwiftyJSON
+@testable import Sync
+
+import XCTest
+
+private let log = Logger.syncLogger
+
+private func optTimestamp(x: AnyObject?) -> Timestamp? {
+    guard let str = x as? String else {
+        return nil
+    }
+    return decimalSecondsStringToTimestamp(str)
+}
+
+private func optStringArray(x: AnyObject?) -> [String]? {
+    guard let str = x as? String else {
+        return nil
+    }
+    return str.components(separatedBy: ",").map { $0.trimmingCharacters(in:NSCharacterSet.whitespacesAndNewlines) }
+}
+
+private struct SyncRequestSpec {
+    let collection: String
+    let id: String?
+    let ids: [String]?
+    let limit: Int?
+    let offset: String?
+    let sort: SortOption?
+    let newer: Timestamp?
+    let full: Bool
+
+    static func fromRequest(request: GCDWebServerRequest) -> SyncRequestSpec? {
+        // Input is "/1.5/user/storage/collection", possibly with "/id" at the end.
+        // That means we get five or six path components here, the first being empty.
+
+        let parts = request.path!.components(separatedBy: "/").filter { !$0.isEmpty }
+        let id: String?
+        let query = request.query as! [String: AnyObject]
+        let ids = optStringArray(x: query["ids"])
+        let newer = optTimestamp(x: query["newer"])
+        let full: Bool = query["full"] != nil
+
+        let limit: Int?
+        if let lim = query["limit"] as? String {
+            limit = Int(lim)
+        } else {
+            limit = nil
+        }
+
+        let offset = query["offset"] as? String
+
+        let sort: SortOption?
+        switch query["sort"] as? String ?? "" {
+        case "oldest":
+            sort = SortOption.OldestFirst
+        case "newest":
+            sort = SortOption.NewestFirst
+        case "index":
+            sort = SortOption.Index
+        default:
+            sort = nil
+        }
+
+        if parts.count < 4 {
+            return nil
+        }
+
+        if parts[2] != "storage" {
+            return nil
+        }
+
+        // Use dropFirst, you say! It's buggy.
+        switch parts.count {
+        case 4:
+            id = nil
+        case 5:
+            id = parts[4]
+        default:
+            // Uh oh.
+            return nil
+        }
+        return SyncRequestSpec(collection: parts[3], id: id, ids: ids, limit: limit, offset: offset, sort: sort, newer: newer, full: full)
+    }
+}
+
+struct SyncDeleteRequestSpec {
+    let collection: String?
+    let id: GUID?
+    let ids: [GUID]?
+    let wholeCollection: Bool
+
+    static func fromRequest(request: GCDWebServerRequest) -> SyncDeleteRequestSpec? {
+        // Input is "/1.5/user{/storage{/collection{/id}}}".
+        // That means we get four, five, or six path components here, the first being empty.
+        return SyncDeleteRequestSpec.fromPath(path: request.path!, withQuery: request.query as! [NSString : AnyObject])
+    }
+
+    static func fromPath(path: String, withQuery query: [NSString: AnyObject]) -> SyncDeleteRequestSpec? {
+        let parts = path.components(separatedBy: "/").filter { !$0.isEmpty }
+        let queryIDs: [GUID]? = (query["ids"] as? String)?.components(separatedBy: ",")
+
+        guard [2, 4, 5].contains(parts.count) else {
+            return nil
+        }
+
+        if parts.count == 2 {
+            return SyncDeleteRequestSpec(collection: nil, id: nil, ids: queryIDs, wholeCollection: true)
+        }
+
+        if parts[2] != "storage" {
+            return nil
+        }
+
+        if parts.count == 4 {
+            let hasIDs = queryIDs != nil
+            return SyncDeleteRequestSpec(collection: parts[3], id: nil, ids: queryIDs, wholeCollection: !hasIDs)
+        }
+
+        return SyncDeleteRequestSpec(collection: parts[3], id: parts[4], ids: queryIDs, wholeCollection: false)
+    }
+}
+
+private struct SyncPutRequestSpec {
+    let collection: String
+    let id: String
+
+    static func fromRequest(request: GCDWebServerRequest) -> SyncPutRequestSpec? {
+        // Input is "/1.5/user/storage/collection/id}}}".
+        // That means we get six path components here, the first being empty.
+
+        let parts = request.path!.components(separatedBy: "/").filter { !$0.isEmpty }
+
+        guard parts.count == 5 else {
+            return nil
+        }
+
+        if parts[2] != "storage" {
+            return nil
+        }
+
+        return SyncPutRequestSpec(collection: parts[3], id: parts[4])
+    }
+}
+
+class MockSyncServer {
+    let server = GCDWebServer()
+    let username: String
+
+    var offsets: Int = 0
+    var continuations: [String: [EnvelopeJSON]] = [:]
+    var collections: [String: (modified: Timestamp, records: [String: EnvelopeJSON])] = [:]
+    var baseURL: String!
+
+    init(username: String) {
+        self.username = username
+    }
+
+    class func makeValidEnvelope(guid: GUID, modified: Timestamp) -> EnvelopeJSON {
+        let clientBody: [String: Any] = [
+            "id": guid,
+            "name": "Foobar",
+            "commands": [],
+            "type": "mobile",
+        ]
+        let clientBodyString = JSON(object: clientBody).stringValue()!
+        let clientRecord: [String : Any] = [
+            "id": guid,
+            "collection": "clients",
+            "payload": clientBodyString,
+            "modified": Double(modified) / 1000,
+        ]
+        return EnvelopeJSON(JSON(object: clientRecord).stringValue()!)
+    }
+
+    class func withHeaders(response: GCDWebServerResponse, lastModified: Timestamp? = nil, records: Int? = nil, timestamp: Timestamp? = nil) -> GCDWebServerResponse {
+        let timestamp = timestamp ?? Date.now()
+        let xWeaveTimestamp = millisecondsToDecimalSeconds(timestamp)
+        response.setValue("\(xWeaveTimestamp)", forAdditionalHeader: "X-Weave-Timestamp")
+
+        if let lastModified = lastModified {
+            let xLastModified = millisecondsToDecimalSeconds(lastModified)
+            response.setValue("\(xLastModified)", forAdditionalHeader: "X-Last-Modified")
+        }
+
+        if let records = records {
+            response.setValue("\(records)", forAdditionalHeader: "X-Weave-Records")
+        }
+
+        return response
+    }
+
+    func storeRecords(records: [EnvelopeJSON], inCollection collection: String, now: Timestamp? = nil) {
+        let now = now ?? Date.now()
+        let coll = self.collections[collection]
+        var out = coll?.records ?? [:]
+        records.forEach {
+            out[$0.id] = $0.withModified(now)
+        }
+        let newModified = max(now, coll?.modified ?? 0)
+        self.collections[collection] = (modified: newModified, records: out)
+    }
+
+    private func splitArray<T>(items: [T], at: Int) -> ([T], [T]) {
+        return (Array(items.dropLast(items.count - at)), Array(items.dropFirst(at)))
+    }
+
+    private func recordsMatchingSpec(spec: SyncRequestSpec) -> (records: [EnvelopeJSON], offsetID: String?)? {
+        // If we have a provided offset, handle that directly.
+        if let offset = spec.offset {
+            log.debug("Got provided offset \(offset).")
+            guard let remainder = self.continuations[offset] else {
+                log.error("Unknown offset.")
+                return nil
+            }
+
+            // Remove the old one.
+            self.continuations.removeValue(forKey: offset)
+
+            // Handle the smaller-than-limit or no-provided-limit cases.
+            guard let limit = spec.limit, limit < remainder.count else {
+                log.debug("Returning all remaining items.")
+                return (remainder, nil)
+            }
+
+            // Record the next continuation and return the first slice of records.
+            let next = "\(self.offsets)"
+            self.offsets += 1
+            let (returned, remaining) = splitArray(items: remainder, at: limit)
+            self.continuations[next] = remaining
+            log.debug("Returning \(limit) items; next continuation is \(next).")
+            return (returned, next)
+        }
+
+        guard let records = self.collections[spec.collection]?.records.values else {
+            // No matching records.
+            return ([], nil)
+        }
+
+        var items = Array(records)
+        log.debug("Got \(items.count) candidate records.")
+
+        if spec.newer ?? 0 > 0 {
+            items = items.filter { $0.modified > spec.newer! }
+        }
+
+        if let ids = spec.ids {
+            let ids = Set(ids)
+            items = items.filter { ids.contains($0.id) }
+        }
+
+        if let sort = spec.sort {
+            switch sort {
+            case SortOption.NewestFirst:
+                items = items.sorted { $0.modified > $1.modified }
+                log.debug("Sorted items newest first: \(items.map { $0.modified })")
+            case SortOption.OldestFirst:
+                items = items.sorted { $0.modified < $1.modified }
+                log.debug("Sorted items oldest first: \(items.map { $0.modified })")
+            case SortOption.Index:
+                log.warning("Index sorting not yet supported.")
+            }
+        }
+
+        if let limit = spec.limit, items.count > limit {
+            let next = "\(self.offsets)"
+            self.offsets += 1
+            let (returned, remaining) = splitArray(items: items, at: limit)
+            self.continuations[next] = remaining
+            return (returned, next)
+        }
+
+        return (items, nil)
+    }
+
+    private func recordResponse(record: EnvelopeJSON) -> GCDWebServerResponse {
+        let body = record.asJSON().stringValue()!
+        let bodyData = body.utf8EncodedData
+        let response = GCDWebServerDataResponse(data: bodyData, contentType: "application/json")
+        return MockSyncServer.withHeaders(response: response!, lastModified: record.modified)
+    }
+
+    private func modifiedResponse(timestamp: Timestamp) -> GCDWebServerResponse {
+        let body = JSON(object: ["modified": timestamp]).stringValue()
+        let bodyData = body?.utf8EncodedData
+        let response = GCDWebServerDataResponse(data: bodyData, contentType: "application/json")!
+        return MockSyncServer.withHeaders(response: response)
+    }
+
+    func modifiedTimeForCollection(collection: String) -> Timestamp? {
+        return self.collections[collection]?.modified
+    }
+
+    func removeAllItemsFromCollection(collection: String, atTime: Timestamp) {
+        if self.collections[collection] != nil {
+            self.collections[collection] = (atTime, [:])
+        }
+    }
+
+    func start() {
+        let basePath = "/1.5/\(self.username)"
+        let storagePath = "\(basePath)/storage/"
+
+        let infoCollectionsPath = "\(basePath)/info/collections"
+        server?.addHandler(forMethod: "GET", path: infoCollectionsPath, request: GCDWebServerRequest.self) { (request) -> GCDWebServerResponse! in
+            var ic = [String: Any]()
+            var lastModified: Timestamp = 0
+            for collection in self.collections.keys {
+                if let timestamp = self.modifiedTimeForCollection(collection: collection) {
+                    ic[collection] = Double(timestamp) / 1000
+                    lastModified = max(lastModified, timestamp)
+                }
+
+            }
+            let body = JSON(object: ic).stringValue()
+            let bodyData = body?.utf8EncodedData
+
+            let response = GCDWebServerDataResponse(data: bodyData, contentType: "application/json")!
+            return MockSyncServer.withHeaders(response: response, lastModified: lastModified, records: ic.count)
+        }
+
+        let matchPut: GCDWebServerMatchBlock = { method, url, headers, path, query -> GCDWebServerRequest! in
+            guard method == "PUT",
+                path?.startsWith(basePath) ?? false else {
+                return nil
+            }
+            return GCDWebServerDataRequest(method: method, url: url, headers: headers, path: path, query: query)
+        }
+
+        server?.addHandler(match: matchPut) { (request) -> GCDWebServerResponse! in
+            guard let request = request as? GCDWebServerDataRequest else {
+                return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 400))
+            }
+
+            guard let spec = SyncPutRequestSpec.fromRequest(request: request) else {
+                return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 400))
+            }
+            var body = JSON(object: request.jsonObject)
+            body["modified"] = JSON(stringLiteral: millisecondsToDecimalSeconds(Date.now()))
+            let record = EnvelopeJSON(body)
+
+            self.storeRecords(records: [record], inCollection: spec.collection)
+            let timestamp = self.modifiedTimeForCollection(collection: spec.collection)!
+
+            let response = GCDWebServerDataResponse(data: millisecondsToDecimalSeconds(timestamp).utf8EncodedData, contentType: "application/json")
+            return MockSyncServer.withHeaders(response: response!)
+        }
+
+        let matchDelete: GCDWebServerMatchBlock = { method, url, headers, path, query -> GCDWebServerRequest! in
+            guard method == "DELETE" && (path?.startsWith(basePath))! else {
+                return nil
+            }
+            return GCDWebServerRequest(method: method, url: url, headers: headers, path: path, query: query)
+        }
+
+        server?.addHandler(match: matchDelete) { (request) -> GCDWebServerResponse! in
+            guard let spec = SyncDeleteRequestSpec.fromRequest(request: request!) else {
+                return GCDWebServerDataResponse(statusCode: 400)
+            }
+
+            if let collection = spec.collection, let id = spec.id {
+                guard var items = self.collections[collection]?.records else {
+                    // Unable to find the requested collection.
+                    return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 404))
+                }
+
+                guard let item = items[id] else {
+                    // Unable to find the requested id.
+                    return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 404))
+                }
+                items.removeValue(forKey: id)
+                return self.modifiedResponse(timestamp: item.modified)
+            }
+
+            if let collection = spec.collection {
+                if spec.wholeCollection {
+                    self.collections.removeValue(forKey: collection)
+                } else {
+                    if let ids = spec.ids,
+                       var map = self.collections[collection]?.records {
+                            for id in ids {
+                                map.removeValue(forKey: id)
+                            }
+                            self.collections[collection] = (Date.now(), records: map)
+                    }
+                }
+                return self.modifiedResponse(timestamp: Date.now())
+            }
+
+            self.collections = [:]
+            return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(data: "{}".utf8EncodedData, contentType: "application/json"))
+        }
+
+        let match: GCDWebServerMatchBlock = { method, url, headers, path, query -> GCDWebServerRequest! in
+            guard method == "GET", path?.startsWith(storagePath) ?? false else {
+                return nil
+            }
+            return GCDWebServerRequest(method: method, url: url, headers: headers, path: path, query: query)
+        }
+
+        server?.addHandler(match: match) { (request) -> GCDWebServerResponse! in
+            // 1. Decide what the URL is asking for. It might be a collection fetch or
+            //    an individual record, and it might have query parameters.
+
+            guard let spec = SyncRequestSpec.fromRequest(request: request!) else {
+                return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 400))
+            }
+
+            // 2. Grab the matching set of records. Prune based on TTL, exclude with X-I-U-S, etc.
+            if let id = spec.id {
+                guard let collection = self.collections[spec.collection], let record = collection.records[id] else {
+                    // Unable to find the requested collection/id.
+                    return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 404))
+                }
+
+                return self.recordResponse(record: record)
+            }
+
+            guard let (items, offset) = self.recordsMatchingSpec(spec: spec) else {
+                // Unable to find the provided offset.
+                return MockSyncServer.withHeaders(response: GCDWebServerDataResponse(statusCode: 400))
+            }
+
+            // TODO: TTL
+            // TODO: X-I-U-S
+
+            let body = JSON(object: items.map { $0.asJSON() }).stringValue()
+            let bodyData = body?.utf8EncodedData
+            let response = GCDWebServerDataResponse(data: bodyData, contentType: "application/json")
+
+            // 3. Compute the correct set of headers: timestamps, X-Weave-Records, etc.
+            if let offset = offset {
+                response?.setValue(offset, forAdditionalHeader: "X-Weave-Next-Offset")
+            }
+
+            let timestamp = self.modifiedTimeForCollection(collection: spec.collection)!
+            log.debug("Returning GET response with X-Last-Modified for \(items.count) records: \(timestamp).")
+            return MockSyncServer.withHeaders(response: response!, lastModified: timestamp, records: items.count)
+        }
+
+        if server?.start(withPort: 0, bonjourName: nil) == false {
+            XCTFail("Can't start the GCDWebServer.")
+        }
+
+        baseURL = "http://localhost:\(server!.port)\(basePath)"
+    }
+}

mobile/ios/SyncTests/MockSyncServerTests.swift

@@ -0,0 +1,194 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Shared
+import Storage
+@testable import Sync
+import UIKit
+
+import XCTest
+
+class MockSyncServerTests: XCTestCase {
+    var server: MockSyncServer!
+    var client: Sync15StorageClient!
+
+    override func setUp() {
+        server = MockSyncServer(username: "1234567")
+        server.start()
+        client = getClient(server: server)
+    }
+
+    private func getClient(server: MockSyncServer) -> Sync15StorageClient? {
+        guard let url = server.baseURL.asURL else {
+            XCTFail("Couldn't get URL.")
+            return nil
+        }
+
+        let authorizer: Authorizer = identity
+        let queue = DispatchQueue.global(qos: DispatchQoS.background.qosClass)
+        return Sync15StorageClient(serverURI: url, authorizer: authorizer, workQueue: queue, resultQueue: queue, backoff: MockBackoffStorage())
+    }
+
+    func testDeleteSpec() {
+        // Deletion of a collection path itself, versus trailing slash, sets the right flags.
+        let all = SyncDeleteRequestSpec.fromPath(path: "/1.5/123456/storage/bookmarks", withQuery: [:])!
+        XCTAssertTrue(all.wholeCollection)
+        XCTAssertNil(all.ids)
+
+        let some = SyncDeleteRequestSpec.fromPath(path: "/1.5/123456/storage/bookmarks", withQuery: ["ids": "123456,abcdef" as AnyObject])!
+        XCTAssertFalse(some.wholeCollection)
+        XCTAssertEqual(["123456", "abcdef"], some.ids!)
+
+        let one = SyncDeleteRequestSpec.fromPath(path: "/1.5/123456/storage/bookmarks/123456", withQuery: [:])!
+        XCTAssertFalse(one.wholeCollection)
+        XCTAssertNil(one.ids)
+    }
+
+    func testInfoCollections() {
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: Bytes.generateGUID(), modified: 0)], inCollection: "bookmarks", now: 1326251111000)
+        server.storeRecords(records: [], inCollection: "tabs", now: 1326252222500)
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: Bytes.generateGUID(), modified: 0)], inCollection: "bookmarks", now: 1326252222000)
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: Bytes.generateGUID(), modified: 0)], inCollection: "clients", now: 1326253333000)
+
+        let expectation = self.expectation(description: "Waiting for result.")
+        let before = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+        client.getInfoCollections().upon { result in
+            XCTAssertNotNil(result.successValue)
+            guard let response = result.successValue else {
+                expectation.fulfill()
+                return
+            }
+            let after = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+
+            // JSON contents.
+            XCTAssertEqual(response.value.collectionNames().sorted(), ["bookmarks", "clients", "tabs"])
+            XCTAssertEqual(response.value.modified("bookmarks"), 1326252222000)
+            XCTAssertEqual(response.value.modified("clients"), 1326253333000)
+
+            // X-Weave-Timestamp.
+            XCTAssertLessThanOrEqual(before, response.metadata.timestampMilliseconds)
+            XCTAssertLessThanOrEqual(response.metadata.timestampMilliseconds, after)
+            // X-Weave-Records.
+            XCTAssertEqual(response.metadata.records, 3) // bookmarks, clients, tabs.
+
+            // X-Last-Modified, max of all collection modified timestamps.
+            XCTAssertEqual(response.metadata.lastModifiedMilliseconds, 1326253333000)
+
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testGet() {
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: "guid", modified: 0)], inCollection: "bookmarks", now: 1326251111000)
+        let collectionClient = client.clientForCollection("bookmarks", encrypter: getEncrypter())
+
+        let expectation = self.expectation(description: "Waiting for result.")
+        let before = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+        collectionClient.get("guid").upon { result in
+            XCTAssertNotNil(result.successValue)
+            guard let response = result.successValue else {
+                expectation.fulfill()
+                return
+            }
+            let after = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+
+            // JSON contents.
+            XCTAssertEqual(response.value.id, "guid")
+            XCTAssertEqual(response.value.modified, 1326251111000)
+
+            // X-Weave-Timestamp.
+            XCTAssertLessThanOrEqual(before, response.metadata.timestampMilliseconds)
+            XCTAssertLessThanOrEqual(response.metadata.timestampMilliseconds, after)
+            // X-Weave-Records.
+            XCTAssertNil(response.metadata.records)
+            // X-Last-Modified.
+            XCTAssertEqual(response.metadata.lastModifiedMilliseconds, 1326251111000)
+
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+
+        // And now a missing record, which should produce a 404.
+        collectionClient.get("missing").upon { result in
+            XCTAssertNotNil(result.failureValue)
+            guard let response = result.failureValue else {
+                expectation.fulfill()
+                return
+            }
+            XCTAssertNotNil(response as? NotFound<HTTPURLResponse>)
+        }
+    }
+
+    func testWipeStorage() {
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: "a", modified: 0)], inCollection: "bookmarks", now: 1326251111000)
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: "b", modified: 0)], inCollection: "bookmarks", now: 1326252222000)
+        server.storeRecords(records: [MockSyncServer.makeValidEnvelope(guid: "c", modified: 0)], inCollection: "clients", now: 1326253333000)
+        server.storeRecords(records: [], inCollection: "tabs")
+
+        // For now, only testing wiping the storage root, which is the only thing we use in practice.
+        let expectation = self.expectation(description: "Waiting for result.")
+        let before = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+        client.wipeStorage().upon { result in
+            XCTAssertNotNil(result.successValue)
+            guard let response = result.successValue else {
+                expectation.fulfill()
+                return
+            }
+            let after = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+
+            // JSON contents: should be the empty object.
+            let jsonData = try! response.value.rawData()
+            let jsonString = String(data: jsonData, encoding: String.Encoding.utf8)!
+            XCTAssertEqual(jsonString, "{}")
+
+            // X-Weave-Timestamp.
+            XCTAssertLessThanOrEqual(before, response.metadata.timestampMilliseconds)
+            XCTAssertLessThanOrEqual(response.metadata.timestampMilliseconds, after)
+            // X-Weave-Records.
+            XCTAssertNil(response.metadata.records)
+            // X-Last-Modified.
+            XCTAssertNil(response.metadata.lastModifiedMilliseconds)
+
+            // And we really wiped the data.
+            XCTAssertTrue(self.server.collections.isEmpty)
+
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testPut() {
+        // For now, only test uploading crypto/keys.  There's nothing special about this PUT, however.
+        let expectation = self.expectation(description: "Waiting for result.")
+        let before = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+        client.uploadCryptoKeys(Keys.random(), withSyncKeyBundle: KeyBundle.random(), ifUnmodifiedSince: nil).upon { result in
+            XCTAssertNotNil(result.successValue)
+            guard let response = result.successValue else {
+                expectation.fulfill()
+                return
+            }
+            let after = decimalSecondsStringToTimestamp(millisecondsToDecimalSeconds(Date.now()))!
+
+            // Contents: should be just the record timestamp.
+            XCTAssertLessThanOrEqual(before, response.value)
+            XCTAssertLessThanOrEqual(response.value, after)
+
+            // X-Weave-Timestamp.
+            XCTAssertLessThanOrEqual(before, response.metadata.timestampMilliseconds)
+            XCTAssertLessThanOrEqual(response.metadata.timestampMilliseconds, after)
+            // X-Weave-Records.
+            XCTAssertNil(response.metadata.records)
+            // X-Last-Modified.
+            XCTAssertNil(response.metadata.lastModifiedMilliseconds)
+
+            // And we really uploaded the record.
+            XCTAssertNotNil(self.server.collections["crypto"])
+            XCTAssertNotNil(self.server.collections["crypto"]?.records["keys"])
+
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+}

mobile/ios/SyncTests/RecordTests.swift

@@ -0,0 +1,572 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Shared
+import Storage
+@testable import Sync
+import UIKit
+
+import XCTest
+import SwiftyJSON
+
+class RecordTests: XCTestCase {
+    func testGUIDs() {
+        let s = Bytes.generateGUID()
+        print("Got GUID: \(s)", terminator: "\n")
+        XCTAssertEqual(12, s.lengthOfBytes(using: String.Encoding.utf8))
+    }
+
+    func testSwiftyJSONSerializingControlChars() {
+        let input = "{\"foo\":\"help \\u000b this\"}"
+        let json = JSON(parseJSON: input)
+        XCTAssertNil(json.error)
+        XCTAssertNil(json.null)
+        XCTAssertEqual(input, json.stringValue())
+
+        let pairs: [String: Any] = ["foo": "help \(Character(UnicodeScalar(11))) this"]
+        let built = JSON(object: pairs)
+        XCTAssertEqual(input, built.stringValue())
+    }
+
+    func testEnvelopeNullTTL() {
+        let p = CleartextPayloadJSON(JSON(object: ["id": "guid"]))
+        let r = Record<CleartextPayloadJSON>(id: "guid", payload: p, modified: Date.now(), sortindex: 15, ttl: nil)
+        let k = KeyBundle.random()
+        let s = k.serializer({ $0.json })
+        let json = s(r)!
+        XCTAssertEqual(json["id"].stringValue, "guid")
+        XCTAssertTrue(json["ttl"].isNull())
+    }
+
+    func testParsedNulls() {
+        // Make this a thorough test: use a real-ish blob of JSON.
+        // Look to see whether fields with explicit null values match isNull().
+        let fullRecord = "{\"id\":\"global\"," +
+            "\"payload\":" +
+            "\"{\\\"syncID\\\":\\\"zPSQTm7WBVWB\\\"," +
+            "\\\"declined\\\":[\\\"bookmarks\\\"]," +
+            "\\\"storageVersion\\\":5," +
+            "\\\"engines\\\":{" +
+            "\\\"clients\\\":{\\\"version\\\":1,\\\"syncID\\\": null}," +
+            "\\\"tabs\\\":null}}\"," +
+            "\"username\":\"5817483\"," +
+        "\"modified\":1.32046073744E9}"
+
+        let record = EnvelopeJSON(fullRecord)
+        let bodyJSON = JSON(parseJSON: record.payload)
+
+        XCTAssertTrue(bodyJSON["engines"]["tabs"].isNull())
+
+        let clients = bodyJSON["engines"]["clients"]
+
+        // Make sure we're really getting a value out.
+        XCTAssertEqual(clients["version"].int, 1)
+
+        // An explicit null in the input has .type == null, so our .isNull works.
+        XCTAssertTrue(clients["syncID"].isNull())
+
+        // Oh, and it's a valid meta/global.
+        let global = MetaGlobal.fromJSON(bodyJSON)
+        XCTAssertTrue(global != nil)
+    }
+
+    func testEnvelopeJSON() {
+        let e = EnvelopeJSON(JSON(parseJSON: "{}"))
+        XCTAssertFalse(e.isValid())
+        
+        let ee = EnvelopeJSON("{\"id\": \"foo\"}")
+        XCTAssertFalse(ee.isValid())
+        XCTAssertEqual(ee.id, "foo")
+        
+        let eee = EnvelopeJSON(JSON(parseJSON: "{\"id\": \"foo\", \"collection\": \"bar\", \"payload\": \"baz\"}"))
+        XCTAssertTrue(eee.isValid())
+        XCTAssertEqual(eee.id, "foo")
+        XCTAssertEqual(eee.collection, "bar")
+        XCTAssertEqual(eee.payload, "baz")
+    }
+
+    func testRecord() {
+        // This is malformed JSON (no closing brace).
+        let malformedPayload = "{\"id\": \"abcdefghijkl\", \"collection\": \"clients\", \"payload\": \"in"
+
+        // Invalid: the payload isn't stringified JSON.
+        let invalidPayload = "{\"id\": \"abcdefghijkl\", \"collection\": \"clients\", \"payload\": \"invalid\"}"
+
+        // Invalid: the payload is missing a GUID.
+        let emptyPayload = "{\"id\": \"abcdefghijkl\", \"collection\": \"clients\", \"payload\": \"{}\"}"
+
+        // This one is invalid because the payload "id" isn't a string.
+        // (It'll also fail implicitly because the guid doesn't match the envelope.)
+        let badPayloadGUIDPayload: [String: Any] = ["id": 0]
+        let badPayloadGUIDPayloadString = JSON(object: badPayloadGUIDPayload).stringValue()!
+        let badPayloadGUIDRecord: [String: Any] = ["id": "abcdefghijkl",
+                                                   "collection": "clients",
+                                                   "payload": badPayloadGUIDPayloadString]
+        let badPayloadGUIDRecordString = JSON(object: badPayloadGUIDRecord).stringValue()!
+
+        // This one is invalid because the payload doesn't contain an "id" at all, but it's non-empty.
+        // See also `emptyPayload` above.
+        // (It'll also fail implicitly because the guid doesn't match the envelope.)
+        let noPayloadGUIDPayload: [String: Any] = ["some": "thing"]
+        let noPayloadGUIDPayloadString = JSON(object: noPayloadGUIDPayload).stringValue()!
+        let noPayloadGUIDRecord: [String: Any] = ["id": "abcdefghijkl",
+                                                  "collection": "clients",
+                                                  "payload": noPayloadGUIDPayloadString]
+        let noPayloadGUIDRecordString = JSON(object: noPayloadGUIDRecord).stringValue()!
+
+        // And this is a valid record.
+        let clientBody: [String: Any] = ["id": "abcdefghijkl", "name": "Foobar", "commands": [], "type": "mobile"]
+        let clientBodyString = JSON(object: clientBody).stringValue()!
+        let clientRecord: [String: Any] = ["id": "abcdefghijkl", "collection": "clients", "payload": clientBodyString]
+        let clientPayload = JSON(object: clientRecord).stringValue()!
+
+        let cleartextClientsFactory: (String) -> ClientPayload? = {
+            (s: String) -> ClientPayload? in
+            return ClientPayload(s)
+        }
+
+        let clearFactory: (String) -> CleartextPayloadJSON? = {
+            (s: String) -> CleartextPayloadJSON? in       
+            return CleartextPayloadJSON(s)
+        }
+
+        print(clientPayload, terminator: "\n")
+
+        // Non-JSON malformed payloads don't even yield a value.
+        XCTAssertNil(Record<CleartextPayloadJSON>.fromEnvelope(EnvelopeJSON(malformedPayload), payloadFactory: clearFactory))
+
+        // Only payloads that parse as JSON objects are valid.
+        XCTAssertNil(Record<CleartextPayloadJSON>.fromEnvelope(EnvelopeJSON(invalidPayload), payloadFactory: clearFactory))
+
+        // Missing ID.
+        XCTAssertNil(Record<CleartextPayloadJSON>.fromEnvelope(EnvelopeJSON(emptyPayload), payloadFactory: clearFactory))
+
+        // No ID in non-empty payload.
+        let noPayloadGUIDEnvelope = EnvelopeJSON(noPayloadGUIDRecordString)
+
+        // The envelope is valid...
+        XCTAssertTrue(noPayloadGUIDEnvelope.isValid())
+
+        // ... but the payload is not.
+        let noID = Record<CleartextPayloadJSON>.fromEnvelope(noPayloadGUIDEnvelope, payloadFactory: cleartextClientsFactory)
+        XCTAssertNil(noID)
+
+        // Non-string ID in payload.
+        let badPayloadGUIDEnvelope = EnvelopeJSON(badPayloadGUIDRecordString)
+
+        // The envelope is valid...
+        XCTAssertTrue(badPayloadGUIDEnvelope.isValid())
+
+        // ... but the payload is not.
+        let badID = Record<CleartextPayloadJSON>.fromEnvelope(badPayloadGUIDEnvelope, payloadFactory: cleartextClientsFactory)
+        XCTAssertNil(badID)
+
+        // Only valid ClientPayloads are valid.
+        XCTAssertNil(Record<ClientPayload>.fromEnvelope(EnvelopeJSON(invalidPayload), payloadFactory: cleartextClientsFactory))
+        XCTAssertTrue(Record<ClientPayload>.fromEnvelope(EnvelopeJSON(clientPayload), payloadFactory: cleartextClientsFactory)!.payload.isValid())
+    }
+
+    func testEncryptedClientRecord() {
+        let b64E = "0A7mU5SZ/tu7ZqwXW1og4qHVHN+zgEi4Xwfwjw+vEJw="
+        let b64H = "11GN34O9QWXkjR06g8t0gWE1sGgQeWL0qxxWwl8Dmxs="
+        
+        let expectedGUID = "0-P9fabp9vJD"
+        let expectedSortIndex = 131
+        let expectedLastModified: Timestamp = 1326254123650
+
+        let inputString = "{\"sortindex\": 131, \"payload\": \"{\\\"ciphertext\\\":\\\"YJB4dr0vZEIWPirfU2FCJvfzeSLiOP5QWasol2R6ILUxdHsJWuUuvTZVhxYQfTVNou6hVV67jfAvi5Cs+bqhhQsv7icZTiZhPTiTdVGt+uuMotxauVA5OryNGVEZgCCTvT3upzhDFdDbJzVd9O3/gU/b7r/CmAHykX8bTlthlbWeZ8oz6gwHJB5tPRU15nM/m/qW1vyKIw5pw/ZwtAy630AieRehGIGDk+33PWqsfyuT4EUFY9/Ly+8JlnqzxfiBCunIfuXGdLuqTjJOxgrK8mI4wccRFEdFEnmHvh5x7fjl1ID52qumFNQl8zkB75C8XK25alXqwvRR6/AQSP+BgQ==\\\",\\\"IV\\\":\\\"v/0BFgicqYQsd70T39rraA==\\\",\\\"hmac\\\":\\\"59605ed696f6e0e6e062a03510cff742bf6b50d695c042e8372a93f4c2d37dac\\\"}\", \"id\": \"0-P9fabp9vJD\", \"modified\": 1326254123.65}"
+
+        let keyBundle = KeyBundle(encKeyB64: b64E, hmacKeyB64: b64H)!
+        let decryptClient = keyBundle.factory({ CleartextPayloadJSON($0) })
+        let encryptClient = keyBundle.serializer({ $0.json })   // It's already a JSON.
+
+        let toRecord = {
+            return Record<CleartextPayloadJSON>.fromEnvelope($0, payloadFactory: decryptClient)
+        }
+
+        let envelope = EnvelopeJSON(inputString)
+        if let r = toRecord(envelope) {
+            XCTAssertEqual(r.id, expectedGUID)
+            XCTAssertTrue(r.modified == expectedLastModified) //1326254123650
+            XCTAssertEqual(r.sortindex, expectedSortIndex)
+
+            if let ee = encryptClient(r) {
+                let envelopePrime = EnvelopeJSON(ee)
+                XCTAssertEqual(envelopePrime.id, expectedGUID)
+                XCTAssertEqual(envelopePrime.id, envelope.id)
+                XCTAssertEqual(envelopePrime.sortindex, envelope.sortindex)
+                XCTAssertTrue(envelopePrime.modified == 0)
+
+                if let rPrime = toRecord(envelopePrime) {
+                    // The payloads should be identical.
+                    XCTAssertTrue(rPrime.equalPayloads(r))
+                } else {
+                    XCTFail("No record.")
+                }
+            } else {
+                XCTFail("No record.")
+            }
+        } else {
+            XCTFail("No record.")
+        }
+
+        // Test invalid Base64.
+        let badInputString =  "{\"sortindex\": 131, \"payload\": \"{\\\"ciphertext\\\":\\\"~~~YJB4dr0vZEIWPirfU2FCJvfzeSLiOP5QWasol2R6ILUxdHsJWuUuvTZVhxYQfTVNou6hVV67jfAvi5Cs+bqhhQsv7icZTiZhPTiTdVGt+uuMotxauVA5OryNGVEZgCCTvT3upzhDFdDbJzVd9O3/gU/b7r/CmAHykX8bTlthlbWeZ8oz6gwHJB5tPRU15nM/m/qW1vyKIw5pw/ZwtAy630AieRehGIGDk+33PWqsfyuT4EUFY9/Ly+8JlnqzxfiBCunIfuXGdLuqTjJOxgrK8mI4wccRFEdFEnmHvh5x7fjl1ID52qumFNQl8zkB75C8XK25alXqwvRR6/AQSP+BgQ==\\\",\\\"IV\\\":\\\"v/0BFgicqYQsd70T39rraA==\\\",\\\"hmac\\\":\\\"59605ed696f6e0e6e062a03510cff742bf6b50d695c042e8372a93f4c2d37dac\\\"}\", \"id\": \"0-P9fabp9vJD\", \"modified\": 1326254123.65}"
+        let badEnvelope = EnvelopeJSON(badInputString)
+        XCTAssertTrue(badEnvelope.isValid())      // It's a valid envelope containing nonsense ciphertext.
+        XCTAssertNil(toRecord(badEnvelope))       // Even though the envelope is valid, the payload is invalid, so we can't construct a record.
+    }
+
+    func testMeta() {
+        let fullRecord = "{\"id\":\"global\"," +
+            "\"payload\":" +
+            "\"{\\\"syncID\\\":\\\"zPSQTm7WBVWB\\\"," +
+            "\\\"declined\\\":[\\\"bookmarks\\\"]," +
+            "\\\"storageVersion\\\":5," +
+            "\\\"engines\\\":{" +
+            "\\\"clients\\\":{\\\"version\\\":1,\\\"syncID\\\":\\\"fDg0MS5bDtV7\\\"}," +
+            "\\\"forms\\\":{\\\"version\\\":1,\\\"syncID\\\":\\\"GXF29AFprnvc\\\"}," +
+            "\\\"history\\\":{\\\"version\\\":1,\\\"syncID\\\":\\\"av75g4vm-_rp\\\"}," +
+            "\\\"passwords\\\":{\\\"version\\\":1,\\\"syncID\\\":\\\"LT_ACGpuKZ6a\\\"}," +
+            "\\\"prefs\\\":{\\\"version\\\":2,\\\"syncID\\\":\\\"-3nsksP9wSAs\\\"}," +
+            "\\\"tabs\\\":{\\\"version\\\":1,\\\"syncID\\\":\\\"W4H5lOMChkYA\\\"}}}\"," +
+            "\"username\":\"5817483\"," +
+            "\"modified\":1.32046073744E9}"
+
+        let record = EnvelopeJSON(fullRecord)
+        XCTAssertTrue(record.isValid())
+
+        let global = MetaGlobal.fromJSON(JSON(parseJSON: record.payload))
+        XCTAssertTrue(global != nil)
+
+        if let global = global {
+            XCTAssertEqual(["bookmarks"], global.declined)
+            XCTAssertEqual(5, global.storageVersion)
+            let modified = record.modified
+            XCTAssertTrue(1320460737440 == modified)
+            let forms = global.engines["forms"]
+            let syncID = forms!.syncID
+            XCTAssertEqual("GXF29AFprnvc", syncID)
+
+            let payload: JSON = global.asPayload().json
+            XCTAssertEqual("GXF29AFprnvc", payload["engines"]["forms"]["syncID"].stringValue)
+            XCTAssertEqual(1, payload["engines"]["forms"]["version"].intValue)
+            XCTAssertEqual("bookmarks", payload["declined"].arrayValue[0].stringValue)
+        }
+    }
+
+    func testHistoryPayload() {
+        let payloadJSON = "{\"id\":\"--DzSJTCw-zb\",\"histUri\":\"https://bugzilla.mozilla.org/show_bug.cgi?id=1154549\",\"title\":\"1154549 – Encapsulate synced profile data within an account-centric object\",\"visits\":[{\"date\":1429061233163240,\"type\":1}]}"
+        let json = JSON(parseJSON: payloadJSON)
+        if let payload = HistoryPayload.fromJSON(json) {
+            XCTAssertEqual("--DzSJTCw-zb", payload["id"].stringValue)
+            XCTAssertEqual("1154549 – Encapsulate synced profile data within an account-centric object", payload["title"].stringValue)
+            XCTAssertEqual(1, payload.visits[0].type.rawValue)
+            XCTAssertEqual(1429061233163240, payload.visits[0].date)
+
+            let v = payload.visits[0]
+            let j = v.toJSON()
+            XCTAssertEqual(1, j["type"] as! Int)
+            XCTAssertEqual(1429061233163240, j["date"] as! Int64)
+        } else {
+            XCTFail("Should have parsed.")
+        }
+    }
+
+    func testHistoryPayloadWithNoURL() {
+        let payloadJSON = "{\"id\":\"--DzSJTCw-zb\",\"histUri\":null,\"visits\":[{\"date\":1429061233163240,\"type\":1}]}"
+        let json = JSON(parseJSON: payloadJSON)
+        XCTAssertNil(HistoryPayload.fromJSON(json))
+    }
+
+    func testHistoryPayloadWithNoTitle() {
+        let payloadJSON = "{\"id\":\"--DzSJTCw-zb\",\"histUri\":\"https://foo.com/\",\"visits\":[{\"date\":1429061233163240,\"type\":1}]}"
+        let json = JSON(parseJSON: payloadJSON)
+        if let payload = HistoryPayload.fromJSON(json) {
+            // Missing fields are null-valued in SwiftyJSON.
+            XCTAssertTrue(payload["title"].isNull())
+            XCTAssertEqual("", payload.title)
+        } else {
+            XCTFail("Should have parsed.")
+        }
+    }
+
+    func testHistoryPayloadWithNullTitle() {
+        let payloadJSON = "{\"id\":\"--DzSJTCw-zb\",\"histUri\":\"https://foo.com/\",\"title\":null,\"visits\":[{\"date\":1429061233163240,\"type\":1}]}"
+        let json = JSON(parseJSON: payloadJSON)
+        if let payload = HistoryPayload.fromJSON(json) {
+            XCTAssertEqual("", payload.title)
+        } else {
+            XCTFail("Should have parsed.")
+        }
+    }
+
+    func testLoginPayload() {
+        let input = JSON([
+            "id": "abcdefabcdef",
+            "hostname": "http://foo.com/",
+            "username": "foo",
+            "password": "bar",
+            "usernameField": "field",
+            "passwordField": "bar",
+            // No formSubmitURL.
+            "httpRealm": "",
+        ])
+
+        // fromJSON returns nil if not valid.
+        XCTAssertNotNil(LoginPayload.fromJSON(input))
+    }
+
+    func testSeparators() {
+        // Mistyped parentid.
+        let invalidSeparator = JSON(["type": "separator", "arentid": "toolbar", "parentName": "Bookmarks Toolbar", "pos": 3])
+        let sep = BookmarkType.payloadFromJSON(invalidSeparator)
+        XCTAssertTrue(sep is SeparatorPayload)
+        XCTAssertFalse(sep!.isValid())
+
+        // This one's right.
+        let validSeparator = JSON(["id": "abcabcabcabc", "type": "separator", "parentid": "toolbar", "parentName": "Bookmarks Toolbar", "pos": 3])
+        let separator = BookmarkType.payloadFromJSON(validSeparator)!
+        XCTAssertTrue(separator is SeparatorPayload)
+        XCTAssertTrue(separator.isValid())
+        XCTAssertEqual(3, separator["pos"].intValue)
+    }
+
+    func testFolders() {
+        let validFolder = JSON([
+            "id": "abcabcabcabc",
+            "type": "folder",
+            "parentid": "toolbar",
+            "parentName": "Bookmarks Toolbar",
+            "title": "Sóme stüff",
+            "description": "",
+            "children": ["foo", "bar"],
+        ])
+        let folder = BookmarkType.payloadFromJSON(validFolder)!
+        XCTAssertTrue(folder is FolderPayload)
+        XCTAssertTrue(folder.isValid())
+        XCTAssertEqual((folder as! FolderPayload).children, ["foo", "bar"])
+    }
+
+// swiftlint:disable line_length
+    func testMobileBookmarksFolder() {
+        let children = ["M87np9Vfh_2s", "-JxRyqNte-ue", "6lIQzUtbjE8O", "eOg3jPSslzXl", "1WJIi9EjQErp", "z5uRo45Rvfbd", "EK3lcNd0sUFN", "gFD3GTljgu12", "eRZGsbN1ew9-", "widfEdgGn9de", "l7eTOR4Uf6xq", "vPbxG-gpN4Rb", "4dwJ8CototFe", "zK-kw9Ii6ScW", "eDmDU-gtEFW6", "lKjqWQaL_syt", "ETVDvWgGT31Q", "3Z_bMIHPSZQ8", "Fqu4_bJOk7fT", "Uo_5K1QrA67j", "gDTXNg4m1AJZ", "zpds8P-9xews", "87zjNtVGPtEp", "ZJru8Sn3qhW7", "txVnzBBBOgLP", "JTnRqFaj_oNa", "soaMlfmM4kjR", "g8AcVBjo6IRf", "uPUDaiG4q637", "rfq2bUud_w4d", "XBGxsiuUG2UD", "-VQRnJlyAvMs", "6wu7TScKdTU7", "ZeFji2hLVpLj", "HpCn_TVizMWX", "IPR5HZwRdlwi", "00JFOGuWnhWB", "P1jb3qKt32Vg", "D6MQJ43V1Ir5", "qWSoXFteRfsq", "o2avfYqEdomL", "xRS0U0YnjK9G", "VgOgzE_xfP4w", "SwP3rMJGvoO3", "Hf2jEgI_-PWa", "AyhmBi7Cv598", "-PaMuzTJXxVk", "JMhYrg8SlY5K", "SQeySEjzyplL", "GTAwd2UkEQEe", "x3RsZj5Ilebr", "sRZWZqPi74FP", "amHR50TpygA6", "XSk782ceVNN6", "ipiMyYQzeypI", "ph2k3Nqfhau4", "m5JKC3hAEQ0H", "yTVerkmQbNxk", "7taA6FbbbUbH", "PZvpbSRuJLPs", "C8atoa25U94F", "KOfNJk_ISLc6", "Bt74lBG9tJq6", "BuHoY2rUhuKA", "XTmoWKnwfIPl", "ZATwa3oTD1m0", "e8TczN5It6Am", "6kCUYs8hQtKg", "jDD8s5aiKoex", "QmpmcrYwLU29", "nCRcekynuJ08", "resttaI4J9tu", "EKSX3HV55VU3", "2-yCz0EIsVls", "sSeeGw3VbBY-", "qfpCrU34w9y0", "RKDgzPWecD6m", "5SgXEKu_dICW", "R143WAeB5E5r", "8Ns4-NiKG62r", "4AHuZDvop5XX", "YCP1OsO1goFF", "CYYaU1mQ_N6t", "UGkzEOMK8cuU", "1RzZOarkzQBa", "qSW2Z3cZSI9c", "ooPlKEAfQsnn", "jIUScoKLiXQt", "bjNTKugzRRL1", "hR24ZVnHUZcs", "3j2IDAZgUyYi", "xnWcy-sQDJRu", "UCcgJqGk3bTV", "WSSRWeptH9tq", "4ugv47OGD2E2", "XboCZgUx-x3x", "HrmWqiqsuLrm", "OjdxvRJ3Jb6j"]
+        let json = JSON([
+            "id": "UjAHxFOGEqU8",
+            "type": "folder",
+            "parentName": "",
+            "title": "mobile",
+            "description": JSON.null,
+            "children": children,
+            "parentid": "places",
+        ])
+
+        let bookmark = BookmarkType.payloadFromJSON(json)
+        XCTAssertTrue(bookmark is FolderPayload)
+        XCTAssertTrue(bookmark?.isValid() ?? false)
+    }
+// swiftlint:enable line_length
+
+    func testLivemarkMissingFields() {
+        let json = JSON([
+            "id": "M5bwUKK8hPyF",
+            "type": "livemark",
+            "siteUri": "http://www.bbc.co.uk/go/rss/int/news/-/news/",
+            "feedUri": "http://fxfeeds.mozilla.com/en-US/firefox/headlines.xml",
+            "parentName": "Bookmarks Toolbar",
+            "parentid": "toolbar",
+            "children": ["3Qr13GucOtEh"]])
+
+        let bookmark = BookmarkType.payloadFromJSON(json)
+        XCTAssertTrue(bookmark is LivemarkPayload)
+
+        let livemark = bookmark as! LivemarkPayload
+        XCTAssertTrue(livemark.isValid())
+        let siteURI = "http://www.bbc.co.uk/go/rss/int/news/-/news/"
+        let feedURI = "http://fxfeeds.mozilla.com/en-US/firefox/headlines.xml"
+        XCTAssertEqual(feedURI, livemark.feedURI)
+        XCTAssertEqual(siteURI, livemark.siteURI)
+
+        let m = (livemark as MirrorItemable).toMirrorItem(Date.now())
+        XCTAssertEqual("http://fxfeeds.mozilla.com/en-US/firefox/headlines.xml", m.feedURI)
+        XCTAssertEqual("http://www.bbc.co.uk/go/rss/int/news/-/news/", m.siteURI)
+    }
+
+    func testDeletedRecord() {
+        let json = JSON([
+            "id": "abcdefghijkl",
+            "deleted": true,
+            "type": "bookmark",
+        ])
+
+        guard let payload = BookmarkType.payloadFromJSON(json) else {
+            XCTFail()
+            return
+        }
+
+        XCTAssertFalse(payload is BookmarkPayload)    // Only BookmarkBasePayload.
+        XCTAssertTrue(payload.isValid())
+    }
+
+    func testUnknownRecordType() {
+        // It'll return a base payload that's invalid because its type is unknown.
+        let json = JSON([
+            "parentid": "mobile",
+            "tags": [],
+            "title": "Dispozitivul meu",
+            "id": "pQSMHiA7fD0Z",
+            "type": "something",
+            "parentName": "mobile",
+        ])
+
+        XCTAssertNil(BookmarkType.payloadFromJSON(json))
+
+        let payload = BookmarkType.somePayloadFromJSON(json)
+        XCTAssertFalse(payload.isValid())        // Not valid because type is unknown.
+    }
+
+    func testInvalidRecordWithType() {
+        // It should still return the right payload type, even if it's not valid.
+        let json = JSON([
+            "parentid": "mobile",
+            "bmkUri": JSON.null,
+            "tags": [],
+            "title": "Dispozitivul meu",
+            "id": "pQSMHiA7fD0Z",
+            "type": "bookmark",
+            "parentName": "mobile",
+        ])
+
+        guard let payload = BookmarkType.payloadFromJSON(json) else {
+            XCTFail()
+            return
+        }
+
+        XCTAssertFalse(payload.isValid())
+        XCTAssertTrue(payload is BookmarkPayload)
+    }
+
+    func testLivemark() {
+        let json = JSON([
+            "id": "M5bwUKK8hPyF",
+            "type": "livemark",
+            "siteUri": "http://www.bbc.co.uk/go/rss/int/news/-/news/",
+            "feedUri": "http://fxfeeds.mozilla.com/en-US/firefox/headlines.xml",
+            "parentName": "Bookmarks Toolbar",
+            "parentid": "toolbar",
+            "title": "Latest Headlines",
+            "description": "",
+            "children":
+            ["7oBdEZB-8BMO", "SUd1wktMNCTB", "eZe4QWzo1BcY", "YNBhGwhVnQsN",
+            "92Aw2SMEkFg0", "uw0uKqrVFwd-", "x7mx2P3--8FJ", "d-jVF8UuC9Ye",
+            "DV1XVtKLEiZ5", "g4mTaTjr837Z", "1Zi5W3lwBw8T", "FEYqlUHtbBWS",
+            "qQd2u7LjosCB", "VUs2djqYfbvn", "KuhYnHocu7eg", "u2gcg9ILRg-3",
+            "hfK_RP-EC7Ol", "Aq5qsa4E5msH", "6pZIbxuJTn-K", "k_fp0iN3yYMR",
+            "59YD3iNOYO8O", "01afpSdAk2iz", "Cq-kjXDEPIoP", "HtNTjt9UwWWg",
+            "IOU8QRSrTR--", "HJ5lSlBx6d1D", "j2dz5R5U6Khc", "5GvEjrNR0yJl",
+            "67ozIBF5pNVP", "r5YB0cUx6C_w", "FtmFDBNxDQ6J", "BTACeZq9eEtw",
+            "ll4ozQ-_VNJe", "HpImsA4_XuW7", "nJvCUQPLSXwA", "94LG-lh6TUYe",
+            "WHn_QoOL94Os", "l-RvjgsZYlej", "LipQ8abcRstN", "74TiLvarE3n_",
+            "8fCiLQpQGK1P", "Z6h4WkbwfQFa", "GgAzhqakoS6g", "qyt92T8vpMsK",
+            "RyOgVCe2EAOE", "bgSEhW3w6kk5", "hWODjHKGD7Ph", "Cky673aqOHbT",
+            "gZCYT7nx3Nwu", "iJzaJxxrM58L", "rUHCRv68aY5L", "6Jc1hNJiVrV9",
+            "lmNgoayZ-ym8", "R1lyXsDzlfOd", "pinrXwDnRk6g", "Sn7TmZV01vMM",
+            "qoXyU6tcS1dd", "TRLanED-QfBK", "xHbhMeX_FYEA", "aPqacdRlAtaW",
+            "E3H04Wn2RfSi", "eaSIMI6kSrcz", "rtkRxFoG5Vqi", "dectkUglV0Dz",
+            "B4vUE0BE15No", "qgQFW5AQrgB0", "SxAXvwOhu8Zi", "0S6cRPOg-5Z2",
+            "zcZZBGeLnaWW", "B0at8hkQqVZQ", "sgPtgGulbP66", "lwtwGHSCPYaQ",
+            "mNTdpgoRZMbW", "-L8Vci6CbkJY", "bVzudKSQERc1", "Gxl9lb4DXsmL",
+            "3Qr13GucOtEh"]])
+
+        let bookmark = BookmarkType.payloadFromJSON(json)
+        XCTAssertTrue(bookmark is LivemarkPayload)
+
+        let livemark = bookmark as! LivemarkPayload
+        XCTAssertTrue(livemark.isValid())
+        let siteURI = "http://www.bbc.co.uk/go/rss/int/news/-/news/"
+        let feedURI = "http://fxfeeds.mozilla.com/en-US/firefox/headlines.xml"
+        XCTAssertEqual(feedURI, livemark.feedURI)
+        XCTAssertEqual(siteURI, livemark.siteURI)
+
+        let m = (livemark as MirrorItemable).toMirrorItem(Date.now())
+        XCTAssertEqual("http://fxfeeds.mozilla.com/en-US/firefox/headlines.xml", m.feedURI)
+        XCTAssertEqual("http://www.bbc.co.uk/go/rss/int/news/-/news/", m.siteURI)
+    }
+
+    func testMobileBookmark() {
+        let json = JSON([
+            "id": "jIUScoKLiXQt",
+            "type": "bookmark",
+            "title": "Join the Engineering Leisure Class — Medium",
+            "parentName": "mobile",
+            "bmkUri": "https://medium.com/@chrisloer/join-the-engineering-leisure-class-b3083c09a78e",
+            "tags": [],
+            "keyword": JSON.null,
+            "description": JSON.null,
+            "loadInSidebar": false,
+            "parentid": "mobile",
+        ])
+
+        let bookmark = BookmarkType.payloadFromJSON(json)
+        XCTAssertTrue(bookmark is BookmarkPayload)
+        XCTAssertTrue(bookmark?.isValid() ?? false)
+    }
+
+    func testQuery() {
+        let str = "{\"title\":\"Downloads\",\"parentName\":\"\",\"bmkUri\":\"place:transition=7&sort=4\",\"id\":\"7gdp9S1okhKf\",\"parentid\":\"rq6WHyfHkoUV\",\"type\":\"query\"}"
+        
+        let query = BookmarkType.payloadFromJSON(JSON(parseJSON: str))
+        XCTAssertTrue(query is BookmarkQueryPayload)
+        let mirror = query?.toMirrorItem(Date.now())
+        let roundtrip = mirror?.asPayload()
+        XCTAssertTrue(roundtrip! is BookmarkQueryPayload)
+    }
+
+    func testBookmarks() {
+        let validBookmark = JSON([
+            "id": "abcabcabcabc",
+            "type": "bookmark",
+            "parentid": "menu",
+            "parentName": "Bookmarks Menu",
+            "title": "Anøther",
+            "bmkUri": "http://terrible.sync/naming",
+            "description": "",
+            "tags": [],
+            "keyword": "",
+            ])
+        let bookmark = BookmarkType.payloadFromJSON(validBookmark)
+        XCTAssertTrue(bookmark is BookmarkPayload)
+
+        let query = JSON(parseJSON: "{\"id\":\"ShCZLGEFQMam\",\"type\":\"query\",\"title\":\"Downloads\",\"parentName\":\"\",\"bmkUri\":\"place:transition=7&sort=4\",\"tags\":[],\"keyword\":null,\"description\":null,\"loadInSidebar\":false,\"parentid\":\"T6XK5oJMU8ih\"}")
+
+        guard let q = BookmarkType.payloadFromJSON(query) else {
+            XCTFail("Failed to generate payload from json: \(query)")
+            return 
+        }
+
+        XCTAssertTrue(q is BookmarkQueryPayload)
+
+        let item = q.toMirrorItem(Date.now())
+        XCTAssertEqual(6, item.type.rawValue)
+        XCTAssertEqual("ShCZLGEFQMam", item.guid)
+
+        let places = JSON(parseJSON: "{\"id\":\"places\",\"type\":\"folder\",\"title\":\"\",\"description\":null,\"children\":[\"menu________\",\"toolbar_____\",\"tags________\",\"unfiled_____\",\"jKnyPDrBQSDg\",\"T6XK5oJMU8ih\"],\"parentid\":\"2hYxKgBwvkEH\"}")
+        guard let p = BookmarkType.payloadFromJSON(places) else {
+            XCTFail("Failed to generate payload from json: \(places)")
+            return
+        }
+
+        XCTAssertTrue(p is FolderPayload)
+
+        // Items keep their GUID until they're written into the mirror table.
+        XCTAssertEqual("places", p.id)
+
+        let pMirror = p.toMirrorItem(Date.now())
+
+        XCTAssertEqual(2, pMirror.type.rawValue)
+
+        // The mirror item has a translated GUID.
+        XCTAssertEqual(BookmarkRoots.RootGUID, pMirror.guid)
+    }
+}

mobile/ios/SyncTests/StateTests.swift

@@ -0,0 +1,68 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+@testable import Sync
+
+import XCTest
+
+func compareScratchpads(tuple: (lhs: Scratchpad, rhs: Scratchpad)) {
+    // This one is set in the constructor!
+    XCTAssertEqual(tuple.lhs.syncKeyBundle, tuple.rhs.syncKeyBundle)
+
+    XCTAssertEqual(tuple.lhs.clientName, tuple.rhs.clientName)
+    XCTAssertEqual(tuple.lhs.clientGUID, tuple.rhs.clientGUID)
+    if let lkeys = tuple.lhs.keys {
+        if let rkeys = tuple.rhs.keys {
+            XCTAssertEqual(lkeys.timestamp, rkeys.timestamp)
+            XCTAssertEqual(lkeys.value, rkeys.value)
+        } else {
+            XCTAssertTrue(tuple.rhs.keys != nil)
+        }
+    } else {
+        XCTAssertTrue(tuple.rhs.keys == nil)
+    }
+
+    XCTAssertTrue(tuple.lhs.global == tuple.rhs.global)
+    XCTAssertEqual(tuple.lhs.localCommands, tuple.rhs.localCommands)
+    XCTAssertEqual(tuple.lhs.engineConfiguration, tuple.rhs.engineConfiguration)
+}
+
+func roundtrip(s: Scratchpad) -> (Scratchpad, rhs: Scratchpad) {
+    let prefs = MockProfilePrefs()
+    let _ = s.pickle(prefs)
+    return (s, rhs: Scratchpad.restoreFromPrefs(prefs, syncKeyBundle: s.syncKeyBundle)!)
+}
+
+class StateTests: XCTestCase {
+    func getGlobal() -> Fetched<MetaGlobal> {
+        let g = MetaGlobal(syncID: "abcdefghiklm", storageVersion: 5, engines: ["bookmarks": EngineMeta(version: 1, syncID: "dddddddddddd")], declined: ["tabs"])
+        return Fetched(value: g, timestamp: Date.now())
+    }
+
+    func getEngineConfiguration() -> EngineConfiguration {
+        return EngineConfiguration(enabled: ["bookmarks", "clients"], declined: ["tabs"])
+    }
+
+    func baseScratchpad() -> Scratchpad {
+        let syncKeyBundle = KeyBundle.fromKB(Bytes.generateRandomBytes(32))
+        let keys = Fetched(value: Keys(defaultBundle: syncKeyBundle), timestamp: 1001)
+        let b = Scratchpad(b: syncKeyBundle, persistingTo: MockProfilePrefs()).evolve()
+        let _ = b.setKeys(keys)
+        b.localCommands = Set([
+            .enableEngine(engine: "tabs"),
+            .disableEngine(engine: "passwords"),
+            .resetAllEngines(except: Set<String>(["bookmarks", "clients"])),
+            .resetEngine(engine: "clients")])
+        return b.build()
+    }
+
+    func testPickling() {
+        compareScratchpads(tuple: roundtrip(s: baseScratchpad()))
+        compareScratchpads(tuple: roundtrip(s: baseScratchpad().evolve().setGlobal(getGlobal()).build()))
+        compareScratchpads(tuple: roundtrip(s: baseScratchpad().evolve().clearLocalCommands().build()))
+        compareScratchpads(tuple: roundtrip(s: baseScratchpad().evolve().setEngineConfiguration(getEngineConfiguration()).build()))
+    }
+}

mobile/ios/SyncTests/StorageClientTests.swift

@@ -0,0 +1,108 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+@testable import Sync
+
+import XCTest
+import SwiftyJSON
+
+// Always return a gigantic encoded payload.
+func massivify(record: Record<CleartextPayloadJSON>) -> JSON? {
+    return JSON([
+        "id": record.id,
+        "foo": String(repeating: "X", count: Sync15StorageClient.maxRecordSizeBytes + 1)
+    ])
+}
+
+class StorageClientTests: XCTestCase {
+    func testPartialJSON() {
+        let body = "0"
+        let o: Any? = try! JSONSerialization.jsonObject(with: body.data(using: String.Encoding.utf8)!, options: JSONSerialization.ReadingOptions.allowFragments)
+        XCTAssertTrue(JSON(object: o!).isInt())
+    }
+
+    func testPOSTResult() {
+        // Pulled straight from <http://docs.services.mozilla.com/storage/apis-1.5.html>.
+        let r = "{" +
+            "\"success\": [\"GXS58IDC_12\", \"GXS58IDC_13\", \"GXS58IDC_15\"," +
+            "\"GXS58IDC_16\", \"GXS58IDC_18\", \"GXS58IDC_19\"]," +
+            "\"failed\": {\"GXS58IDC_11\": \"invalid ttl\"," +
+            "\"GXS58IDC_14\": \"invalid sortindex\"}" +
+        "}"
+
+        let p = POSTResult.fromJSON(JSON(parseJSON: r))
+        XCTAssertTrue(p != nil)
+        XCTAssertEqual(p!.success[0], "GXS58IDC_12")
+        XCTAssertEqual(p!.failed["GXS58IDC_14"]!, "invalid sortindex")
+
+        XCTAssertTrue(nil == POSTResult.fromJSON(JSON(parseJSON: "{\"foo\": 5}")))
+    }
+
+    func testNumeric() {
+        let m = ResponseMetadata(status: 200, headers: [
+            "X-Last-Modified": "2174380461.12",
+        ])
+        XCTAssertTrue(m.lastModifiedMilliseconds == 2174380461120)
+
+        XCTAssertEqual("2174380461.12", millisecondsToDecimalSeconds(2174380461120))
+    }
+
+    // Trivial test for struct semantics that we might want to pay attention to if they change,
+    // and for response header parsing.
+    func testResponseHeaders() {
+        let v: JSON = JSON(parseJSON: "{\"a:\": 2}")
+        let m = ResponseMetadata(status: 200, headers: [
+            "X-Weave-Timestamp": "1274380461.12",
+            "X-Last-Modified": "2174380461.12",
+            "X-Weave-Next-Offset": "abdef",
+            ])
+
+        XCTAssertTrue(m.lastModifiedMilliseconds == 2174380461120)
+        XCTAssertTrue(m.timestampMilliseconds    == 1274380461120)
+        XCTAssertTrue(m.nextOffset == "abdef")
+
+        // Just to avoid consistent overflow allowing ==.
+        XCTAssertTrue(m.lastModifiedMilliseconds?.description == "2174380461120")
+        XCTAssertTrue(m.timestampMilliseconds.description == "1274380461120")
+
+        let x: StorageResponse<JSON> = StorageResponse<JSON>(value: v, metadata: m)
+
+        func doTesting(y: StorageResponse<JSON>) {
+            // Make sure that reference fields in a struct are copies of the same reference,
+            // not references to a copy.
+            XCTAssertTrue(x.value == y.value)
+
+            XCTAssertTrue(y.metadata.lastModifiedMilliseconds == x.metadata.lastModifiedMilliseconds, "lastModified is the same.")
+
+            XCTAssertTrue(x.metadata.quotaRemaining == nil, "No quota.")
+            XCTAssertTrue(y.metadata.lastModifiedMilliseconds == 2174380461120, "lastModified is correct.")
+            XCTAssertTrue(x.metadata.timestampMilliseconds == 1274380461120, "timestamp is correct.")
+            XCTAssertTrue(x.metadata.nextOffset == "abdef", "nextOffset is correct.")
+            XCTAssertTrue(x.metadata.records == nil, "No X-Weave-Records.")
+        }
+
+        doTesting(y: x)
+    }
+
+    func testOverSizeRecords() {
+        let delegate = MockSyncDelegate()
+
+        // We can use these useless values because we're directly injecting decrypted
+        // payloads; no need for real keys etc.
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let synchronizer = IndependentRecordSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled, collection: "foo")
+        let jA = "{\"id\":\"aaaaaa\",\"histUri\":\"http://foo.com/\",\"title\": \"ñ\",\"visits\":[{\"date\":1222222222222222,\"type\":1}]}"
+        let rA = Record<CleartextPayloadJSON>(id: "aaaaaa", payload: CleartextPayloadJSON(JSON(parseJSON: jA)), modified: 10000, sortindex: 123, ttl: 1000000)
+
+        let storageClient = Sync15StorageClient(serverURI: "http://example.com/".asURL!, authorizer: identity, workQueue: DispatchQueue.main, resultQueue: DispatchQueue.main, backoff: MockBackoffStorage())
+        let collectionClient = storageClient.clientForCollection("foo", encrypter: RecordEncrypter<CleartextPayloadJSON>(serializer: massivify, factory: { CleartextPayloadJSON($0) }))
+        let result = synchronizer.uploadRecords([rA], lastTimestamp: Date.now(), storageClient: collectionClient, onUpload: { _ in deferMaybe(Date.now()) })
+
+        XCTAssertTrue(result.value.failureValue is RecordTooLargeError)
+    }
+}

mobile/ios/SyncTests/SyncTests-Bridging-Header.h

@@ -0,0 +1,9 @@
+#ifndef Client_SyncTests_Bridging_Header_h
+#define Client_SyncTests_Bridging_Header_h
+
+#import <Foundation/Foundation.h>
+#import "Shared-Bridging-Header.h"
+#import "Storage-Bridging-Header.h"
+#import "Sync-Bridging-Header.h"
+
+#endif

mobile/ios/SyncTests/TabsPayloadTests.swift

@@ -0,0 +1,90 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import XCTest
+@testable import Sync
+
+class TabsPayloadTests: XCTestCase {
+
+    func testFromInvalidJSON() {
+        let tabsPayload1 = TabsPayload("")
+        XCTAssertFalse(tabsPayload1.isValid())
+
+        let tabsPayload2 = TabsPayload("null")
+        XCTAssertFalse(tabsPayload2.isValid())
+
+        let tabsPayload3 = TabsPayload("{}")
+        XCTAssertFalse(tabsPayload3.isValid())
+
+        let tabsPayload4 = TabsPayload("{\"id\": \"abc\"}")
+        XCTAssertFalse(tabsPayload4.isValid())
+    }
+
+    func testFromJSON() {
+        let tabsPayload = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": []}")
+        XCTAssertTrue(tabsPayload.isValid())
+    }
+
+    func testFromJSONWithInvalidRecord() {
+        let tabsPayload = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": null}")
+        XCTAssertFalse(tabsPayload.isValid())
+
+        let tabsPayload2 = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": 1}")
+        XCTAssertFalse(tabsPayload2.isValid())
+
+        let tabsPayload3 = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": {}}")
+        XCTAssertFalse(tabsPayload3.isValid())
+
+        let tabsPayload4 = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": true}")
+        XCTAssertFalse(tabsPayload4.isValid())
+    }
+
+    func testTabWithBadTabs() {
+        let tabsPayload1 = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{}]}")
+        XCTAssertTrue(tabsPayload1.isValid())
+        let tabs1 = tabsPayload1.tabs
+        XCTAssert(tabs1.count == 0)
+
+        let tabsPayload2 = TabsPayload("{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [null, {}, [], 123, false, true, \"\"]}")
+        XCTAssertTrue(tabsPayload2.isValid())
+        let tabs2 = tabsPayload2.tabs
+        XCTAssert(tabs2.count == 0)
+    }
+
+    func testTabWithCorrectTabLastUsed() {
+        let payloads = [
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": 1492649651}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": \"1492316843992\"}]}"
+        ]
+
+        for payload in payloads {
+            let tabsPayload = TabsPayload(payload)
+            XCTAssertTrue(tabsPayload.isValid())
+            let tabs = tabsPayload.tabs
+            XCTAssert(tabs.count == 1)
+        }
+    }
+
+    func testTabWithBadTabLastUsed() {
+        let payloads = [
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": null}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": \"\"}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": \"cheese\"}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": true}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": false}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": 9223372036854775807}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": 123456789012345678901234567890}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": -1}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": \"123456789012345678901234567890\"}]}",
+            "{\"id\": \"abc\", \"deleted\": false, \"clientName\": \"Foo\", \"tabs\": [{\"title\": \"Some Title\", \"urlHistory\": [\"http://www.example.com\"], \"icon\": null, \"lastUsed\": \"-1\"}]}"
+        ]
+
+        for payload in payloads {
+            let tabsPayload = TabsPayload(payload)
+            XCTAssertTrue(tabsPayload.isValid(), "Should not be valid: \(payload)")
+            let tabs = tabsPayload.tabs
+            XCTAssert(tabs.count == 0, "Should not have valid tabs: \(payload)")
+        }
+    }
+}

mobile/ios/SyncTests/TestBookmarkModel.swift

@@ -0,0 +1,201 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Deferred
+import Foundation
+import Shared
+@testable import Storage
+@testable import Sync
+import XCTest
+
+// Thieved mercilessly from TestSQLiteBookmarks.
+private func getBrowserDBForFile(filename: String, files: FileAccessor) -> BrowserDB? {
+    return BrowserDB(filename: filename, schema: BrowserSchema(), files: files)
+}
+
+class TestBookmarkModel: FailFastTestCase {
+    let files = MockFiles()
+
+    override func tearDown() {
+        do {
+            try self.files.removeFilesInDirectory()
+        } catch {
+        }
+        super.tearDown()
+    }
+
+    private func getBrowserDB(name: String) -> BrowserDB? {
+        let file = "TBookmarkModel\(name).db"
+        print("DB file named: \(file)")
+        return getBrowserDBForFile(filename: file, files: self.files)
+    }
+
+    func getSyncableBookmarks(name: String) -> MergedSQLiteBookmarks? {
+        guard let db = self.getBrowserDB(name: name) else {
+            XCTFail("Couldn't get prepared DB.")
+            return nil
+        }
+
+        return MergedSQLiteBookmarks(db: db)
+    }
+
+    func testBookmarkEditableIfNeverSyncedAndEmptyBuffer() {
+        guard let bookmarks = self.getSyncableBookmarks(name: "A") else {
+            XCTFail("Couldn't get bookmarks.")
+            return
+        }
+
+        // Set a local bookmark
+        let bookmarkURL = "http://AAA.com".asURL!
+        bookmarks.local.insertBookmark(bookmarkURL, title: "AAA", favicon: nil, intoFolder: BookmarkRoots.MenuFolderGUID, withTitle: "").succeeded()
+
+        XCTAssertTrue(bookmarks.isMirrorEmpty().value.successValue!)
+        XCTAssertTrue(bookmarks.buffer.isEmpty().value.successValue!)
+
+        let menuFolder = bookmarks.menuFolder()
+        XCTAssertEqual(menuFolder.current.count, 1)
+        XCTAssertTrue(menuFolder.current[0]!.isEditable)
+    }
+
+    func testBookmarkEditableIfNeverSyncedWithBufferedChanges() {
+        guard let bookmarks = self.getSyncableBookmarks(name: "B") else {
+            XCTFail("Couldn't get bookmarks.")
+            return
+        }
+
+        let bookmarkURL = "http://AAA.com".asURL!
+        bookmarks.local.insertBookmark(bookmarkURL, title: "AAA", favicon: nil, intoFolder: BookmarkRoots.MenuFolderGUID, withTitle: "").succeeded()
+
+        // Add a buffer into the buffer
+        let mirrorDate = Date.now() - 100000
+        bookmarks.applyRecords([
+            BookmarkMirrorItem.folder(BookmarkRoots.MenuFolderGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "Bookmarks Menu", description: "", children: ["BBB"]),
+            BookmarkMirrorItem.bookmark("BBB", dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.MenuFolderGUID, parentName: "Bookmarks Menu", title: "BBB", description: nil, URI: "http://BBB.com", tags: "", keyword: nil)
+        ]).succeeded()
+
+        XCTAssertFalse(bookmarks.buffer.isEmpty().value.successValue!)
+        XCTAssertTrue(bookmarks.isMirrorEmpty().value.successValue!)
+
+        // Check to see if we're editable
+        let menuFolder = bookmarks.menuFolder()
+        XCTAssertEqual(menuFolder.current.count, 1)
+        XCTAssertTrue(menuFolder.current[0]!.isEditable)
+    }
+
+    func testBookmarksEditableWithEmptyBufferAndRemoteBookmark() {
+        guard let bookmarks = self.getSyncableBookmarks(name: "C") else {
+            XCTFail("Couldn't get bookmarks.")
+            return
+        }
+
+        // Add a bookmark to the menu folder in our mirror
+        let mirrorDate = Date.now() - 100000
+        bookmarks.populateMirrorViaBuffer(items: [
+            BookmarkMirrorItem.folder(BookmarkRoots.RootGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "", description: "", children: BookmarkRoots.RootChildren),
+            BookmarkMirrorItem.folder(BookmarkRoots.MenuFolderGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "Bookmarks Menu", description: "", children: ["CCC"]),
+            BookmarkMirrorItem.bookmark("CCC", dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.MenuFolderGUID, parentName: "Bookmarks Menu", title: "CCC", description: nil, URI: "http://CCC.com", tags: "", keyword: nil)
+        ], atDate: mirrorDate)
+
+        // Set a local bookmark
+        let bookmarkURL = "http://AAA.com".asURL!
+        bookmarks.local.insertBookmark(bookmarkURL, title: "AAA", favicon: nil, intoFolder: BookmarkRoots.MenuFolderGUID, withTitle: "").succeeded()
+
+        XCTAssertTrue(bookmarks.buffer.isEmpty().value.successValue!)
+        XCTAssertFalse(bookmarks.isMirrorEmpty().value.successValue!)
+
+        // Check to see if we're editable
+        let menuFolder = bookmarks.menuFolder()
+        XCTAssertEqual(menuFolder.current.count, 2)
+        XCTAssertTrue(menuFolder.current[0]!.isEditable)
+        XCTAssertTrue(menuFolder.current[1]!.isEditable)
+    }
+
+    func testBookmarksNotEditableForUnmergedChanges() {
+        guard let bookmarks = self.getSyncableBookmarks(name: "D") else {
+            XCTFail("Couldn't get bookmarks.")
+            return
+        }
+
+        // Add a bookmark to the menu folder in our mirror
+        let mirrorDate = Date.now() - 100000
+        bookmarks.populateMirrorViaBuffer(items: [
+            BookmarkMirrorItem.folder(BookmarkRoots.RootGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "", description: "", children: BookmarkRoots.RootChildren),
+            BookmarkMirrorItem.folder(BookmarkRoots.MenuFolderGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "Bookmarks Menu", description: "", children: ["EEE"]),
+            BookmarkMirrorItem.bookmark("EEE", dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.MenuFolderGUID, parentName: "Bookmarks Menu", title: "EEE", description: nil, URI: "http://EEE.com", tags: "", keyword: nil)
+        ], atDate: mirrorDate)
+
+        bookmarks.local.insertBookmark("http://AAA.com".asURL!, title: "AAA", favicon: nil, intoFolder: BookmarkRoots.MobileFolderGUID, withTitle: "Bookmarks Menu").succeeded()
+
+        // Add some unmerged bookmarks into the menu folder in the buffer.
+        bookmarks.applyRecords([
+            BookmarkMirrorItem.folder(BookmarkRoots.MenuFolderGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "Bookmarks Menu", description: "", children: ["EEE", "FFF"]),
+            BookmarkMirrorItem.bookmark("FFF", dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.MenuFolderGUID, parentName: "Bookmarks Menu", title: "FFF", description: nil, URI: "http://FFF.com", tags: "", keyword: nil)
+        ]).succeeded()
+
+        XCTAssertFalse(bookmarks.buffer.isEmpty().value.successValue!)
+        XCTAssertFalse(bookmarks.isMirrorEmpty().value.successValue!)
+
+        // Check to see that we can't edit these bookmarks
+        let menuFolder = bookmarks.menuFolder()
+        XCTAssertEqual(menuFolder.current.count, 1)
+        XCTAssertFalse(menuFolder.current[0]!.isEditable)
+
+    }
+
+    func testLocalBookmarksEditableWhileHavingUnmergedChangesAndEmptyMirror() {
+        guard let bookmarks = self.getSyncableBookmarks(name: "D") else {
+            XCTFail("Couldn't get bookmarks.")
+            return
+        }
+
+        bookmarks.local.insertBookmark("http://AAA.com".asURL!, title: "AAA", favicon: nil, intoFolder: BookmarkRoots.MobileFolderGUID, withTitle: "Bookmarks Menu").succeeded()
+
+        // Add some unmerged bookmarks into the menu folder in the buffer.
+        let mirrorDate = Date.now() - 100000
+        bookmarks.applyRecords([
+            BookmarkMirrorItem.folder(BookmarkRoots.MenuFolderGUID, dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: "", title: "Bookmarks Menu", description: "", children: ["EEE", "FFF"]),
+            BookmarkMirrorItem.bookmark("FFF", dateAdded: mirrorDate, modified: mirrorDate, hasDupe: false, parentID: BookmarkRoots.MenuFolderGUID, parentName: "Bookmarks Menu", title: "FFF", description: nil, URI: "http://FFF.com", tags: "", keyword: nil)
+        ]).succeeded()
+
+        // Local bookmark should be editable
+        let mobileFolder = bookmarks.mobileFolder()
+        XCTAssertEqual(mobileFolder.current.count, 2)
+        XCTAssertTrue(mobileFolder.current[1]!.isEditable)
+    }
+}
+
+private extension MergedSQLiteBookmarks {
+    func isMirrorEmpty() -> Deferred<Maybe<Bool>> {
+        return self.local.db.queryReturnsNoResults("SELECT 1 FROM \(TableBookmarksMirror)")
+    }
+
+    func wipeLocal() {
+        self.local.db.run(["DELETE FROM \(TableBookmarksLocalStructure)", "DELETE FROM \(TableBookmarksLocal)"]).succeeded()
+    }
+
+    func populateMirrorViaBuffer(items: [BookmarkMirrorItem], atDate mirrorDate: Timestamp) {
+        self.applyRecords(items).succeeded()
+
+        // … and add the root relationships that will be missing (we don't do those for the buffer,
+        // so we need to manually add them and move them across).
+        self.buffer.db.run([
+            "INSERT INTO \(TableBookmarksBufferStructure) (parent, child, idx) VALUES",
+            "('\(BookmarkRoots.RootGUID)', '\(BookmarkRoots.MenuFolderGUID)', 0),",
+            "('\(BookmarkRoots.RootGUID)', '\(BookmarkRoots.ToolbarFolderGUID)', 1),",
+            "('\(BookmarkRoots.RootGUID)', '\(BookmarkRoots.UnfiledFolderGUID)', 2),",
+            "('\(BookmarkRoots.RootGUID)', '\(BookmarkRoots.MobileFolderGUID)', 3)",
+        ].joined(separator: " ")).succeeded()
+
+        // Move it all to the mirror.
+        self.local.db.moveBufferToMirrorForTesting()
+    }
+
+    func menuFolder() -> BookmarksModel {
+        return modelFactory.value.successValue!.modelForFolder(BookmarkRoots.MenuFolderGUID).value.successValue!
+    }
+
+    func mobileFolder() -> BookmarksModel {
+        return modelFactory.value.successValue!.modelForFolder(BookmarkRoots.MobileFolderGUID).value.successValue!
+    }
+}

mobile/ios/SyncTests/TestBookmarksRepairRequestor.swift

@@ -0,0 +1,519 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Deferred
+import Foundation
+import Shared
+@testable import Storage
+@testable import Sync
+@testable import SyncTelemetry
+import SwiftyJSON
+
+import XCTest
+
+class TestBookmarksRepairRequestor: XCTestCase {
+    private let MockHashedDeviceID = "4b66918e184c0a9a49c4a9dc7468d3495642141a08419e69c6cb107367366176"
+
+    private func buildMockScratchpad(prefs: Prefs) -> Scratchpad {
+        var scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+        let b = Scratchpad.Builder(p: scratchpad)
+        b.hashedUID = "1234"
+        scratchpad = b.build()
+
+        XCTAssertEqual(scratchpad.fxaDeviceId, "unknown_fxaDeviceId")
+        XCTAssertEqual(scratchpad.hashedDeviceID!, MockHashedDeviceID)
+        return scratchpad
+    }
+
+    func testNoClients() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: localClient, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2"]]
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo) >>== { result in
+            XCTAssertTrue(result)
+            XCTAssertEqual(remoteClients.commands.count, 1)
+            XCTAssertEqual(remoteClients.commands["localID"]!.count, 0)
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testOneClientNoResponse() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClient = RemoteClient(guid: "client-a", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: remoteClient, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2"]]
+
+        let mockFlowID = Bytes.generateGUID()
+
+        // Mock telemetry events
+        let startedEvent = makeRepairEvent(["started", nil, ["flowID": mockFlowID, "numIDs": "2"]])
+        let uploadEvent = makeRepairEvent(["request", "upload", ["flowID": mockFlowID, "deviceID": MockHashedDeviceID, "numIDs": "2"]])
+        let finishedEvent = makeRepairEvent(["finished", nil, ["flowID": mockFlowID, "numIDs": "2"]])
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo, flowID: mockFlowID) >>== { result -> Deferred<Maybe<Bool>> in
+            XCTAssertTrue(result)
+            XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+
+            checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+            checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+            // asking it to continue stays in that state until we timeout or the command
+            // is removed.
+            return requestor.continueRepairs()
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+
+                // now pretend that client synced.
+                let _ = remoteClients.deleteCommands("client-a")
+                return requestor.continueRepairs()
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent-again")
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent, uploadEvent)
+
+                // the command should be outgoing again.
+                checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+
+                // pretend that client synced again without writing a command.
+                let _ = remoteClients.deleteCommands("client-a")
+                return requestor.continueRepairs()
+            } >>== { result in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), nil)
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent, uploadEvent, finishedEvent)
+
+                expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testOneClientTimeout() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClient = RemoteClient(guid: "client-a", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: remoteClient, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2"]]
+
+        // Mock telemetry events
+        let mockFlowID = Bytes.generateGUID()
+        let startedEvent = makeRepairEvent(["started", nil, ["flowID": mockFlowID, "numIDs": "2"]])
+        let uploadEvent = makeRepairEvent(["request", "upload", ["flowID": mockFlowID, "deviceID": MockHashedDeviceID, "numIDs": "2"]])
+        let abandonEvent = makeRepairEvent(["abandon", "silent", ["flowID": mockFlowID, "deviceID": MockHashedDeviceID]])
+        let finishedEvent = makeRepairEvent(["finished", nil, ["flowID": mockFlowID, "numIDs": "2"]])
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo, flowID: mockFlowID) >>== { result -> Deferred<Maybe<Bool>> in
+            XCTAssertTrue(result)
+            XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+            checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+            checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+            // pretend we are now in the future (well actually, that the request was made a long time ago)
+            prefs.setTimestamp(0, forKey: "repairs.bookmark.when")
+            return requestor.continueRepairs()
+            } >>== { result in
+                // We should be finished as we gave up in disgust.
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), nil)
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent, abandonEvent, finishedEvent)
+                expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testLatestClientUsed() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let clientEarly = RemoteClient(guid: "client-early", name: "Test remote client", modified: (Date.now() - OneWeekInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let clientLate = RemoteClient(guid: "client-late", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: clientEarly, tabs: []), ClientAndTabs(client: clientLate, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2"]]
+
+        // Mock telemetry events
+        let mockFlowID = Bytes.generateGUID()
+        let startedEvent = makeRepairEvent(["started", nil, ["flowID": mockFlowID, "numIDs": "2"]])
+        let uploadEvent = makeRepairEvent(["request", "upload", ["flowID": mockFlowID, "deviceID": MockHashedDeviceID, "numIDs": "2"]])
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo, flowID: mockFlowID) >>== { result in
+            XCTAssertTrue(result)
+            XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+            checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+            // the repair command should be outgoing to the most-recent client.
+            checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-late")
+            expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testClientVanishes() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClientA = RemoteClient(guid: "client-a", name: "Test remote client", modified: Date.now(), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClientB = RemoteClient(guid: "client-b", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: remoteClientA, tabs: []), ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: remoteClientB, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2"]]
+
+        // Mock telemetry events
+        let flowID = Bytes.generateGUID()
+        let startedEvent = makeRepairEvent(["started", nil, ["flowID": flowID, "numIDs": "2"]])
+        let uploadEvent = makeRepairEvent(["request", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "2"]])
+        let missingEvent = makeRepairEvent(["abandon", "missing", ["flowID": flowID, "deviceID": MockHashedDeviceID]])
+        let responseEvent = makeRepairEvent(["response", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "2"]])
+        let finishedEvent = makeRepairEvent(["finished", nil, ["flowID": flowID, "numIDs": "0"]])
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo, flowID: flowID) >>== { result -> Deferred<Maybe<Bool>> in
+            XCTAssertTrue(result)
+            XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+            checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+
+            checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+            // asking it to continue stays in that state until we timeout or the command
+            // is removed.
+            return requestor.continueRepairs()
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+                // the command should now be outgoing.
+                checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+
+                let _ = remoteClients.deleteCommands("client-a")
+                // Now let's pretend the client vanished.
+                remoteClients.clientsAndTabs.removeFirst()
+                return requestor.continueRepairs()
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent, missingEvent, uploadEvent)
+
+                // We should have moved on to client-b.
+                checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-b")
+
+                // Now let's pretend client B wrote all missing IDs.
+                let repairResponse = RepairResponse(collection: "bookmarks", request: "upload", flowID: flowID, clientID: "client-b", ids: ["mock-guid1", "mock-guid2"])
+                return requestor.continueRepairs(response: repairResponse)
+            } >>== { result in
+                XCTAssertTrue(result)
+                checkRecordedEvents(fromPrefs: prefs,
+                                    expected: startedEvent, uploadEvent, missingEvent,
+                                    uploadEvent, responseEvent, finishedEvent)
+                // We should be finished as we got all our IDs.
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), nil)
+                expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testMultiClients() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClientA = RemoteClient(guid: "client-a", name: "Test remote client", modified: Date.now(), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClientB = RemoteClient(guid: "client-b", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: remoteClientA, tabs: []), ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: remoteClientB, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2",
+            "mock-guid3"]]
+
+        let flowID = Bytes.generateGUID()
+
+        // Mock telemetry events
+        let startedEvent = makeRepairEvent(["started", nil, ["flowID": flowID, "numIDs": "3"]])
+        let firstUploadEvent = makeRepairEvent(["request", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "3"]])
+        let secondUploadEvent = makeRepairEvent(["request", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "1"]])
+        let firstResponseEvent = makeRepairEvent(["response", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "2"]])
+        let secondResponseEvent = makeRepairEvent(["response", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "1"]])
+        let finishedEvent = makeRepairEvent(["finished", nil, ["flowID": flowID, "numIDs": "0"]])
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo, flowID: flowID) >>== { result -> Deferred<Maybe<Bool>> in
+            XCTAssertTrue(result)
+            XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+            checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+            // asking it to continue stays in that state until we timeout or the command
+            // is removed.
+            return requestor.continueRepairs()
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, firstUploadEvent)
+
+                // the command should now be outgoing.
+                checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+
+                let _ = remoteClients.deleteCommands("client-a")
+                // Now let's pretend the client wrote a response.
+                let repairResponse = RepairResponse(collection: "bookmarks", request: "upload", flowID: flowID, clientID: "client-a", ids: ["mock-guid1", "mock-guid2"])
+                return requestor.continueRepairs(response: repairResponse)
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+                checkRecordedEvents(fromPrefs: prefs,
+                                    expected: startedEvent, firstUploadEvent, firstResponseEvent, secondUploadEvent)
+
+                // We should have moved on to client-b.
+                checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-b")
+
+                let _ = remoteClients.deleteCommands("client-b")
+                // Now let's pretend client B write the missing ID.
+                let repairResponse = RepairResponse(collection: "bookmarks", request: "upload", flowID: flowID, clientID: "client-b", ids: ["mock-guid3"])
+                return requestor.continueRepairs(response: repairResponse)
+            } >>== { result in
+                XCTAssertTrue(result)
+
+                checkRecordedEvents(fromPrefs: prefs,
+                                    expected: startedEvent, firstUploadEvent, firstResponseEvent, secondUploadEvent, secondResponseEvent, finishedEvent)
+                // We should be finished as we got all our IDs.
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), nil)
+                expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testAlreadyRepairingContinue() {
+        let expectation = self.expectation(description: #function)
+
+        let prefs = MockProfilePrefs()
+        let scratchpad = buildMockScratchpad(prefs: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClientA = RemoteClient(guid: "client-a", name: "Test remote client", modified: Date.now(), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClientB = RemoteClient(guid: "client-b", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: remoteClientA, tabs: []), ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: remoteClientB, tabs: [])])
+        let validationInfo = [BufferInconsistency.missingValues: ["mock-guid1", "mock-guid2", "mock-guid3"]]
+
+        // Mock telemetry events
+        let flowID = Bytes.generateGUID()
+        let startedEvent = makeRepairEvent(["started", nil, ["flowID": flowID, "numIDs": "3"]])
+        let uploadEvent = makeRepairEvent(["request", "upload", ["flowID": flowID, "deviceID": MockHashedDeviceID, "numIDs": "3"]])
+        let abortedEvent = makeRepairEvent(["aborted", nil, ["flowID": flowID, "reason": "other clients repairing", "numIDs": "3"]])
+
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+        requestor.startRepairs(validationInfo: validationInfo, flowID: flowID) >>== { result -> Deferred<Maybe<Bool>> in
+            XCTAssertTrue(result)
+            XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+            checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+            // asking it to continue stays in that state until we timeout or the command
+            // is removed.
+            return requestor.continueRepairs()
+            } >>== { result -> Deferred<Maybe<Bool>> in
+                XCTAssertTrue(result)
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), "repair.sent")
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent)
+                // the command should now be outgoing.
+                checkOutgoingCommand(remoteClients: remoteClients, clientID: "client-a")
+
+                let _ = remoteClients.deleteCommands("client-a")
+                // Now let's pretend the client wrote a response (it doesn't matter what's in here)
+                let repairResponse = RepairResponse(collection: "bookmarks", request: "upload", flowID: flowID, clientID: "client-a", ids: ["mock-guid1", "mock-guid2"])
+
+                // and another client also started a request
+                let otherRequest = RepairRequest(collection: "bookmarks", request: "upload", flowID: "abdc", requestor: "client-c", ids: ["bogusid"])
+                let _ = remoteClients.insertCommand(otherRequest.toSyncCommand(), forClients: [remoteClientB])
+
+                return requestor.continueRepairs(response: repairResponse)
+            } >>== { result in
+                XCTAssertTrue(result)
+
+                // We should have aborted now
+                XCTAssertEqual(prefs.stringForKey("repairs.bookmark.state"), nil)
+
+                checkRecordedEvents(fromPrefs: prefs, expected: startedEvent, uploadEvent, abortedEvent)
+                expectation.fulfill()
+        }
+        waitForExpectations(timeout: 10, handler: nil)
+    }
+
+    func testSyncEventsPickledInPrefs() {
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+        let localClient = RemoteClient(guid: nil, name: "Test local client", modified: (Date.now() - OneMinuteInMilliseconds), type: "mobile", formfactor: "largetablet", os: "iOS", version: nil, fxaDeviceId: nil)
+        let remoteClientA = RemoteClient(guid: "client-a", name: "Test remote client", modified: Date.now(), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClientB = RemoteClient(guid: "client-b", name: "Test remote client", modified: (Date.now() - OneMinuteInMilliseconds), type: "desktop", formfactor: nil, os: nil, version: "55.0.1", fxaDeviceId: nil)
+        let remoteClients = MockRemoteClientsAndTabs([ClientAndTabs(client: remoteClientA, tabs: []), ClientAndTabs(client: localClient, tabs: []), ClientAndTabs(client: remoteClientB, tabs: [])])
+        let requestor = BookmarksRepairRequestor(scratchpad: scratchpad, basePrefs: prefs, remoteClients: remoteClients)
+
+        let mockTimestamp = Date.now()
+        let mockEvent = Event(timestamp: mockTimestamp, category: "test",
+                              method: "method", object: "object", value: "value",
+                              extra: ["test": "value"])
+
+        requestor.recordTelemetry(event: mockEvent)
+
+        let events = requestor.basePrefs.arrayForKey(PrefKeySyncEvents) as! [Data]
+        XCTAssertEqual(events.count, 1)
+
+        let pickledEvent = Event.unpickle(events[0])
+        XCTAssertEqual(pickledEvent!.category, mockEvent.category)
+    }
+}
+
+func checkOutgoingCommand(remoteClients: MockRemoteClientsAndTabs, clientID: GUID) {
+    let outgoingCmds = remoteClients.commands[clientID]!
+    XCTAssertEqual(outgoingCmds.count, 1)
+    XCTAssertEqual(JSON.parse(outgoingCmds.first!.value)["command"].stringValue, "repairRequest")
+}
+
+func checkRecordedEvents(fromPrefs prefs: Prefs, expected: Event...) {
+    let eventData = prefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? []
+    let actualEvents = eventData.map(Event.unpickle)
+    XCTAssertEqual(actualEvents.count, expected.count)
+    expected.enumerated().forEach { offset, expected in
+        XCTAssertEqual(expected, actualEvents[offset])
+    }
+}
+
+func makeRepairEvent(_ values: [Any?]) -> Event {
+    return Event(timestamp: Date.now(),
+                 category: "sync",
+                 method: "repair",
+                 object: values[0] as! String,
+                 value: values[1] as? String,
+                 extra: values[2] as? [String: String]
+    )
+}
+
+// Checks equivalence while ignoring the timestamp
+extension Event: Equatable {
+    public static func ==(left: Event, right: Event) -> Bool {
+        let propsAreEqual = (left.category == right.category) &&
+                            (left.method == right.method) &&
+                            (left.object == right.object) &&
+                            (left.value ?? "" == right.value ?? "") &&
+                            (left.extra ?? [:] == right.extra ?? [:])
+        return propsAreEqual
+    }
+}
+
+open class MockRemoteClientsAndTabs: RemoteClientsAndTabs {
+    open var clientsAndTabs: [ClientAndTabs]
+    open var commands: [GUID: [SyncCommand]]
+
+    public init(_ clientsAndTabs: [ClientAndTabs]) {
+        self.clientsAndTabs = clientsAndTabs
+        self.commands = clientsAndTabs.map { $0.client.guid ?? "localID" }.reduce([String: [SyncCommand]]()) { (dict, clientId) -> [String: [SyncCommand]] in
+            var dict = dict
+            dict[clientId] = [SyncCommand]()
+            return dict
+        }
+    }
+
+    open func onRemovedAccount() -> Success {
+        return succeed()
+    }
+
+    open func wipeClients() -> Success {
+        return succeed()
+    }
+
+    open func wipeRemoteTabs() -> Deferred<Maybe<()>> {
+        return succeed()
+    }
+
+    open func wipeTabs() -> Success {
+        return succeed()
+    }
+
+    open func insertOrUpdateClients(_ clients: [RemoteClient]) -> Deferred<Maybe<Int>> {
+        return deferMaybe(0)
+    }
+
+    open func insertOrUpdateClient(_ client: RemoteClient) -> Deferred<Maybe<Int>> {
+        return deferMaybe(0)
+    }
+
+    open func insertOrUpdateTabs(_ tabs: [RemoteTab]) -> Deferred<Maybe<Int>> {
+        return insertOrUpdateTabsForClientGUID(nil, tabs: [RemoteTab]())
+    }
+
+    open func insertOrUpdateTabsForClientGUID(_ clientGUID: String?, tabs: [RemoteTab]) -> Deferred<Maybe<Int>> {
+        return deferMaybe(-1)
+    }
+
+    open func getClientsAndTabs() -> Deferred<Maybe<[ClientAndTabs]>> {
+        return deferMaybe(self.clientsAndTabs)
+    }
+
+    open func getClients() -> Deferred<Maybe<[RemoteClient]>> {
+        return deferMaybe(self.clientsAndTabs.map { $0.client })
+    }
+
+    open func getClientWithId(_ clientID: GUID) -> Deferred<Maybe<RemoteClient?>> {
+        return self.getClient(guid: clientID)
+    }
+
+    open func getClient(fxaDeviceId: String) -> Deferred<Maybe<RemoteClient?>> {
+        return deferMaybe(self.clientsAndTabs.find { clientAndTabs in
+            return clientAndTabs.client.fxaDeviceId == fxaDeviceId
+            }?.client)
+    }
+
+    open func getClient(guid: GUID) -> Deferred<Maybe<RemoteClient?>> {
+        return deferMaybe(self.clientsAndTabs.find { clientAndTabs in
+            return clientAndTabs.client.guid == guid
+            }?.client)
+    }
+
+    open func deleteClient(guid: GUID) -> Success {
+        clientsAndTabs = clientsAndTabs.filter { $0.client.guid != guid }
+        return succeed()
+    }
+
+    open func getClientGUIDs() -> Deferred<Maybe<Set<GUID>>> {
+        return deferMaybe(Set<GUID>(optFilter(self.clientsAndTabs.map { $0.client.guid })))
+    }
+
+    open func getTabsForClientWithGUID(_ guid: GUID?) -> Deferred<Maybe<[RemoteTab]>> {
+        return deferMaybe(optFilter(self.clientsAndTabs.map { $0.client.guid == guid ? $0.tabs : nil })[0])
+    }
+
+    open func deleteCommands() -> Success {
+        self.commands = [GUID: [SyncCommand]]()
+        return succeed()
+    }
+    open func deleteCommands(_ clientGUID: GUID) -> Success {
+        self.commands[clientGUID] = [SyncCommand]()
+        return succeed()
+    }
+
+    open func getCommands() -> Deferred<Maybe<[GUID: [SyncCommand]]>> {
+        return deferMaybe(self.commands)
+    }
+
+    open func insertCommand(_ command: SyncCommand, forClients clients: [RemoteClient]) -> Deferred<Maybe<Int>> {
+        return self.insertCommands([command], forClients: clients)
+    }
+    open func insertCommands(_ commands: [SyncCommand], forClients clients: [RemoteClient]) -> Deferred<Maybe<Int>> {
+        var numInserts = 0
+        for client in clients where self.commands[client.guid ?? "localID"] != nil {
+            self.commands[client.guid ?? "localID"]! += commands
+            numInserts += commands.count
+        }
+        return deferMaybe(numInserts)
+    }
+}

mobile/ios/SyncTests/TestBookmarksSynchronizer.swift

@@ -0,0 +1,434 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Deferred
+@testable import Sync
+@testable import Storage
+
+import XCTest
+import SwiftyJSON
+
+class MockStorage: LocalItemSource, MirrorItemSource, SyncableBookmarks {
+    var local: [GUID: BookmarkMirrorItem] = [:]
+    var localAdditions: [GUID] = []
+    var localDeletions: [GUID] = []
+    var lastBufferUpdatedCompletionOpApplied: BufferUpdatedCompletionOp?
+
+    // LocalItemSource methods.
+
+    func getLocalItemWithGUID(_ guid: GUID) -> Deferred<Maybe<BookmarkMirrorItem>> {
+        guard let item = self.local[guid] else {
+            return deferMaybe(DatabaseError(description: "Couldn't find item \(guid)."))
+        }
+        return deferMaybe(item)
+    }
+
+    func getLocalItemsWithGUIDs<T: Collection>(_ guids: T) -> Deferred<Maybe<[GUID: BookmarkMirrorItem]>> where T.Iterator.Element == GUID {
+        var acc: [GUID: BookmarkMirrorItem] = [:]
+        guids.forEach { guid in
+            if let item = self.local[guid] {
+                acc[guid] = item
+            }
+        }
+        return deferMaybe(acc)
+    }
+
+    func prefetchLocalItemsWithGUIDs<T: Collection>(_ guids: T) -> Success where T.Iterator.Element == GUID {
+        return succeed()
+    }
+
+    // MirrorItemSource methods (not implemented!).
+
+    func getMirrorItemWithGUID(_ guid: GUID) -> Deferred<Maybe<BookmarkMirrorItem>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func getMirrorItemsWithGUIDs<T>(_ guids: T) -> Deferred<Maybe<[GUID : BookmarkMirrorItem]>> where T : Collection, T.Iterator.Element == GUID {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func prefetchMirrorItemsWithGUIDs<T: Collection>(_ guids: T) -> Success where T.Iterator.Element == GUID {
+        return succeed()
+    }
+
+    // SyncableBookmarks methods (partialy implemented!).
+
+    func isUnchanged() -> Deferred<Maybe<Bool>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func getLocalBookmarksModifications(limit: Int) -> Deferred<Maybe<(deletions: [GUID], additions: [BookmarkMirrorItem])>> {
+        let deletions = Array(self.localDeletions.prefix(limit))
+        let additions = Array(self.localAdditions.prefix(limit - deletions.count)).map { self.local[$0]! }
+        return deferMaybe((deletions: deletions, additions: additions))
+    }
+
+    func getLocalDeletions() -> Deferred<Maybe<[(GUID, Timestamp)]>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func treesForEdges() -> Deferred<Maybe<(local: BookmarkTree, buffer: BookmarkTree)>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func treeForMirror() -> Deferred<Maybe<BookmarkTree>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func applyLocalOverrideCompletionOp(_ op: LocalOverrideCompletionOp, itemSources: ItemSources) -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func applyBufferUpdatedCompletionOp(_ op: BufferUpdatedCompletionOp) -> Success {
+        self.lastBufferUpdatedCompletionOpApplied = op
+        return succeed()
+    }
+
+    // Misc methods.
+
+    func resetClient() -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func onRemovedAccount() -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+}
+
+class MockBuffer: BookmarkBufferStorage, BufferItemSource {
+    var buffer: [GUID: BookmarkMirrorItem] = [:]
+    var children: [GUID: [GUID]] = [:]
+
+    // BufferItemSource methods.
+
+    func getBufferItemWithGUID(_ guid: GUID) -> Deferred<Maybe<BookmarkMirrorItem>> {
+        guard let item = self.buffer[guid] else {
+            return deferMaybe(DatabaseError(description: "Couldn't find item \(guid)."))
+        }
+        return deferMaybe(item)
+    }
+
+    func getBufferItemsWithGUIDs<T>(_ guids: T) -> Deferred<Maybe<[GUID : BookmarkMirrorItem]>> where T : Collection, T.Iterator.Element == GUID {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func getBufferChildrenGUIDsForParent(_ guid: GUID) -> Deferred<Maybe<[GUID]>> {
+        guard let children = self.children[guid] else {
+            return deferMaybe(DatabaseError(description: "Couldn't find children for \(guid)."))
+        }
+        return deferMaybe(children)
+    }
+
+    func prefetchBufferItemsWithGUIDs<T>(_ guids: T) -> Success where T : Collection, T.Iterator.Element == GUID {
+        return succeed()
+    }
+
+    // BookmarkBufferStorage methods.
+
+    func isEmpty() -> Deferred<Maybe<Bool>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func applyRecords(_ records: [BookmarkMirrorItem]) -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func doneApplyingRecordsAfterDownload() -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func validate() -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func getBufferedDeletions() -> Deferred<Maybe<[(GUID, Timestamp)]>> {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func applyBufferCompletionOp(_ op: BufferCompletionOp, itemSources: ItemSources) -> Success {
+        return deferMaybe(DatabaseError(description: "Not implemented"))
+    }
+
+    func synchronousBufferCount() -> Int? {
+        return nil
+    }
+
+    func getUpstreamRecordCount() -> Deferred<Int?> {
+        return Deferred(value: nil)
+    }
+}
+
+class TestBookmarksSynchronizer: XCTestCase {
+    func testBuildMobileRootAndChildrenRecords_noMobileRootInBuffer() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let buffer = MockBuffer()
+        let storage = MockStorage()
+        storage.local[BookmarkRoots.MobileFolderGUID] = BookmarkMirrorItem.folder(BookmarkRoots.MobileFolderGUID, dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: nil, title: "Mobile Bookmarks", description: nil, children: ["bk1"])
+        storage.local["bk1"] = BookmarkMirrorItem.bookmark("bk1", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 1", description: nil, URI: "https://example.com/1", tags: "", keyword: nil)
+        storage.local["bk1"] = BookmarkMirrorItem.bookmark("bk1", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 1", description: nil, URI: "https://example.com/1", tags: "", keyword: nil)
+        let bk2 = BookmarkMirrorItem.bookmark("bk2", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 2", description: nil, URI: "https://example.com/2", tags: "", keyword: nil)
+        let bk3 = BookmarkMirrorItem.bookmark("bk3", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 3", description: nil, URI: "https://example.com/3", tags: "", keyword: nil)
+        storage.local["bk2"] = bk2
+        storage.local["bk3"] = bk3
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        let (mobileRootRecord, childrenRecords) = synchronizer.buildMobileRootAndChildrenRecords(storage, buffer, additionalChildren: [bk2, bk3], deletedChildren: []).value.successValue!
+        XCTAssertEqual(mobileRootRecord.id, BookmarkRoots.translateOutgoingRootGUID(BookmarkRoots.MobileFolderGUID))
+        XCTAssertEqual(mobileRootRecord.payload.json["title"], "Mobile Bookmarks")
+        // We are not including bk1 in the call to buildMobileRootRecord() therefore it should NOT be included in the returned record
+        XCTAssertEqual(mobileRootRecord.payload.json["children"], ["bk2", "bk3"])
+
+        XCTAssertEqual(childrenRecords.count, 2)
+        XCTAssertEqual(childrenRecords[0].id, "bk2")
+        XCTAssertEqual(childrenRecords[1].id, "bk3")
+    }
+
+    func testBuildMobileRootAndChildrenRecords_mobileRootInBuffer() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let buffer = MockBuffer()
+        buffer.buffer[BookmarkRoots.MobileFolderGUID] = BookmarkMirrorItem.folder(BookmarkRoots.MobileFolderGUID, dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: nil, title: "Mobile Bookmarks", description: nil, children: ["bk1", "bkmdeleteme"])
+        buffer.children[BookmarkRoots.MobileFolderGUID] = ["bk1", "bkmdeleteme"]
+        buffer.buffer["bk1"] = BookmarkMirrorItem.bookmark("bk1", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 1", description: nil, URI: "https://example.com/1", tags: "", keyword: nil)
+        buffer.buffer["bkmdeleteme"] = BookmarkMirrorItem.bookmark("bkmdeleteme", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark to delete", description: nil, URI: "https://example.com/todelete", tags: "", keyword: nil)
+
+        let storage = MockStorage()
+        storage.local["bk1"] = BookmarkMirrorItem.bookmark("bk1", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 1", description: nil, URI: "https://example.com/1", tags: "", keyword: nil)
+        let bk2 = BookmarkMirrorItem.bookmark("bk2", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 2", description: nil, URI: "https://example.com/2", tags: "", keyword: nil)
+        let bk3 = BookmarkMirrorItem.bookmark("bk3", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "Bookmark 3", description: nil, URI: "https://example.com/3", tags: "", keyword: nil)
+        storage.local["bk2"] = bk2
+        storage.local["bk3"] = bk3
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        let (mobileRootRecord, childrenRecords) = synchronizer.buildMobileRootAndChildrenRecords(storage, buffer, additionalChildren: [bk2, bk3], deletedChildren: ["bkmdeleteme"]).value.successValue!
+        XCTAssertEqual(mobileRootRecord.id, BookmarkRoots.translateOutgoingRootGUID(BookmarkRoots.MobileFolderGUID))
+        XCTAssertEqual(mobileRootRecord.payload.json["title"], "Mobile Bookmarks")
+        // bk1 is a children of mobile root in the buffer, therefore it should be included here.
+        XCTAssertEqual(mobileRootRecord.payload.json["children"], ["bk1", "bk2", "bk3"])
+
+        // We are only sending the new/deleted records though!
+        XCTAssertEqual(childrenRecords.count, 3)
+        XCTAssertEqual(childrenRecords[0].id, "bk2")
+        XCTAssertEqual(childrenRecords[1].id, "bk3")
+        XCTAssertEqual(childrenRecords[2].id, "bkmdeleteme")
+        XCTAssertTrue(childrenRecords[2].payload.deleted)
+    }
+
+    func testUploadSomeLocalRecords_batched_ok() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let storage = MockStorage()
+        let buffer = MockBuffer()
+
+        let records = createMobileRootAndChildrenRecords(numChildren: 80)
+
+        var numPosts = 0
+        let now = Date.now()
+        var firstPostLastModified: Timestamp?
+        var secondPostLastModified: Timestamp?
+        let uploader: BatchUploadFunction = { lines, ifUnmodifiedSince, queryParams in
+            numPosts += 1
+            var headers = [String: Any]()
+
+            let result: POSTResult
+            if numPosts == 1 {
+                result = POSTResult(success: records.childrenRecords.map { $0.id } + ["mobile"], failed: [:], batchToken: "toktok")
+                firstPostLastModified = now
+                headers["X-Last-Modified"] = firstPostLastModified
+            } else {
+                XCTAssertEqual(queryParams![0].name, "batch")
+                XCTAssertEqual(queryParams![0].value, "toktok")
+                XCTAssertEqual(queryParams![1].name, "commit")
+                XCTAssertEqual(queryParams![1].value, "true")
+                result = POSTResult(success: [], failed: [:])
+                secondPostLastModified = now + 5000
+                headers["X-Last-Modified"] = secondPostLastModified
+            }
+            let response = StorageResponse<POSTResult>(value: result, metadata: ResponseMetadata(status: 200, headers: headers))
+            return deferMaybe(response)
+        }
+
+        let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 100, maxPostBytes: 1_048_576, maxTotalRecords: 250, maxTotalBytes: 104_857_600)
+        let collectionClient = MockSyncCollectionClient(uploader: uploader, infoConfig: miniConfig, collection: "bookmarks", encrypter: getBookmarksEncrypter())
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: collectionClient, basePrefs: prefs, collection: "bookmarks", statsSession: SyncEngineStatsSession(collection: "bookmarks"))
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        synchronizer.uploadSomeLocalRecords(storage, mirrorer, collectionClient, mobileRootRecord: records.mobileRootRecord, childrenRecords: records.childrenRecords).succeeded()
+        XCTAssertEqual(storage.lastBufferUpdatedCompletionOpApplied?.bufferValuesToMoveFromLocal.count, 81)
+        XCTAssertEqual(numPosts, 2)
+        XCTAssertEqual(mirrorer.lastModified / 1000, secondPostLastModified)
+    }
+
+    func testUploadSomeLocalRecords_batched_someFailed() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let storage = MockStorage()
+        let buffer = MockBuffer()
+
+        let records = createMobileRootAndChildrenRecords(numChildren: 80)
+
+        var numPosts = 0
+        let uploader: BatchUploadFunction = { lines, ifUnmodifiedSince, queryParams in
+            numPosts += 1
+            var headers = [String: Any]()
+            headers["X-Last-Modified"] = Date.now()
+
+            let result = POSTResult(success: records.childrenRecords.map { $0.id }.dropFirst(20) + ["mobile"], failed: [:], batchToken: "toktok")
+            let response = StorageResponse<POSTResult>(value: result, metadata: ResponseMetadata(status: 200, headers: headers))
+            return deferMaybe(response)
+        }
+
+        let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 100, maxPostBytes: 1_048_576, maxTotalRecords: 250, maxTotalBytes: 104_857_600)
+        let collectionClient = MockSyncCollectionClient(uploader: uploader, infoConfig: miniConfig, collection: "bookmarks", encrypter: getBookmarksEncrypter())
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: collectionClient, basePrefs: prefs, collection: "bookmarks", statsSession: SyncEngineStatsSession(collection: "bookmarks"))
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        let error = synchronizer.uploadSomeLocalRecords(storage, mirrorer, collectionClient, mobileRootRecord: records.mobileRootRecord, childrenRecords: records.childrenRecords).value.failureValue!
+        XCTAssertTrue(error is RecordsFailedToUpload)
+        XCTAssertNil(storage.lastBufferUpdatedCompletionOpApplied)
+        XCTAssertEqual(numPosts, 1)
+    }
+
+    func testUploadSomeLocalRecords_nobatch_ok() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let storage = MockStorage()
+        let buffer = MockBuffer()
+
+        let records = createMobileRootAndChildrenRecords(numChildren: 80)
+
+        var numPosts = 0
+        let uploader: BatchUploadFunction = { lines, ifUnmodifiedSince, queryParams in
+            numPosts += 1
+            var headers = [String: Any]()
+            headers["X-Last-Modified"] = Date.now()
+
+            let result = POSTResult(success: records.childrenRecords.map { $0.id } + ["mobile"], failed: [:])
+            let response = StorageResponse<POSTResult>(value: result, metadata: ResponseMetadata(status: 200, headers: headers))
+            return deferMaybe(response)
+        }
+
+        let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 100, maxPostBytes: 1_048_576, maxTotalRecords: 250, maxTotalBytes: 104_857_600)
+        let collectionClient = MockSyncCollectionClient(uploader: uploader, infoConfig: miniConfig, collection: "bookmarks", encrypter: getBookmarksEncrypter())
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: collectionClient, basePrefs: prefs, collection: "bookmarks", statsSession: SyncEngineStatsSession(collection: "bookmarks"))
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        synchronizer.uploadSomeLocalRecords(storage, mirrorer, collectionClient, mobileRootRecord: records.mobileRootRecord, childrenRecords: records.childrenRecords).succeeded()
+        XCTAssertEqual(storage.lastBufferUpdatedCompletionOpApplied?.bufferValuesToMoveFromLocal.count, 81)
+        XCTAssertEqual(numPosts, 1)
+    }
+
+    func testUploadSomeLocalRecords_nobatch_someFailed() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let storage = MockStorage()
+        let buffer = MockBuffer()
+
+        let records = createMobileRootAndChildrenRecords(numChildren: 80)
+
+        var numPosts = 0
+        let uploader: BatchUploadFunction = { lines, ifUnmodifiedSince, queryParams in
+            numPosts += 1
+            var headers = [String: Any]()
+            headers["X-Last-Modified"] = Date.now()
+
+            let result = POSTResult(success: records.childrenRecords.map { $0.id }.dropFirst(20) + ["mobile"], failed: [:])
+            let response = StorageResponse<POSTResult>(value: result, metadata: ResponseMetadata(status: 200, headers: headers))
+            return deferMaybe(response)
+        }
+
+        let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 100, maxPostBytes: 1_048_576, maxTotalRecords: 250, maxTotalBytes: 104_857_600)
+        let collectionClient = MockSyncCollectionClient(uploader: uploader, infoConfig: miniConfig, collection: "bookmarks", encrypter: getBookmarksEncrypter())
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: collectionClient, basePrefs: prefs, collection: "bookmarks", statsSession: SyncEngineStatsSession(collection: "bookmarks"))
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        synchronizer.uploadSomeLocalRecords(storage, mirrorer, collectionClient, mobileRootRecord: records.mobileRootRecord, childrenRecords: records.childrenRecords).succeeded()
+        XCTAssertEqual(storage.lastBufferUpdatedCompletionOpApplied?.bufferValuesToMoveFromLocal.count, 61)
+        XCTAssertEqual(numPosts, 1)
+    }
+
+    func testUploadSomeLocalRecords_tooManyRecords() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let storage = MockStorage()
+        let buffer = MockBuffer()
+
+        let uploader: BatchUploadFunction = { _, _, _ in
+            // Should never happen
+            XCTFail()
+            return deferMaybe(NSError())
+        }
+
+        let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 100, maxPostBytes: 1_048_576, maxTotalRecords: 250, maxTotalBytes: 104_857_600)
+        let collectionClient = MockSyncCollectionClient(uploader: uploader, infoConfig: miniConfig, collection: "bookmarks", encrypter: getBookmarksEncrypter())
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: collectionClient, basePrefs: prefs, collection: "bookmarks", statsSession: SyncEngineStatsSession(collection: "bookmarks"))
+
+        let records = createMobileRootAndChildrenRecords(numChildren: 120)
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        let error = synchronizer.uploadSomeLocalRecords(storage, mirrorer, collectionClient, mobileRootRecord: records.mobileRootRecord, childrenRecords: records.childrenRecords).value.failureValue!
+        XCTAssertTrue(error is TooManyRecordsError)
+    }
+
+    func testUploadSomeLocalRecords_tooBig() {
+        let delegate = MockSyncDelegate()
+        let prefs = MockProfilePrefs()
+        let scratchpad = Scratchpad(b: KeyBundle.random(), persistingTo: prefs)
+
+        let storage = MockStorage()
+        let buffer = MockBuffer()
+
+        let uploader: BatchUploadFunction = { _, _, _ in
+            // Should never happen
+            XCTFail()
+            return deferMaybe(NSError())
+        }
+
+        let miniConfig = InfoConfiguration(maxRequestBytes: 1_048_576, maxPostRecords: 100, maxPostBytes: 5000, maxTotalRecords: 250, maxTotalBytes: 104_857_600)
+        let collectionClient = MockSyncCollectionClient(uploader: uploader, infoConfig: miniConfig, collection: "bookmarks", encrypter: getBookmarksEncrypter())
+        let mirrorer = BookmarksMirrorer(storage: buffer, client: collectionClient, basePrefs: prefs, collection: "bookmarks", statsSession: SyncEngineStatsSession(collection: "bookmarks"))
+
+        let records = createMobileRootAndChildrenRecords(numChildren: 80)
+
+        let synchronizer = BufferingBookmarksSynchronizer(scratchpad: scratchpad, delegate: delegate, basePrefs: prefs, why: .scheduled)
+        let error = synchronizer.uploadSomeLocalRecords(storage, mirrorer, collectionClient, mobileRootRecord: records.mobileRootRecord, childrenRecords: records.childrenRecords).value.failureValue!
+        XCTAssertTrue(error is TooManyRecordsError)
+    }
+
+    func getBookmarksEncrypter() -> RecordEncrypter<BookmarkBasePayload> {
+        let serializer: (Record<BookmarkBasePayload>) -> JSON? = { $0.payload.json }
+        let factory: (String) -> BookmarkBasePayload = { BookmarkBasePayload($0) }
+        return RecordEncrypter(serializer: serializer, factory: factory)
+    }
+
+    func createMobileRootAndChildrenRecords(numChildren: Int) -> (mobileRootRecord: Record<BookmarkBasePayload>, childrenRecords: [Record<BookmarkBasePayload>]) {
+        var childrenRecords: [Record<BookmarkBasePayload>] = []
+        for i in 0...numChildren {
+            let item = BookmarkMirrorItem.bookmark("bk1\(i)", dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.MobileFolderGUID, parentName: nil, title: "bk1\(i)", description: nil, URI: "https://example.com/\(i)", tags: "", keyword: nil)
+            let record = Record<BookmarkBasePayload>(id: item.guid, payload: item.asPayload())
+            childrenRecords.append(record)
+        }
+        let mobileRoot = BookmarkMirrorItem.folder(BookmarkRoots.MobileFolderGUID, dateAdded: Date.now(), modified: Date.now(), hasDupe: false, parentID: BookmarkRoots.RootGUID, parentName: nil, title: "Mobile Bookmarks", description: nil, children: [])
+        let mobileRootRecord = Record<BookmarkBasePayload>(id: BookmarkRoots.translateOutgoingRootGUID(BookmarkRoots.MobileFolderGUID), payload: mobileRoot.asPayloadWithChildren(childrenRecords.map { $0.id }))
+
+        return (mobileRootRecord: mobileRootRecord, childrenRecords: childrenRecords)
+    }
+}