wuggy/Dactyloidae
wuggy/Dactyloidae
1
0
Fork 0
mirror of https://repo.dactyloidae.xyz/Dactyloidae/UXP.git synced 2026-09-22 08:27:31 +09:00
Code Issues Projects Releases Packages Wiki Activity

Dactyloidae iOS initial commit

Browse source
This commit is contained in:
wuggy 2026-06-26 21:04:09 -07:00
commit 7154a0497e
2123 changed files with 197052 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

161
mobile/ios/Sync/Synchronizers/Bookmarks/BookmarksDownloader.swift Normal file
View file

@ -0,0 +1,161 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
private let log = Logger.syncLogger
let BookmarksStorageVersion = 2
/**
* This is like a synchronizer, but it downloads records bit by bit, eventually
* notifying that the local storage is up to date with the server contents.
*
* Because batches might be separated over time, it's possible for the server
* state to change between calls. These state changes might include:
*
* 1. New changes arriving. This is fairly routine, but it's worth noting that
* changes might affect existing records that have been batched!
* 2. Wipes. The collection (or the server as a whole) might be deleted. This
* should be accompanied by a change in syncID in meta/global; it's the caller's
* responsibility to detect this.
* 3. A storage format change. This should be unfathomably rare, but if it happens
* we must also be prepared to discard our existing batched data.
* 4. TTL expiry. We need to do better about TTL handling in general, but here
* we might find that a downloaded record is no longer live by the time we
* come to apply it! This doesn't apply to bookmark records, so we will ignore
* it for the moment.
*
* Batch downloading without continuation tokens is achieved as follows:
*
* * A minimum timestamp is established. This starts as zero.
* * A fetch is issued against the server for records changed since that timestamp,
* ordered by modified time ascending, and limited to the batch size.
* * If the batch is complete, we flush it to storage and advance the minimum
* timestamp to just before the newest record in the batch. This ensures that
* a divided set of records with the same modified times will be downloaded
* entirely so long as the set is never larger than the batch size.
* * Iterate until we determine that there are no new records to fetch.
*
* Batch downloading with continuation tokens is much easier:
*
* * A minimum timestamp is established.
* * Make a request with limit=N.
* * Look for an X-Weave-Next-Offset header. Supply that in the next request.
* Also supply X-If-Unmodified-Since to avoid missed modifications.
*
* We do the latter, because we only support Sync 1.5. The use of the offset
* allows us to efficiently process batches, particularly those that contain
* large sets of records with the same timestamp. We still maintain the last
* modified timestamp to allow for resuming a batch in the case of a conflicting
* write, detected via X-I-U-S.
*/
public class BookmarksMirrorer {
private let downloader: BatchingDownloader<BookmarkBasePayload>
private let storage: BookmarkBufferStorage
private let batchSize: Int
private let statsSession: SyncEngineStatsSession
public init(storage: BookmarkBufferStorage, client: Sync15CollectionClient<BookmarkBasePayload>, basePrefs: Prefs, collection: String, statsSession: SyncEngineStatsSession, batchSize: Int=100) {
self.storage = storage
self.downloader = BatchingDownloader(collectionClient: client, basePrefs: basePrefs, collection: collection)
self.batchSize = batchSize
self.statsSession = statsSession
}
var lastModified: Timestamp {
get {
return self.downloader.lastModified
}
}
// TODO
public func storageFormatDidChange() {
}
// TODO
public func onWipeWasAppliedToStorage() {
}
private func applyRecordsFromBatcher() -> Success {
let retrieved = self.downloader.retrieve()
let invalid = retrieved.filter { !$0.payload.isValid() }
if !invalid.isEmpty {
// There's nothing we can do with invalid input. There's also no point in
// tracking failing GUIDs here yet: if another client reuploads those records
// correctly, we'll encounter them routinely due to a newer timestamp.
// The only improvement we could make is to drop records from the buffer if we
// happen to see a new, invalid one before somehow syncing again, but that's
// unlikely enough that it's not worth doing.
//
// Bug 1258801 tracks recording telemetry for these invalid items, which is
// why we don't simply drop them on the ground at the download stage.
//
// We might also choose to perform certain simple recovery actions here: for example,
// bookmarks with null URIs are clearly invalid, and could be treated as if they
// weren't present on the server, or transparently deleted.
log.warning("Invalid records: \(invalid.map { $0.id }.joined(separator: ", ")).")
}
let mirrorItems = retrieved.flatMap { record -> BookmarkMirrorItem? in
guard record.payload.isValid() else {
return nil
}
return (record.payload as MirrorItemable).toMirrorItem(record.modified)
}
if mirrorItems.isEmpty {
log.debug("Got empty batch.")
return succeed()
}
log.debug("Applying \(mirrorItems.count) downloaded bookmarks.")
return self.storage.applyRecords(mirrorItems)
}
public func go(info: InfoCollections, greenLight: @escaping () -> Bool) -> SyncResult {
if !greenLight() {
log.info("Green light turned red. Stopping mirror operation.")
return deferMaybe(SyncStatus.notStarted(.redLight))
}
log.debug("Downloading up to \(self.batchSize) records.")
return self.downloader.go(info, limit: self.batchSize)
.bind { result in
guard let end = result.successValue else {
log.warning("Got failure: \(result.failureValue!)")
return deferMaybe(result.failureValue!)
}
switch end {
case .complete:
log.info("Done with batched mirroring.")
return self.applyRecordsFromBatcher()
>>> effect(self.downloader.advance)
>>> self.storage.doneApplyingRecordsAfterDownload
>>> always(SyncStatus.completed(self.statsSession.end()))
case .incomplete:
log.debug("Running another batch.")
// This recursion is fine because Deferred always pushes callbacks onto a queue.
return self.applyRecordsFromBatcher()
>>> effect(self.downloader.advance)
>>> { self.go(info: info, greenLight: greenLight) }
case .interrupted:
log.info("Interrupted. Aborting batching this time.")
return deferMaybe(SyncStatus.partial(self.statsSession))
case .noNewData:
log.info("No new data. No need to continue batching.")
self.downloader.advance()
return deferMaybe(SyncStatus.completed(self.statsSession.end()))
}
}
}
func advanceNextDownloadTimestampTo(timestamp: Timestamp) {
self.downloader.advanceTimestampTo(timestamp)
}
}

571
mobile/ios/Sync/Synchronizers/Bookmarks/BookmarksRepairRequestor.swift Normal file
View file

@ -0,0 +1,571 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import Deferred
import SwiftyJSON
import SyncTelemetry
private let log = Logger.syncLogger
// How long should we wait after sending a repair request before we give up?
private let ResponseIntervalTimeout = OneDayInMilliseconds * 3
// The maximum number of IDs we will request to be repaired. Beyond this
// number we assume that trying to repair may do more harm than good and may
// ask another client to wipe the server and reupload everything. Bug 1341972
// is tracking that work for Desktop.
private let MaxRequestedIDs = 1000
// If a repair is in progress, this is the generated GUID for the "flow ID".
private let PrefFlowID = "flowID"
// The IDs we are currently trying to obtain via the repair process.
private let PrefMissingIDs = "ids"
// The ID of the client we're currently trying to get the missing items from.
private let PrefCurrentClient = "currentClient"
// The IDs of the clients we've previously tried to get the missing items
// from.
private let PrefPreviousClients = "previousClients"
// The time, in seconds, when we initiated the most recent client request.
private let PrefLastRepair = "when"
// Our current state.
private let PrefCurrentState = "state"
private enum RepairState: String {
// We have not started the repair process.
case notRepairing = ""
// We need to try to find another client to use.
case needNewClient = "repair.need-new-client"
// We've sent the first request to a client.
case sentRequest = "repair.sent"
// We've retried a request to a client.
case sentSecondRequest = "repair.sent-again"
// There were no problems, but we've gone as far as we can.
case finished = "repair.finished"
// We've found an error that forces us to abort this entire repair cycle.
case aborted = "repair.aborted"
}
struct RepairResponse {
let collection: String
let request: String
let flowID: String
let clientID: String
let ids: [String]
static func fromJSON(args: JSON) -> RepairResponse {
return RepairResponse(collection: args["collection"].stringValue, request: args["request"].stringValue,
flowID: args["flowID"].stringValue, clientID: args["clientID"].stringValue,
ids: args["ids"].arrayValue.map { $0.stringValue })
}
}
struct RepairRequest {
let collection: String
let request: String
let flowID: String
let requestor: String
let ids: [String]
func toSyncCommand() -> SyncCommand {
let jsonObj: [String: Any] = [
"command": "repairRequest",
"args": [
[
"collection": collection,
"request": request,
"flowID": flowID,
"requestor": requestor,
"ids": ids
]
]
]
return SyncCommand(value: JSON(object: jsonObj).stringValue()!)
}
}
private class AbortRepairError: MaybeErrorType {
let description: String
init(_ description: String) {
self.description = description
}
}
private class UnknownClientError: MaybeErrorType {
let description: String
init(_ description: String) {
self.description = description
}
}
private class InvalidStateError: MaybeErrorType {
let description: String
init(_ description: String) {
self.description = description
}
}
class BookmarksRepairRequestor {
let prefs: Prefs
let basePrefs: Prefs
let remoteClients: RemoteClientsAndTabs
let scratchpad: Scratchpad
init(scratchpad: Scratchpad, basePrefs: Prefs, remoteClients: RemoteClientsAndTabs) {
self.scratchpad = scratchpad
self.basePrefs = basePrefs
self.prefs = basePrefs.branch("repairs.bookmark")
self.remoteClients = remoteClients
}
/**
* See if the repairer is willing and able to begin a repair process given
* the specified validation information.
*
* - returns: true if a repair was started and false otherwise.
*/
func startRepairs(validationInfo: [BufferInconsistency: [GUID]], flowID: String = Bytes.generateGUID()) -> Deferred<Maybe<Bool>> {
guard self.currentState == .notRepairing else {
log.info("Can't start a repair - repair with ID \(self.flowID) is already in progress")
return deferMaybe(false)
}
let ids = self.getProblemIDs(validationInfo)
guard ids.count > 0 else {
log.info("Not starting a repair as there are no problems")
return deferMaybe(false)
}
guard ids.count <= MaxRequestedIDs else {
log.info("Not starting a repair as there are over \(MaxRequestedIDs) problems")
let extra = [
"flowID": flowID,
"reason": "too many problems: \(ids.count)"
]
let event = Event(category: "sync", method: "repair", object: "aborted", extra: extra)
recordTelemetry(event: event)
return deferMaybe(false)
}
return self.anyClientsRepairing() >>== { clientsRepairing in
guard !clientsRepairing else {
log.info("Can't start repair, since other clients are already repairing bookmarks")
let extra = [
"flowID": flowID,
"reason": "other clients repairing"
]
let event = Event(category: "sync", method: "repair", object: "aborted", extra: extra)
self.recordTelemetry(event: event)
return deferMaybe(false)
}
log.info("Starting a repair, looking for \(ids.count) missing item(s)")
// Setup our prefs to indicate we are on our way.
self.flowID = flowID
self.currentIDs = ids
self.currentState = .needNewClient
let extra = ["flowID": flowID, "numIDs": String(ids.count)]
let event = Event(category: "sync", method: "repair", object: "started", extra: extra)
self.recordTelemetry(event: event)
return self.continueRepairs()
}
}
/**
* Work out what state our current repair request is in, and whether it can
* proceed to a new state.
*
* - returns: true if we could continue the repair - even if the state didn't
* actually move. Returns false if we aren't actually repairing.
*/
func continueRepairs(response: RepairResponse? = nil) -> Deferred<Maybe<Bool>> {
// Note that "aborted" and "finished" should never be current when this
// function returns - this function resets to notRepairing in those cases.
guard self.currentState != .notRepairing else {
return deferMaybe(false)
}
var abortReason: String?
func runStateMachine(iteration: Int = 0) -> Deferred<Maybe<(state: RepairState, newState: RepairState)>> {
let state = self.currentState
log.info("continueRepairs starting with state \(state)")
return self.advanceRepairState(state: state, response: response).bind { result in
let newState: RepairState
if result.isSuccess {
newState = result.successValue!
log.info("continueRepairs has next state \(newState)")
} else {
let failure = result.failureValue!
if failure is AbortRepairError {
let reason = failure.description
log.info("Repair has been aborted: \(reason)")
newState = .aborted
abortReason = reason
} else {
return deferMaybe(failure)
}
}
let done = deferMaybe((state: state, newState: newState))
if newState == .aborted {
return done
}
self.currentState = newState
if state == newState {
return done
}
// we loop until the state doesn't change - but enforce a max of 10 times
// to prevent errors causing infinite loops.
return (iteration < 10) ? runStateMachine(iteration: iteration + 1) : done
}
}
return runStateMachine() >>== { stateMachineResult in
let state = stateMachineResult.state
let newState = stateMachineResult.newState
if state != newState {
log.error("continueRepairs spun without getting a new state")
}
if newState == .finished || newState == .aborted {
let method = newState == .finished ? "finished" : "aborted"
var extra = [
"flowID": self.flowID,
"numIDs": String(self.currentIDs.count),
]
if abortReason != nil {
extra["reason"] = abortReason
}
let event = Event(category: "sync", method: "repair", object: method, extra: extra)
self.recordTelemetry(event: event)
self.prefs.clearAll()
}
return deferMaybe(true)
}
}
func recordTelemetry(event: Event) {
var events = self.basePrefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? []
if let data = event.pickle(), event.validate() {
events.append(data)
self.basePrefs.setObject(events, forKey: PrefKeySyncEvents)
} else {
log.info("Event not recorded due to validation failure or pickling error!")
}
}
private func advanceRepairState(state: RepairState, response: RepairResponse?) -> Deferred<Maybe<RepairState>> {
return self.anyClientsRepairing(flowID: self.flowID) >>== { anyClientsRepairing in
guard !anyClientsRepairing else {
return deferMaybe(AbortRepairError("other clients repairing"))
}
switch state {
case .sentRequest, .sentSecondRequest:
let flowID = self.flowID
guard let clientID = self.currentRemoteClient else {
return deferMaybe(InvalidStateError("currentRemoteClient should be defined"))
}
if let response = response {
// We got an explicit response - let's see how we went.
return deferMaybe(self.handleResponse(state: state, response: response))
}
// So we've sent a request - and don't yet have a response. See if the
// client we sent it to has removed it from its list (ie, whether it
// has synced since we wrote the request.)
return self.remoteClients.getClientWithId(clientID) >>== { client in
guard let client = client else {
// hrmph - the client has disappeared.
log.info("previously requested client \(clientID) has vanished - moving to next step")
let extra = [
"deviceID": self.scratchpad.hashedDeviceID ?? "unknown_deviceID",
"flowID": flowID
]
let event = Event(category: "sync", method: "repair", object: "abandon", value: "missing", extra: extra)
self.recordTelemetry(event: event)
return deferMaybe(.needNewClient)
}
return self.isCommandPending(clientID: clientID, flowID: flowID) >>== { isCommandPending in
if isCommandPending {
// So the command we previously sent is still queued for the client
// (ie, that client is yet to have synced). Let's see if we should
// give up on that client.
if self.lastRepair + ResponseIntervalTimeout <= Date.now() {
log.info("previous request to client \(clientID) is pending, but has taken too long")
// XXX - should we remove the command?
let extra = [
"deviceID": self.scratchpad.hashedDeviceID ?? "unknown_deviceID",
"flowID": flowID
]
let event = Event(category: "sync", method: "repair", object: "abandon", value: "silent", extra: extra)
self.recordTelemetry(event: event)
return deferMaybe(.needNewClient)
}
// Let's continue to wait for that client to respond.
// We are now sure that timeLeft > 0, so we can calculate it (Timestamp type is UInt64)
let timeLeft = self.lastRepair + ResponseIntervalTimeout - Date.now()
log.verbose("previous request to client \(clientID) has \(timeLeft) seconds before we give up on it")
return deferMaybe(state)
}
// The command isn't pending - if this was the first request, we give
// it another go (as that client may have cleared the command but is yet
// to complete the sync)
// XXX - note that this is no longer true - the responders don't remove
// their command until they have written a response. This might mean
// we could drop the entire STATE.SENT_SECOND_REQUEST concept???
if state == .sentRequest {
log.info("previous request to client \(clientID) was removed - trying a second time")
return self.writeRequest(client: client) >>== { success in
return deferMaybe(.sentSecondRequest)
}
} else {
// this was the second time around, so give up on this client
log.info("previous 2 requests to client \(clientID) were removed - need a new client")
return deferMaybe(.needNewClient)
}
}
}
case .needNewClient:
// We need to find a new client to request.
return self.findNextClient() >>== { client in
guard let nextClient = client else {
return deferMaybe(.finished)
}
if let currentRemoteClient = self.currentRemoteClient {
var previousRemoteClients = self.previousRemoteClients
previousRemoteClients.append(currentRemoteClient)
self.previousRemoteClients = previousRemoteClients
}
self.currentRemoteClient = nextClient.guid!
return self.writeRequest(client: nextClient) >>== { success in
return deferMaybe(.sentRequest)
}
}
case .aborted:
break // our caller will take the abort action.
case .finished:
break
case .notRepairing:
// No repair is in progress. This is a common case, so only log trace.
log.verbose("continue repairs called but no repair in progress.")
break
}
return deferMaybe(state)
}
}
/**
* Handle being in the SENT_REQUEST or SENT_SECOND_REQUEST state with an
* explicit response.
*/
private func handleResponse(state: RepairState, response: RepairResponse) -> RepairState {
guard let clientID = self.currentRemoteClient else {
log.error("Cannot handle the response of an unknown client")
return state
}
let flowID = self.flowID
guard response.flowID == flowID && response.clientID == clientID &&
response.request == "upload" else {
log.info("got a response to a different repair request: \(response)")
// hopefully just a stale request that finally came in (either from
// an entirely different repair flow, or from a client we've since
// given up on.) It doesn't mean we need to abort though...
return state
}
// Pull apart the response and see if it provided everything we asked for.
let remainingIDs = Array(Set(self.currentIDs).subtracting(Set(response.ids)))
log.info("repair response from \(clientID) provided '\(response.ids)', remaining now '\(remainingIDs)'")
self.currentIDs = remainingIDs
let newState: RepairState
if remainingIDs.count > 0 {
// try a new client for the remaining ones.
newState = .needNewClient
} else {
newState = .finished
}
// record telemetry about this
let extra = [
"deviceID": scratchpad.hashedDeviceID ?? "unknown_deviceID",
"flowID": flowID,
"numIDs": String(response.ids.count)
]
let event = Event(category: "sync", method: "repair", object: "response", value: "upload", extra: extra)
recordTelemetry(event: event)
return newState
}
/**
* Issue a repair request to a specific client.
*/
private func writeRequest(client: RemoteClient) -> Success {
log.verbose("writing repair request to client \(client.guid!)")
let ids = self.currentIDs
let flowID = self.flowID
// Post a command to that client.
let request = RepairRequest(collection: "bookmarks", request: "upload", flowID: flowID, requestor: self.scratchpad.clientGUID, ids: ids)
return self.remoteClients.insertCommand(request.toSyncCommand(), forClients: [client]) >>== { (_: Int) -> Success in
self.lastRepair = Date.now()
// record telemetry about this
let extra = [
"deviceID": self.scratchpad.hashedDeviceID ?? "unknown_deviceID",
"flowID": flowID,
"numIDs": String(ids.count),
]
let event = Event(category: "sync", method: "repair", object: "request", value: "upload", extra: extra)
self.recordTelemetry(event: event)
return succeed()
}
}
private func findNextClient() -> Deferred<Maybe<RemoteClient?>> {
var alreadyDone = self.previousRemoteClients
if let currentRemoteClient = self.currentRemoteClient {
alreadyDone.append(currentRemoteClient)
}
return self.remoteClients.getClients() >>== { remoteClients in
// we want to consider the most-recently synced clients first.
let sortedClients = remoteClients.sorted(by: { (a, b) -> Bool in
return a.modified > b.modified
})
for client in sortedClients {
log.verbose("findNextClient considering \(client)")
guard let clientID = client.guid else {
continue
}
if !alreadyDone.contains(clientID) && self.isSuitableClient(client) {
return deferMaybe(client)
}
}
log.verbose("findNextClient found no client")
return deferMaybe(nil)
}
}
private func isSuitableClient(_ client: RemoteClient) -> Bool {
if let type = client.type,
let version = client.version,
let major = Int(version.components(separatedBy: ".")[0]) {
return type == "desktop" && major > 53
}
return false
}
private func getProblemIDs(_ validations: [BufferInconsistency: [GUID]]) -> [GUID] {
return validations.reduce([]) { acc, pair in acc + pair.value }
}
private func anyClientsRepairing(flowID: String? = nil) -> Deferred<Maybe<Bool>> {
return self.remoteClients.getCommands() >>== { allCommands in
return deferMaybe(allCommands.contains { (clientID: GUID, commands: [SyncCommand]) in
return commands.contains { (command: SyncCommand) in
let json = JSON(parseJSON: command.value)
guard let cmdName = json["command"].string,
let argsArray = json["args"].array,
let argObj = argsArray[0].dictionary,
let argCol = argObj["collection"]?.string,
let argFlowID = argObj["flowID"]?.string
else {
return false
}
return !((cmdName != "repairResponse" && cmdName != "repairRequest") ||
argsArray.count != 1 ||
argCol != "bookmarks" ||
argFlowID == flowID
)
}
})
}
}
private func isCommandPending(clientID: String, flowID: String) -> Deferred<Maybe<Bool>> {
return self.remoteClients.getCommands() >>== { allCommands in
guard let commands = allCommands[clientID] else {
return deferMaybe(false)
}
return deferMaybe(commands.contains { (command: SyncCommand) in
let json = JSON(parseJSON: command.value)
guard let cmdName = json["command"].string,
let argsArray = json["args"].array,
let argObj = argsArray[0].dictionary,
let argCol = argObj["collection"]?.string,
let argFlowID = argObj["flowID"]?.string,
let argRequest = argObj["request"]?.string
else {
return false
}
return cmdName == "repairRequest" && argsArray.count == 1 &&
argCol == "bookmarks" && argRequest == "upload" &&
argFlowID == flowID
})
}
}
private var flowID: String {
get { return self.prefs.stringForKey(PrefFlowID)! }
set { self.prefs.setString(newValue, forKey: PrefFlowID) }
}
private var currentIDs: [String] {
get { return self.prefs.stringArrayForKey(PrefMissingIDs) ?? [String]() }
set { self.prefs.setObject(newValue, forKey: PrefMissingIDs) }
}
private var currentRemoteClient: String? {
get { return self.prefs.stringForKey(PrefCurrentClient) }
set { self.prefs.setString(newValue!, forKey: PrefCurrentClient) }
}
private var previousRemoteClients: [String] {
get { return self.prefs.stringArrayForKey(PrefPreviousClients) ?? [String]() }
set { self.prefs.setObject(newValue, forKey: PrefPreviousClients) }
}
private var lastRepair: Timestamp {
get { return self.prefs.timestampForKey(PrefLastRepair)! }
set { self.prefs.setTimestamp(newValue, forKey: PrefLastRepair) }
}
private var currentState: RepairState {
get {
let Default = RepairState.notRepairing
guard let raw = self.prefs.stringForKey(PrefCurrentState) else {
return Default
}
return RepairState(rawValue: raw) ?? Default
}
set { self.prefs.setString(newValue.rawValue, forKey: PrefCurrentState) }
}
}

511
mobile/ios/Sync/Synchronizers/Bookmarks/BookmarksSynchronizer.swift Normal file
View file

@ -0,0 +1,511 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Deferred
import Foundation
import Shared
import Storage
import XCGLogger
private let log = Logger.syncLogger
typealias UploadFunction = ([Record<BookmarkBasePayload>], _ lastTimestamp: Timestamp?, _ onUpload: @escaping (POSTResult, Timestamp?) -> DeferredTimestamp) -> DeferredTimestamp
class TrivialBookmarkStorer: BookmarkStorer {
let uploader: UploadFunction
init(uploader: @escaping UploadFunction) {
self.uploader = uploader
}
func applyUpstreamCompletionOp(_ op: UpstreamCompletionOp, itemSources: ItemSources, trackingTimesInto local: LocalOverrideCompletionOp) -> Deferred<Maybe<POSTResult>> {
log.debug("Uploading \(op.records.count) modified records.")
log.debug("Uploading \(op.amendChildrenFromBuffer.count) amended buffer records.")
log.debug("Uploading \(op.amendChildrenFromMirror.count) amended mirror records.")
log.debug("Uploading \(op.amendChildrenFromLocal.count) amended local records.")
var records: [Record<BookmarkBasePayload>] = []
records.reserveCapacity(op.records.count + op.amendChildrenFromBuffer.count + op.amendChildrenFromLocal.count + op.amendChildrenFromMirror.count)
records.append(contentsOf: op.records)
func accumulateFromAmendMap(_ itemsWithNewChildren: [GUID: [GUID]], fetch: ([GUID: [GUID]]) -> Maybe<[GUID: BookmarkMirrorItem]>) throws /* MaybeErrorType */ {
if itemsWithNewChildren.isEmpty {
return
}
let fetched = fetch(itemsWithNewChildren)
guard let items = fetched.successValue else {
log.warning("Couldn't fetch items to amend.")
throw fetched.failureValue!
}
items.forEach { (guid, item) in
let payload = item.asPayloadWithChildren(itemsWithNewChildren[guid])
let mappedGUID = payload["id"].string ?? guid
let record = Record<BookmarkBasePayload>(id: mappedGUID, payload: payload)
records.append(record)
}
}
do {
try accumulateFromAmendMap(op.amendChildrenFromBuffer, fetch: { itemSources.buffer.getBufferItemsWithGUIDs($0.keys).value })
try accumulateFromAmendMap(op.amendChildrenFromMirror, fetch: { itemSources.mirror.getMirrorItemsWithGUIDs($0.keys).value })
try accumulateFromAmendMap(op.amendChildrenFromLocal, fetch: { itemSources.local.getLocalItemsWithGUIDs($0.keys).value })
} catch {
return deferMaybe(error as MaybeErrorType)
}
var success: [GUID] = []
var failed: [GUID: String] = [:]
func onUpload(_ result: POSTResult, lastModified: Timestamp?) -> DeferredTimestamp {
success.append(contentsOf: result.success)
result.failed.forEach { guid, message in
failed[guid] = message
}
log.debug("Uploaded records got timestamp \(lastModified ??? "nil").")
let modified = lastModified ?? 0
local.setModifiedTime(modified, guids: result.success)
return deferMaybe(modified)
}
// Chain the last upload timestamp right into our lastFetched timestamp.
// This is what Sync clients tend to do, but we can probably do better.
return uploader(records, op.ifUnmodifiedSince, onUpload)
// As if we uploaded everything in one go.
>>> { deferMaybe(POSTResult(success: success, failed: failed)) }
}
}
open class MalformedRecordError: MaybeErrorType, SyncPingFailureFormattable {
open var description: String {
return "Malformed record."
}
open var failureReasonName: SyncPingFailureReasonName {
return .otherError
}
}
// MARK: - External synchronizer interface.
open class BufferingBookmarksSynchronizer: TimestampedSingleCollectionSynchronizer, Synchronizer {
public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "bookmarks")
}
override var storageVersion: Int {
return BookmarksStorageVersion
}
fileprivate func buildMobileRootRecord(_ local: LocalItemSource, _ buffer: BufferItemSource, additionalChildren: [BookmarkMirrorItem], deletedChildren: [GUID]) -> Deferred<Maybe<Record<BookmarkBasePayload>>> {
let newBookmarkGUIDs = additionalChildren.map { $0.guid }
return buffer.getBufferItemWithGUID(BookmarkRoots.MobileFolderGUID).bind { maybeMobileRoot in
// Update (or create!) the Mobile Root folder with its new children.
if let mobileRoot = maybeMobileRoot.successValue {
return buffer.getBufferChildrenGUIDsForParent(mobileRoot.guid)
.map { $0.map({ (mobileRoot: mobileRoot, children: $0.filter { !deletedChildren.contains($0) } + newBookmarkGUIDs) }) }
} else {
return local.getLocalItemWithGUID(BookmarkRoots.MobileFolderGUID)
.map { $0.map({ (mobileRoot: $0, children: newBookmarkGUIDs) }) }
}
} >>== { (mobileRoot: BookmarkMirrorItem, children: [GUID]) in
let payload = mobileRoot.asPayloadWithChildren(children)
guard let mappedGUID = payload["id"].string else {
return deferMaybe(MalformedRecordError())
}
return deferMaybe(Record<BookmarkBasePayload>(id: mappedGUID, payload: payload))
}
}
func buildMobileRootAndChildrenRecords(_ local: LocalItemSource, _ buffer: BufferItemSource, additionalChildren: [BookmarkMirrorItem], deletedChildren: [GUID]) -> Deferred<Maybe<(mobileRootRecord: Record<BookmarkBasePayload>, childrenRecords: [Record<BookmarkBasePayload>])>> {
let childrenRecords =
additionalChildren.map { bkm -> Record<BookmarkBasePayload> in
let payload = bkm.asPayload()
let mappedGUID = payload["id"].string ?? bkm.guid
return Record<BookmarkBasePayload>(id: mappedGUID, payload: payload)
} +
deletedChildren.map { guid -> Record<BookmarkBasePayload> in
let payload = BookmarkBasePayload.deletedPayload(guid)
let mappedGUID = payload["id"].string ?? guid
return Record<BookmarkBasePayload>(id: mappedGUID, payload: payload)
}
return self.buildMobileRootRecord(local, buffer, additionalChildren: additionalChildren, deletedChildren: deletedChildren) >>== { mobileRootRecord in
return deferMaybe((mobileRootRecord: mobileRootRecord, childrenRecords: childrenRecords))
}
}
func uploadSomeLocalRecords(_ storage: SyncableBookmarks & LocalItemSource & MirrorItemSource, _ mirrorer: BookmarksMirrorer, _ bookmarksClient: Sync15CollectionClient<BookmarkBasePayload>, mobileRootRecord: Record<BookmarkBasePayload>, childrenRecords: [Record<BookmarkBasePayload>]) -> Success {
var newBookmarkGUIDs: [GUID] = []
var deletedBookmarksGUIDs: [GUID] = []
for record in childrenRecords {
// No mutable l-values in Swift :(
if record.payload.deleted {
deletedBookmarksGUIDs.append(record.id)
} else {
newBookmarkGUIDs.append(record.id)
}
}
let records = [mobileRootRecord] + childrenRecords
return self.uploadRecordsSingleBatch(records, lastTimestamp: mirrorer.lastModified, storageClient: bookmarksClient) >>== { (timestamp: Timestamp, succeeded: [GUID]) -> Success in
let bufferValuesToMoveFromLocal = Set(newBookmarkGUIDs).intersection(Set(succeeded))
let deletedValues = Set(deletedBookmarksGUIDs).intersection(Set(succeeded))
let mobileRoot = (mobileRootRecord.payload as MirrorItemable).toMirrorItem(timestamp)
let bufferOP = BufferUpdatedCompletionOp(bufferValuesToMoveFromLocal: bufferValuesToMoveFromLocal, deletedValues: deletedValues, mobileRoot: mobileRoot, modifiedTime: timestamp)
return storage.applyBufferUpdatedCompletionOp(bufferOP) >>> {
mirrorer.advanceNextDownloadTimestampTo(timestamp: timestamp) // We need to advance our batching downloader timestamp to match. See Bug 1253458.
return succeed()
}
}
}
open func synchronizeBookmarksToStorage(_ storage: SyncableBookmarks & LocalItemSource & MirrorItemSource, usingBuffer buffer: BookmarkBufferStorage & BufferItemSource, withServer storageClient: Sync15StorageClient, info: InfoCollections, greenLight: @escaping () -> Bool, remoteClientsAndTabs: RemoteClientsAndTabs) -> SyncResult {
if self.prefs.boolForKey("dateAddedMigrationDone") != true {
self.lastFetched = 0
self.prefs.setBool(true, forKey: "dateAddedMigrationDone")
}
if let reason = self.reasonToNotSync(storageClient) {
return deferMaybe(.notStarted(reason))
}
let encoder = RecordEncoder<BookmarkBasePayload>(decode: BookmarkType.somePayloadFromJSON, encode: { $0.json })
guard let bookmarksClient = self.collectionClient(encoder, storageClient: storageClient) else {
log.error("Couldn't make bookmarks factory.")
return deferMaybe(FatalError(message: "Couldn't make bookmarks factory."))
}
let start = Date.nowMicroseconds()
let mirrorer = BookmarksMirrorer(storage: buffer, client: bookmarksClient, basePrefs: self.prefs, collection: "bookmarks", statsSession: self.statsSession)
let storer = TrivialBookmarkStorer(uploader: { records, lastTimestamp, onUpload in
let timestamp = lastTimestamp ?? self.lastFetched
return self.uploadRecords(records, lastTimestamp: timestamp, storageClient: bookmarksClient, onUpload: onUpload)
>>== effect { timestamp in
// We need to advance our batching downloader timestamp to match. See Bug 1253458.
self.setTimestamp(timestamp)
mirrorer.advanceNextDownloadTimestampTo(timestamp: timestamp)
}
})
statsSession.start()
let doMirror = mirrorer.go(info: info, greenLight: greenLight)
let run: SyncResult
if !AppConstants.shouldMergeBookmarks {
run = doMirror >>== { result -> SyncResult in
// Validate the buffer to report statistics.
if case .completed = result {
log.debug("Validating completed buffer download.")
return buffer.validate().bind { validationResult in
guard let invalidError = validationResult.failureValue as? BufferInvalidError else {
return deferMaybe(result)
}
return buffer.getUpstreamRecordCount().bind { checked -> Success in
self.statsSession.validationStats = self.validationStatsFrom(error: invalidError, checked: checked)
return self.maybeStartRepairProcedure(greenLight: greenLight, error: invalidError, remoteClientsAndTabs: remoteClientsAndTabs)
} >>> {
return deferMaybe(result)
}
}
}
return deferMaybe(result)
} >>== { result in
guard AppConstants.MOZ_SIMPLE_BOOKMARKS_SYNCING else {
return deferMaybe(result)
}
guard case .completed = result else {
return deferMaybe(result)
}
// -1 because we also need to upload the mobile root.
return (storage.getLocalBookmarksModifications(limit: bookmarksClient.maxBatchPostRecords - 1) >>== { (deletedGUIDs, newBookmarks) -> Success in
guard newBookmarks.count > 0 || deletedGUIDs.count > 0 else {
return succeed()
}
return self.buildMobileRootAndChildrenRecords(storage, buffer, additionalChildren: newBookmarks, deletedChildren: deletedGUIDs) >>== { (mobileRootRecord, childrenRecords) in
return self.uploadSomeLocalRecords(storage, mirrorer, bookmarksClient, mobileRootRecord: mobileRootRecord, childrenRecords: childrenRecords)
}
}).bind { simpleSyncingResult in
if let failure = simpleSyncingResult.failureValue {
let description = failure is RecordTooLargeError ? "Record too large" : failure.description
Sentry.shared.send(message: "Failed to simple sync bookmarks", tag: SentryTag.bookmarks, severity: .error, description: description)
}
return deferMaybe(result)
}
}
} else {
run = doMirror >>== { result in
// Only bother trying to sync if the mirror operation wasn't interrupted or partial.
if case .completed = result {
return buffer.validate().bind { result in
if let invalidError = result.failureValue as? BufferInvalidError {
return buffer.getUpstreamRecordCount().bind { checked in
self.statsSession.validationStats = self.validationStatsFrom(error: invalidError, checked: checked)
return self.maybeStartRepairProcedure(greenLight: greenLight, error: invalidError, remoteClientsAndTabs: remoteClientsAndTabs) >>> {
return deferMaybe(invalidError)
}
}
}
let applier = MergeApplier(buffer: buffer, storage: storage, client: storer, statsSession: self.statsSession, greenLight: greenLight)
return applier.go()
}
}
return deferMaybe(result)
}
}
run.upon { result in
let end = Date.nowMicroseconds()
let duration = end - start
log.info("Bookmark \(AppConstants.shouldMergeBookmarks ? "sync" : "mirroring") took \(duration)µs. Result was \(result.successValue?.description ?? result.failureValue?.description ?? "failure")")
}
return run
}
private func validationStatsFrom(error: BufferInvalidError, checked: Int?) -> ValidationStats {
let problems = error.inconsistencies.map { ValidationProblem(name: $0.trackingEvent, count: $1.count) }
return ValidationStats(problems: problems, took: error.validationDuration, checked: checked)
}
private func maybeStartRepairProcedure(greenLight: () -> Bool, error: BufferInvalidError, remoteClientsAndTabs: RemoteClientsAndTabs) -> Success {
guard AppConstants.MOZ_BOOKMARKS_REPAIR_REQUEST && greenLight() else {
return succeed()
}
log.warning("Buffer inconsistent, starting repair procedure")
let repairer = BookmarksRepairRequestor(scratchpad: self.scratchpad, basePrefs: self.basePrefs, remoteClients: remoteClientsAndTabs)
return repairer.startRepairs(validationInfo: error.inconsistencies).bind { result in
if let repairFailure = result.failureValue {
Sentry.shared.send(message: "Bookmarks repair failure", tag: SentryTag.bookmarks, severity: .error, description: repairFailure.description)
} else {
Sentry.shared.send(message: "Bookmarks repair succeeded", tag: SentryTag.bookmarks, severity: .debug)
}
return succeed()
}
}
}
class MergeApplier {
let greenLight: () -> Bool
let buffer: BookmarkBufferStorage
let storage: SyncableBookmarks
let client: BookmarkStorer
let merger: BookmarksStorageMerger
let statsSession: SyncEngineStatsSession
init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource, client: BookmarkStorer, statsSession: SyncEngineStatsSession, greenLight: @escaping () -> Bool) {
self.greenLight = greenLight
self.buffer = buffer
self.storage = storage
self.merger = ThreeWayBookmarksStorageMerger(buffer: buffer, storage: storage)
self.client = client
self.statsSession = statsSession
}
// Exposed for use from tests.
func applyResult(_ result: BookmarksMergeResult) -> Success {
return result.applyToClient(self.client, storage: self.storage, buffer: self.buffer)
}
func go() -> SyncResult {
guard self.greenLight() else {
log.info("Green light turned red; not merging bookmarks.")
return deferMaybe(SyncStatus.completed(statsSession.end()))
}
return self.merger.merge()
>>== self.applyResult
>>> always(SyncStatus.completed(statsSession.end()))
}
}
/**
* The merger takes as input an existing storage state (mirror and local override),
* a buffer of new incoming records that relate to the mirror, and performs a three-way
* merge.
*
* The merge itself does not mutate storage. The result of the merge is conceptually a
* tuple: a new mirror state, a set of reconciled + locally changed records to upload,
* and two checklists of buffer and local state to discard.
*
* Typically the merge will be complete, resulting in a new mirror state, records to
* upload, and completely emptied buffer and local. In the case of partial inconsistency
* this will not be the case; incomplete subtrees will remain in the buffer. (We don't
* expect local subtrees to ever be incomplete.)
*
* It is expected that the caller will immediately apply the result in this order:
*
* 1. Upload the remote changes, if any. If this fails we can retry the entire process.
*
* 2(a). Apply the local changes, if any. If this fails we will re-download the records
* we just uploaded, and should reach the same end state.
* This step takes a timestamp key from (1), because pushing a record into the mirror
* requires a server timestamp.
*
* 2(b). Switch to the new mirror state. If this fails, we should find that our reconciled
* server contents apply neatly to our mirror and empty local, and we'll reach the
* same end state.
*
* Mirror state is applied in a sane order to respect relational constraints, even though
* we configure sqlite to defer constraint validation until the transaction is committed.
* That means:
*
* - Add any new records in the value table.
* - Change any existing records in the value table.
* - Update structure.
* - Remove records from the value table.
*
* 3. Apply buffer changes. We only do this after the mirror has advanced; if we fail to
* clean up the buffer, it'll reconcile neatly with the mirror on a subsequent try.
*
* 4. Update bookkeeping timestamps. If this fails we will download uploaded records,
* find they match, and have no repeat merging work to do.
*
* The goal of merging is that the buffer is empty (because we reconciled conflicts and
* updated the server), our local overlay is empty (because we reconciled conflicts and
* applied our changes to the server), and the mirror matches the server.
*
* Note that upstream application is robust: we can use XIUS to ensure that writes don't
* race. Buffer application is similarly robust, because this code owns all writes to the
* buffer. Local and mirror application, however, is not: the user's actions can cause
* changes to write to the database before we're done applying the results of a sync.
* We mitigate this a little by being explicit about the local changes that we're flushing
* (rather than, say, `DELETE FROM local`), but to do better we'd need change detection
* (e.g., an in-memory monotonic counter) or locking to prevent bookmark operations from
* racing. Later!
*/
protocol BookmarksStorageMerger: class {
init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource)
func merge() -> Deferred<Maybe<BookmarksMergeResult>>
}
class NoOpBookmarksMerger: BookmarksStorageMerger {
let buffer: BookmarkBufferStorage & BufferItemSource
let storage: SyncableBookmarks & LocalItemSource & MirrorItemSource
required init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource) {
self.buffer = buffer
self.storage = storage
}
func merge() -> Deferred<Maybe<BookmarksMergeResult>> {
return deferMaybe(BookmarksMergeResult.NoOp(ItemSources(local: self.storage, mirror: self.storage, buffer: self.buffer)))
}
}
class ThreeWayBookmarksStorageMerger: BookmarksStorageMerger {
let buffer: BookmarkBufferStorage & BufferItemSource
let storage: SyncableBookmarks & LocalItemSource & MirrorItemSource
required init(buffer: BookmarkBufferStorage & BufferItemSource, storage: SyncableBookmarks & LocalItemSource & MirrorItemSource) {
self.buffer = buffer
self.storage = storage
}
// MARK: - BookmarksStorageMerger.
// Trivial one-way sync.
fileprivate func applyLocalDirectlyToMirror() -> Deferred<Maybe<BookmarksMergeResult>> {
// Theoretically, we do the following:
// * Construct a virtual bookmark tree overlaying local on the mirror.
// * Walk the tree to produce Sync records.
// * Upload those records.
// * Flatten that tree into the mirror, clearing local.
//
// This is simpler than a full three-way merge: it's tree delta then flatten.
//
// But we are confident that our local changes, when overlaid on the mirror, are
// consistent. So we can take a little bit of a shortcut: process records
// directly, rather than building a tree.
//
// So do the following:
// * Take everything in `local` and turn it into a Sync record. This means pulling
// folder hierarchies out of localStructure, values out of local, and turning
// them into records. Do so in hierarchical order if we can, and set sortindex
// attributes to put folders first.
// * Upload those records in as few batches as possible. Ensure that each batch
// is consistent, if at all possible, though we're hoping for server support for
// atomic writes.
// * Take everything in local that was successfully uploaded and move it into the
// mirror, using the timestamps we tracked from the upload.
//
// Optionally, set 'again' to true in our response, and do this work only for a
// particular subtree (e.g., a single root, or a single branch of changes). This
// allows us to make incremental progress.
// TODO
log.debug("No special-case local-only merging yet. Falling back to three-way merge.")
return self.threeWayMerge()
}
fileprivate func applyIncomingDirectlyToMirror() -> Deferred<Maybe<BookmarksMergeResult>> {
// If the incoming buffer is consistent -- and the result of the mirrorer
// gives us a hint about that! -- then we can move the buffer records into
// the mirror directly.
//
// Note that this is also true for entire subtrees: if none of the children
// of, say, 'menu________' are modified locally, then we can apply it without
// merging.
//
// TODO
log.debug("No special-case remote-only merging yet. Falling back to three-way merge.")
return self.threeWayMerge()
}
// This is exposed for testing.
func getMerger() -> Deferred<Maybe<ThreeWayTreeMerger>> {
return self.storage.treesForEdges() >>== { (local, remote) in
// At this point *might* have two empty trees. This should only be the case if
// there are value-only changes (e.g., a renamed bookmark).
// We don't fail in that case, but we could optimize here.
// Find the mirror tree so we can compare.
return self.storage.treeForMirror() >>== { mirror in
// At this point we know that there have been changes both locally and remotely.
// (Or, in the general case, changes either locally or remotely.)
let itemSources = ItemSources(local: CachingLocalItemSource(source: self.storage), mirror: CachingMirrorItemSource(source: self.storage), buffer: CachingBufferItemSource(source: self.buffer))
return deferMaybe(ThreeWayTreeMerger(local: local, mirror: mirror, remote: remote, itemSources: itemSources))
}
}
}
func getMergedTree() -> Deferred<Maybe<MergedTree>> {
return self.getMerger() >>== { $0.produceMergedTree() }
}
func threeWayMerge() -> Deferred<Maybe<BookmarksMergeResult>> {
return self.getMerger() >>== { $0.produceMergedTree() >>== $0.produceMergeResultFromMergedTree }
}
func merge() -> Deferred<Maybe<BookmarksMergeResult>> {
return self.buffer.isEmpty() >>== { noIncoming in
// TODO: the presence of empty desktop roots in local storage
// isn't something we really need to worry about. Can we skip it here?
return self.storage.isUnchanged() >>== { noOutgoing in
switch (noIncoming, noOutgoing) {
case (true, true):
// Nothing to do!
log.debug("No incoming and no outgoing records: no-op.")
return deferMaybe(BookmarksMergeResult.NoOp(ItemSources(local: self.storage, mirror: self.storage, buffer: self.buffer)))
case (true, false):
// No incoming records to apply. Unilaterally apply local changes.
return self.applyLocalDirectlyToMirror()
case (false, true):
// No outgoing changes. Unilaterally apply remote changes if they're consistent.
return self.buffer.validate() >>> self.applyIncomingDirectlyToMirror
default:
// Changes on both sides. Merge.
return self.buffer.validate() >>> self.threeWayMerge
}
}
}
}
}

336
mobile/ios/Sync/Synchronizers/Bookmarks/Merging.swift Normal file
View file

@ -0,0 +1,336 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Deferred
import Foundation
import Shared
import Storage
import XCGLogger
private let log = Logger.syncLogger
// Because generic protocols in Swift are a pain in the ass.
public protocol BookmarkStorer: class {
// TODO: this should probably return a timestamp.
func applyUpstreamCompletionOp(_ op: UpstreamCompletionOp, itemSources: ItemSources, trackingTimesInto local: LocalOverrideCompletionOp) -> Deferred<Maybe<POSTResult>>
}
open class UpstreamCompletionOp: PerhapsNoOp {
// Upload these records from the buffer, but with these child lists.
open var amendChildrenFromBuffer: [GUID: [GUID]] = [:]
// Upload these records from the mirror, but with these child lists.
open var amendChildrenFromMirror: [GUID: [GUID]] = [:]
// Upload these records from local, but with these child lists.
open var amendChildrenFromLocal: [GUID: [GUID]] = [:]
// Upload these records as-is.
open var records: [Record<BookmarkBasePayload>] = []
open let ifUnmodifiedSince: Timestamp?
open var isNoOp: Bool {
return records.isEmpty
}
public init(ifUnmodifiedSince: Timestamp?=nil) {
self.ifUnmodifiedSince = ifUnmodifiedSince
}
}
open class BookmarksMergeResult: PerhapsNoOp {
let uploadCompletion: UpstreamCompletionOp
let overrideCompletion: LocalOverrideCompletionOp
let bufferCompletion: BufferCompletionOp
let itemSources: ItemSources
open var isNoOp: Bool {
return self.uploadCompletion.isNoOp &&
self.overrideCompletion.isNoOp &&
self.bufferCompletion.isNoOp
}
func applyToClient(_ client: BookmarkStorer, storage: SyncableBookmarks, buffer: BookmarkBufferStorage) -> Success {
return client.applyUpstreamCompletionOp(self.uploadCompletion, itemSources: self.itemSources, trackingTimesInto: self.overrideCompletion)
>>> { storage.applyLocalOverrideCompletionOp(self.overrideCompletion, itemSources: self.itemSources) }
>>> { buffer.applyBufferCompletionOp(self.bufferCompletion, itemSources: self.itemSources) }
}
init(uploadCompletion: UpstreamCompletionOp, overrideCompletion: LocalOverrideCompletionOp, bufferCompletion: BufferCompletionOp, itemSources: ItemSources) {
self.uploadCompletion = uploadCompletion
self.overrideCompletion = overrideCompletion
self.bufferCompletion = bufferCompletion
self.itemSources = itemSources
}
static func NoOp(_ itemSources: ItemSources) -> BookmarksMergeResult {
return BookmarksMergeResult(uploadCompletion: UpstreamCompletionOp(), overrideCompletion: LocalOverrideCompletionOp(), bufferCompletion: BufferCompletionOp(), itemSources: itemSources)
}
}
// MARK: - Errors.
open class BookmarksMergeError: MaybeErrorType, SyncPingFailureFormattable {
fileprivate let error: Error?
init(error: Error?=nil) {
self.error = error
}
open var description: String {
return "Merge error: \(self.error ??? "nil")"
}
open var failureReasonName: SyncPingFailureReasonName {
return .otherError
}
}
open class BookmarksMergeConsistencyError: BookmarksMergeError {
override open var description: String {
return "Merge consistency error"
}
}
open class BookmarksMergeErrorTreeIsUnrooted: BookmarksMergeConsistencyError {
open let roots: Set<GUID>
public init(roots: Set<GUID>) {
self.roots = roots
}
override open var description: String {
return "Tree is unrooted: roots are \(self.roots)"
}
}
enum MergeState<T> {
case unknown // Default state.
case unchanged // Nothing changed: no work needed.
case remote // Take the associated remote value.
case local // Take the associated local value.
case new(value: T) // Take this synthesized value.
var isUnchanged: Bool {
if case .unchanged = self {
return true
}
return false
}
var isUnknown: Bool {
if case .unknown = self {
return true
}
return false
}
var label: String {
switch self {
case .unknown:
return "Unknown"
case .unchanged:
return "Unchanged"
case .remote:
return "Remote"
case .local:
return "Local"
case .new:
return "New"
}
}
}
func ==<T: Equatable>(lhs: MergeState<T>, rhs: MergeState<T>) -> Bool {
switch (lhs, rhs) {
case (.unknown, .unknown):
return true
case (.unchanged, .unchanged):
return true
case (.remote, .remote):
return true
case (.local, .local):
return true
case let (.new(lh), .new(rh)):
return lh == rh
default:
return false
}
}
/**
* Using this:
*
* You get one for the root. Then you give it children for the roots
* from the mirror.
*
* Then you walk those, populating the remote and local nodes by looking
* at the left/right trees.
*
* By comparing left and right, and doing value-based comparisons if necessary,
* a merge state is decided and assigned for both value and structure.
*
* One then walks both left and right child structures (both to ensure that
* all nodes on both left and right will be visited!) recursively.
*/
class MergedTreeNode {
let guid: GUID
let mirror: BookmarkTreeNode?
var remote: BookmarkTreeNode?
var local: BookmarkTreeNode?
var hasLocal: Bool { return self.local != nil }
var hasMirror: Bool { return self.mirror != nil }
var hasRemote: Bool { return self.remote != nil }
var valueState: MergeState<BookmarkMirrorItem> = MergeState.unknown
var structureState: MergeState<BookmarkTreeNode> = MergeState.unknown
var hasDecidedChildren: Bool {
return !self.structureState.isUnknown
}
var mergedChildren: [MergedTreeNode]?
// One-sided constructors.
static func forRemote(_ remote: BookmarkTreeNode, mirror: BookmarkTreeNode?=nil) -> MergedTreeNode {
let n = MergedTreeNode(guid: remote.recordGUID, mirror: mirror, structureState: MergeState.remote)
n.remote = remote
n.valueState = MergeState.remote
return n
}
static func forLocal(_ local: BookmarkTreeNode, mirror: BookmarkTreeNode?=nil) -> MergedTreeNode {
let n = MergedTreeNode(guid: local.recordGUID, mirror: mirror, structureState: MergeState.local)
n.local = local
n.valueState = MergeState.local
return n
}
static func forUnchanged(_ mirror: BookmarkTreeNode) -> MergedTreeNode {
let n = MergedTreeNode(guid: mirror.recordGUID, mirror: mirror, structureState: MergeState.unchanged)
n.valueState = MergeState.unchanged
return n
}
init(guid: GUID, mirror: BookmarkTreeNode?, structureState: MergeState<BookmarkTreeNode>) {
self.guid = guid
self.mirror = mirror
self.structureState = structureState
}
init(guid: GUID, mirror: BookmarkTreeNode?) {
self.guid = guid
self.mirror = mirror
}
// N.B., you cannot recurse down `decidedStructure`: you'll depart from the
// merged tree. You need to use `mergedChildren` instead.
fileprivate var decidedStructure: BookmarkTreeNode? {
switch self.structureState {
case .unknown:
return nil
case .unchanged:
return self.mirror
case .remote:
return self.remote
case .local:
return self.local
case let .new(node):
return node
}
}
func asUnmergedTreeNode() -> BookmarkTreeNode {
return self.decidedStructure ?? BookmarkTreeNode.unknown(guid: self.guid)
}
// Recursive. Starts returning Unknown when nodes haven't been processed.
func asMergedTreeNode() -> BookmarkTreeNode {
guard let decided = self.decidedStructure,
let merged = self.mergedChildren else {
return BookmarkTreeNode.unknown(guid: self.guid)
}
if case .folder = decided {
let children = merged.map { $0.asMergedTreeNode() }
return BookmarkTreeNode.folder(guid: self.guid, children: children)
}
return decided
}
var isFolder: Bool {
return self.mergedChildren != nil
}
func dump(_ indent: Int) {
precondition(indent < 200)
let r: Character = "R"
let l: Character = "L"
let m: Character = "M"
let ind = indenting(indent)
print(ind, "[V: ", box(self.remote, r), box(self.mirror, m), box(self.local, l), self.guid, self.valueState.label, "]")
guard self.isFolder else {
return
}
print(ind, "[S: ", self.structureState.label, "]")
if let children = self.mergedChildren {
print(ind, " ..")
for child in children {
child.dump(indent + 2)
}
}
}
}
private func box<T>(_ x: T?, _ c: Character) -> Character {
if x == nil {
return "□"
}
return c
}
private func indenting(_ by: Int) -> String {
return String(repeating: " ", count: by)
}
class MergedTree {
var root: MergedTreeNode
var deleteLocally: Set<GUID> = Set()
var deleteRemotely: Set<GUID> = Set()
var deleteFromMirror: Set<GUID> = Set()
var acceptLocalDeletion: Set<GUID> = Set()
var acceptRemoteDeletion: Set<GUID> = Set()
var allGUIDs: Set<GUID> {
var out = Set<GUID>([self.root.guid])
func acc(_ node: MergedTreeNode) {
guard let children = node.mergedChildren else {
return
}
out.formUnion(Set(children.map { $0.guid }))
children.forEach(acc)
}
acc(self.root)
return out
}
init(mirrorRoot: BookmarkTreeNode) {
self.root = MergedTreeNode(guid: mirrorRoot.recordGUID, mirror: mirrorRoot, structureState: MergeState.unchanged)
self.root.valueState = MergeState.unchanged
}
func dump() {
print("Deleted locally: \(self.deleteLocally.joined(separator: ", "))")
print("Deleted remotely: \(self.deleteRemotely.joined(separator: ", "))")
print("Deleted from mirror: \(self.deleteFromMirror.joined(separator: ", "))")
print("Accepted local deletions: \(self.acceptLocalDeletion.joined(separator: ", "))")
print("Accepted remote deletions: \(self.acceptRemoteDeletion.joined(separator: ", "))")
print("Root: ")
self.root.dump(0)
}
}

1423
mobile/ios/Sync/Synchronizers/Bookmarks/ThreeWayTreeMerger.swift Normal file
View file

File diff suppressed because it is too large Load diff

432
mobile/ios/Sync/Synchronizers/ClientsSynchronizer.swift Normal file
View file

@ -0,0 +1,432 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
import SwiftyJSON
private let log = Logger.syncLogger
let ClientsStorageVersion = 1
// TODO
public protocol Command {
static func fromName(_ command: String, args: [JSON]) -> Command?
func run(_ synchronizer: ClientsSynchronizer) -> Success
static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command?
}
// Shit.
// We need a way to wipe or reset engines.
// We need a way to log out the account.
// So when we sync commands, we're gonna need a delegate of some kind.
open class WipeCommand: Command {
public init?(command: String, args: [JSON]) {
return nil
}
open class func fromName(_ command: String, args: [JSON]) -> Command? {
return WipeCommand(command: command, args: args)
}
open func run(_ synchronizer: ClientsSynchronizer) -> Success {
return succeed()
}
open static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command? {
let json = JSON(parseJSON: syncCommand.value)
if let name = json["command"].string,
let args = json["args"].array {
return WipeCommand.fromName(name, args: args)
}
return nil
}
}
open class DisplayURICommand: Command {
let uri: URL
let title: String
let sender: String
public init?(command: String, args: [JSON]) {
if let uri = args[0].string?.asURL,
let sender = args[1].string,
let title = args[2].string {
self.uri = uri
self.sender = sender
self.title = title
} else {
// Oh, Swift.
self.uri = "http://localhost/".asURL!
self.title = ""
return nil
}
}
open class func fromName(_ command: String, args: [JSON]) -> Command? {
return DisplayURICommand(command: command, args: args)
}
open func run(_ synchronizer: ClientsSynchronizer) -> Success {
func display(_ deviceName: String? = nil) -> Success {
synchronizer.delegate.displaySentTab(for: uri, title: title, from: deviceName)
return succeed()
}
guard let getClientWithId = synchronizer.localClients?.getClientWithId(sender) else {
return display()
}
return getClientWithId >>== { client in
return display(client?.name)
}
}
open static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command? {
let json = JSON(parseJSON: syncCommand.value)
if let name = json["command"].string,
let args = json["args"].array {
return DisplayURICommand.fromName(name, args: args)
}
return nil
}
}
open class RepairResponseCommand: Command {
let repairResponse: RepairResponse
public init(command: String, args: [JSON]) {
self.repairResponse = RepairResponse.fromJSON(args: args[0])
}
open class func fromName(_ command: String, args: [JSON]) -> Command? {
return RepairResponseCommand(command: command, args: args)
}
open func run(_ synchronizer: ClientsSynchronizer) -> Success {
let repairer = BookmarksRepairRequestor(scratchpad: synchronizer.scratchpad, basePrefs: synchronizer.basePrefs, remoteClients: synchronizer.localClients!)
return repairer.continueRepairs(response: self.repairResponse) >>> succeed
}
open static func commandFromSyncCommand(_ syncCommand: SyncCommand) -> Command? {
let json = JSON(parseJSON: syncCommand.value)
if let name = json["command"].string,
let args = json["args"].array {
return RepairResponseCommand.fromName(name, args: args)
}
return nil
}
}
let Commands: [String: (String, [JSON]) -> Command?] = [
"wipeAll": WipeCommand.fromName,
"wipeEngine": WipeCommand.fromName,
// resetEngine
// resetAll
// logout
"displayURI": DisplayURICommand.fromName,
"repairResponse": RepairResponseCommand.fromName
]
open class ClientsSynchronizer: TimestampedSingleCollectionSynchronizer, Synchronizer {
public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "clients")
}
var localClients: RemoteClientsAndTabs?
override var storageVersion: Int {
return ClientsStorageVersion
}
var clientRecordLastUpload: Timestamp {
set(value) {
self.prefs.setLong(value, forKey: "lastClientUpload")
}
get {
return self.prefs.unsignedLongForKey("lastClientUpload") ?? 0
}
}
// Sync Object Format (Version 1) for Form Factors: http://docs.services.mozilla.com/sync/objectformats.html#id2
fileprivate enum SyncFormFactorFormat: String {
case phone = "phone"
case tablet = "tablet"
}
open func getOurClientRecord() -> Record<ClientPayload> {
let guid = self.scratchpad.clientGUID
let formfactor = formFactorString()
let json = JSON(object: [
"id": guid,
"fxaDeviceId": self.scratchpad.fxaDeviceId,
"version": AppInfo.appVersion,
"protocols": ["1.5"],
"name": self.scratchpad.clientName,
"os": "iOS",
"commands": [JSON](),
"type": "mobile",
"appPackage": AppInfo.baseBundleIdentifier,
"application": AppInfo.displayName,
"device": DeviceInfo.deviceModel(),
"formfactor": formfactor])
let payload = ClientPayload(json)
return Record(id: guid, payload: payload, ttl: ThreeWeeksInSeconds)
}
fileprivate func formFactorString() -> String {
let userInterfaceIdiom = UIDevice.current.userInterfaceIdiom
var formfactor: String
switch userInterfaceIdiom {
case .phone:
formfactor = SyncFormFactorFormat.phone.rawValue
case .pad:
formfactor = SyncFormFactorFormat.tablet.rawValue
default:
formfactor = SyncFormFactorFormat.phone.rawValue
}
return formfactor
}
fileprivate func clientRecordToLocalClientEntry(_ record: Record<ClientPayload>) -> RemoteClient {
let modified = record.modified
let payload = record.payload
return RemoteClient(json: payload.json, modified: modified)
}
// If this is a fresh start, do a wipe.
// N.B., we don't wipe outgoing commands! (TODO: check this when we implement commands!)
// N.B., but perhaps we should discard outgoing wipe/reset commands!
fileprivate func wipeIfNecessary(_ localClients: RemoteClientsAndTabs) -> Success {
if self.lastFetched == 0 {
return localClients.wipeClients()
}
return succeed()
}
/**
* Returns whether any commands were found (and thus a replacement record
* needs to be uploaded). Also returns the commands: we run them after we
* upload a replacement record.
*/
fileprivate func processCommandsFromRecord(_ record: Record<ClientPayload>?, withServer storageClient: Sync15CollectionClient<ClientPayload>) -> Deferred<Maybe<(Bool, [Command])>> {
log.debug("Processing commands from downloaded record.")
// TODO: short-circuit based on the modified time of the record we uploaded, so we don't need to skip ahead.
if let record = record {
let commands = record.payload.commands
if !commands.isEmpty {
func parse(_ json: JSON) -> Command? {
if let name = json["command"].string,
let args = json["args"].array,
let constructor = Commands[name] {
return constructor(name, args)
}
return nil
}
// TODO: can we do anything better if a command fails?
return deferMaybe((true, optFilter(commands.map(parse))))
}
}
return deferMaybe((false, []))
}
fileprivate func uploadClientCommands(toLocalClients localClients: RemoteClientsAndTabs, withServer storageClient: Sync15CollectionClient<ClientPayload>) -> Success {
return localClients.getCommands() >>== { clientCommands in
return clientCommands.map { (clientGUID, commands) -> Success in
self.syncClientCommands(clientGUID, commands: commands, clientsAndTabs: localClients, withServer: storageClient)
}.allSucceed()
}
}
fileprivate func syncClientCommands(_ clientGUID: GUID, commands: [SyncCommand], clientsAndTabs: RemoteClientsAndTabs, withServer storageClient: Sync15CollectionClient<ClientPayload>) -> Success {
let deleteCommands: () -> Success = {
return clientsAndTabs.deleteCommands(clientGUID).bind({ x in return succeed() })
}
log.debug("Fetching current client record for client \(clientGUID).")
let fetch = storageClient.get(clientGUID)
return fetch.bind() { result in
if let response = result.successValue, response.value.payload.isValid() {
let record = response.value
if var clientRecord = record.payload.json.dictionary {
clientRecord["commands"] = JSON(record.payload.commands + commands.map { JSON(parseJSON: $0.value) })
let uploadRecord = Record(id: clientGUID, payload: ClientPayload(JSON(clientRecord)), ttl: ThreeWeeksInSeconds)
return storageClient.put(uploadRecord, ifUnmodifiedSince: record.modified)
>>== { resp in
log.debug("Client \(clientGUID) commands upload succeeded.")
// Always succeed, even if we couldn't delete the commands.
return deleteCommands()
}
}
} else {
if let failure = result.failureValue {
log.warning("Failed to fetch record with GUID \(clientGUID).")
if failure is NotFound<HTTPURLResponse> {
log.debug("Not waiting to see if the client comes back.")
// TODO: keep these around and retry, expiring after a while.
// For now we just throw them away so we don't fail every time.
return deleteCommands()
}
if failure is BadRequestError<HTTPURLResponse> {
log.debug("We made a bad request. Throwing away queued commands.")
return deleteCommands()
}
}
}
log.error("Client \(clientGUID) commands upload failed: No remote client for GUID")
return deferMaybe(UnknownError())
}
}
/**
* Upload our record if either (a) we know we should upload, or (b)
* our own notes tell us we're due to reupload.
*/
fileprivate func maybeUploadOurRecord(_ should: Bool, ifUnmodifiedSince: Timestamp?, toServer storageClient: Sync15CollectionClient<ClientPayload>) -> Success {
let lastUpload = self.clientRecordLastUpload
let expired = lastUpload < (Date.now() - (2 * OneDayInMilliseconds))
log.debug("Should we upload our client record? Caller = \(should), expired = \(expired).")
if !should && !expired {
return succeed()
}
let iUS: Timestamp? = ifUnmodifiedSince ?? ((lastUpload == 0) ? nil : lastUpload)
var uploadStats = SyncUploadStats()
return storageClient.put(getOurClientRecord(), ifUnmodifiedSince: iUS)
>>== { resp in
if let ts = resp.metadata.lastModifiedMilliseconds {
// Protocol says this should always be present for success responses.
log.debug("Client record upload succeeded. New timestamp: \(ts).")
self.clientRecordLastUpload = ts
uploadStats.sent += 1
} else {
uploadStats.sentFailed += 1
}
self.statsSession.recordUpload(stats: uploadStats)
return succeed()
}
}
fileprivate func applyStorageResponse(_ response: StorageResponse<[Record<ClientPayload>]>, toLocalClients localClients: RemoteClientsAndTabs, withServer storageClient: Sync15CollectionClient<ClientPayload>, notifier: CollectionChangedNotifier?) -> Success {
log.debug("Applying clients response.")
var downloadStats = SyncDownloadStats()
let records = response.value
let responseTimestamp = response.metadata.lastModifiedMilliseconds
log.debug("Got \(records.count) client records.")
let ourGUID = self.scratchpad.clientGUID
var toInsert = [RemoteClient]()
var ours: Record<ClientPayload>? = nil
for (rec) in records {
guard rec.payload.isValid() else {
log.warning("Client record \(rec.id) is invalid. Skipping.")
continue
}
if rec.id == ourGUID {
if rec.modified == self.clientRecordLastUpload {
log.debug("Skipping our own unmodified record.")
} else {
log.debug("Saw our own record in response.")
ours = rec
}
} else {
toInsert.append(self.clientRecordToLocalClientEntry(rec))
}
}
downloadStats.applied += toInsert.count
// Apply remote changes.
// Collect commands from our own record and reupload if necessary.
// Then run the commands and return.
return localClients.insertOrUpdateClients(toInsert)
>>== { succeeded in
downloadStats.succeeded += succeeded
downloadStats.failed += (toInsert.count - succeeded)
self.statsSession.recordDownload(stats: downloadStats)
return succeed()
}
>>== { self.processCommandsFromRecord(ours, withServer: storageClient) }
>>== { (shouldUpload, commands) in
let isFirstSync = self.lastFetched == 0
return self.maybeUploadOurRecord(shouldUpload || self.why == .didLogin, ifUnmodifiedSince: ours?.modified, toServer: storageClient)
>>> { self.uploadClientCommands(toLocalClients: localClients, withServer: storageClient) }
>>> {
log.debug("Running \(commands.count) commands.")
for command in commands {
_ = command.run(self)
}
self.lastFetched = responseTimestamp!
if isFirstSync,
let notifier = notifier {
DispatchQueue.global(qos: DispatchQoS.background.qosClass).async { _ = notifier.notifyAll(collectionsChanged: ["clients"], reason: "firstsync") }
}
return succeed()
}
}
}
open func synchronizeLocalClients(_ localClients: RemoteClientsAndTabs, withServer storageClient: Sync15StorageClient, info: InfoCollections, notifier: CollectionChangedNotifier?) -> SyncResult {
log.debug("Synchronizing clients.")
self.localClients = localClients // Store for later when we process a repairResponse command
if let reason = self.reasonToNotSync(storageClient) {
switch reason {
case .engineRemotelyNotEnabled:
// This is a hard error for us.
return deferMaybe(FatalError(message: "clients not mentioned in meta/global. Server wiped?"))
default:
return deferMaybe(SyncStatus.notStarted(reason))
}
}
let keys = self.scratchpad.keys?.value
let encoder = RecordEncoder<ClientPayload>(decode: { ClientPayload($0) }, encode: { $0.json })
let encrypter = keys?.encrypter(self.collection, encoder: encoder)
if encrypter == nil {
log.error("Couldn't make clients encrypter.")
return deferMaybe(FatalError(message: "Couldn't make clients encrypter."))
}
let clientsClient = storageClient.clientForCollection(self.collection, encrypter: encrypter!)
// TODO: some of the commands we process might involve wiping collections or the
// entire profile. We should model this as an explicit status, and return it here
// instead of .completed.
statsSession.start()
// XXX: This is terrible. We always force a re-sync of the clients to work around
// the fact that `fxaDeviceId` may not have been populated if the list of clients
// hadn't changed since before the update to v8.0. To force a re-sync, we get all
// clients since the beginning of time instead of looking at `self.lastFetched`.
return clientsClient.getSince(0)
>>== { response in
return self.wipeIfNecessary(localClients)
>>> { self.applyStorageResponse(response, toLocalClients: localClients, withServer: clientsClient, notifier: notifier) }
}
>>> { deferMaybe(self.completedWithStats) }
}
}

231
mobile/ios/Sync/Synchronizers/Downloader.swift Normal file
View file

@ -0,0 +1,231 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
private let log = Logger.syncLogger
class BatchingDownloader<T: CleartextPayloadJSON> {
let client: Sync15CollectionClient<T>
let collection: String
let prefs: Prefs
var batch: [Record<T>] = []
func store(_ records: [Record<T>]) {
self.batch += records
}
func retrieve() -> [Record<T>] {
let ret = self.batch
self.batch = []
return ret
}
var _advance: (() -> Void)?
func advance() {
guard let f = self._advance else {
return
}
self._advance = nil
f()
}
init(collectionClient: Sync15CollectionClient<T>, basePrefs: Prefs, collection: String) {
self.client = collectionClient
self.collection = collection
let branchName = "downloader." + collection + "."
self.prefs = basePrefs.branch(branchName)
log.info("Downloader configured with prefs '\(self.prefs.getBranchPrefix())'.")
}
static func resetDownloaderWithPrefs(_ basePrefs: Prefs, collection: String) {
// This leads to stupid paths like 'profile.sync.synchronizer.history..downloader.history..'.
// Sorry, but it's out in the world now...
let branchName = "downloader." + collection + "."
let prefs = basePrefs.branch(branchName)
let lm = prefs.timestampForKey("lastModified")
let bt = prefs.timestampForKey("baseTimestamp")
log.debug("Resetting downloader prefs \(prefs.getBranchPrefix()). Previous values: \(lm ??? "nil"), \(bt ??? "nil").")
prefs.removeObjectForKey("nextOffset")
prefs.removeObjectForKey("offsetNewer")
prefs.removeObjectForKey("baseTimestamp")
prefs.removeObjectForKey("lastModified")
}
/**
* Clients should provide the same set of parameters alongside an `offset` as was
* provided with the initial request. The only thing that varies in our batch fetches
* is `newer`, so we track the original value alongside.
*/
var nextFetchParameters: (String, Timestamp)? {
get {
let o = self.prefs.stringForKey("nextOffset")
let n = self.prefs.timestampForKey("offsetNewer")
guard let offset = o, let newer = n else {
return nil
}
return (offset, newer)
}
set (value) {
if let (offset, newer) = value {
self.prefs.setString(offset, forKey: "nextOffset")
self.prefs.setTimestamp(newer, forKey: "offsetNewer")
} else {
self.prefs.removeObjectForKey("nextOffset")
self.prefs.removeObjectForKey("offsetNewer")
}
}
}
// Set after each batch, from record timestamps.
var baseTimestamp: Timestamp {
get {
return self.prefs.timestampForKey("baseTimestamp") ?? 0
}
set (value) {
self.prefs.setTimestamp(value, forKey: "baseTimestamp")
}
}
// Only set at the end of a batch, from headers.
var lastModified: Timestamp {
get {
return self.prefs.timestampForKey("lastModified") ?? 0
}
set (value) {
self.prefs.setTimestamp(value, forKey: "lastModified")
}
}
/**
* Call this when a significant structural server change has been detected.
*/
func reset() -> Success {
self.baseTimestamp = 0
self.lastModified = 0
self.nextFetchParameters = nil
self.batch = []
self._advance = nil
return succeed()
}
func go(_ info: InfoCollections, limit: Int) -> Deferred<Maybe<DownloadEndState>> {
guard let modified = info.modified(self.collection) else {
log.debug("No server modified time for collection \(self.collection).")
return deferMaybe(.noNewData)
}
log.debug("Modified: \(modified); last \(self.lastModified).")
if modified == self.lastModified {
log.debug("No more data to batch-download.")
return deferMaybe(.noNewData)
}
// If the caller hasn't advanced after the last batch, strange things will happen --
// potentially looping indefinitely. Warn.
if self._advance != nil && !self.batch.isEmpty {
log.warning("Downloading another batch without having advanced. This might be a bug.")
}
return self.downloadNextBatchWithLimit(limit, infoModified: modified)
}
func advanceTimestampTo(_ timestamp: Timestamp) {
log.debug("Advancing downloader lastModified from \(self.lastModified) to \(timestamp).")
self.lastModified = timestamp
}
// We're either fetching from our current base timestamp with no offset,
// or the timestamp we were using when we last saved an offset.
func fetchParameters() -> (String?, Timestamp) {
if let (offset, since) = self.nextFetchParameters {
return (offset, since)
}
return (nil, max(self.lastModified, self.baseTimestamp))
}
func downloadNextBatchWithLimit(_ limit: Int, infoModified: Timestamp) -> Deferred<Maybe<DownloadEndState>> {
let (offset, since) = self.fetchParameters()
log.debug("Fetching newer=\(since), offset=\(offset ?? "nil").")
let fetch = self.client.getSince(since, sort: SortOption.OldestFirst, limit: limit, offset: offset)
func handleFailure(_ err: MaybeErrorType) -> Deferred<Maybe<DownloadEndState>> {
log.debug("Handling failure.")
guard let badRequest = err as? BadRequestError<[Record<T>]>, badRequest.response.metadata.status == 412 else {
// Just pass through the failure.
return deferMaybe(err)
}
// Conflict. Start again.
log.warning("Server contents changed during offset-based batching. Stepping back.")
self.nextFetchParameters = nil
return deferMaybe(.interrupted)
}
func handleSuccess(_ response: StorageResponse<[Record<T>]>) -> Deferred<Maybe<DownloadEndState>> {
log.debug("Handling success.")
let nextOffset = response.metadata.nextOffset
let responseModified = response.value.last?.modified
// Queue up our metadata advance. We wait until the consumer has fetched
// and processed this batch; they'll call .advance() on success.
self._advance = {
// Shift to the next offset. This might be nil, in which case… fine!
// Note that we preserve the previous 'newer' value from the offset or the original fetch,
// even as we update baseTimestamp.
self.nextFetchParameters = nextOffset == nil ? nil : (nextOffset!, since)
// If there are records, advance to just before the timestamp of the last.
// If our next fetch with X-Weave-Next-Offset fails, at least we'll start here.
//
// This approach is only valid if we're fetching oldest-first.
if let newBase = responseModified {
log.debug("Advancing baseTimestamp to \(newBase) - 1")
self.baseTimestamp = newBase - 1
}
if nextOffset == nil {
// If we can't get a timestamp from the header -- and we should always be able to --
// we fall back on the collection modified time in i/c, as supplied by the caller.
// In any case where there is no racing writer these two values should be the same.
// If they differ, the header should be later. If it's missing, and we use the i/c
// value, we'll simply redownload some records.
// All bets are off if we hit this case and are filtering somehow… don't do that.
let lm = response.metadata.lastModifiedMilliseconds
log.debug("Advancing lastModified to \(String(describing: lm)) ?? \(infoModified).")
self.lastModified = lm ?? infoModified
}
}
log.debug("Got success response with \(response.metadata.records ?? 0) records.")
// Store the incoming records for collection.
self.store(response.value)
return deferMaybe(nextOffset == nil ? .complete : .incomplete)
}
return fetch.bind { result in
guard let response = result.successValue else {
return handleFailure(result.failureValue!)
}
return handleSuccess(response)
}
}
}
public enum DownloadEndState: String {
case complete // We're done. Records are waiting for you.
case incomplete // applyBatch was called, and we think there are more records.
case noNewData // There were no records.
case interrupted // We got a 412 conflict when fetching the next batch.
}

287
mobile/ios/Sync/Synchronizers/HistorySynchronizer.swift Normal file
View file

@ -0,0 +1,287 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
import SwiftyJSON
private let log = Logger.syncLogger
private let HistoryTTLInSeconds = 5184000 // 60 days.
let HistoryStorageVersion = 1
func makeDeletedHistoryRecord(_ guid: GUID) -> Record<HistoryPayload> {
// Local modified time is ignored in upload serialization.
let modified: Timestamp = 0
// Sortindex for history is frecency. Make deleted items more frecent than almost
// anything.
let sortindex = 5_000_000
let ttl = HistoryTTLInSeconds
let json: JSON = JSON([
"id": guid,
"deleted": true,
])
let payload = HistoryPayload(json)
return Record<HistoryPayload>(id: guid, payload: payload, modified: modified, sortindex: sortindex, ttl: ttl)
}
func makeHistoryRecord(_ place: Place, visits: [Visit]) -> Record<HistoryPayload> {
let id = place.guid
let modified: Timestamp = 0 // Ignored in upload serialization.
let sortindex = 1 // TODO: frecency!
let ttl = HistoryTTLInSeconds
let json: JSON = JSON([
"id": id,
"visits": visits.map { $0.toJSON() },
"histUri": place.url,
"title": place.title,
])
let payload = HistoryPayload(json)
return Record<HistoryPayload>(id: id, payload: payload, modified: modified, sortindex: sortindex, ttl: ttl)
}
open class HistorySynchronizer: IndependentRecordSynchronizer, Synchronizer {
public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "history")
}
override var storageVersion: Int {
return HistoryStorageVersion
}
fileprivate let batchSize: Int = 1000 // A balance between number of requests and per-request size.
fileprivate func mask(_ maxFailures: Int) -> (Maybe<()>) -> Success {
var failures = 0
return { result in
if result.isSuccess {
return Deferred(value: result)
}
failures += 1
if failures > maxFailures {
return Deferred(value: result)
}
log.debug("Masking failure \(failures).")
return succeed()
}
}
// TODO: this function should establish a transaction at suitable points.
// TODO: a much more efficient way to do this is to:
// 1. Start a transaction.
// 2. Try to update each place. Note failures.
// 3. bulkInsert all failed updates in one go.
// 4. Store all remote visits for all places in one go, constructing a single sequence of visits.
func applyIncomingToStorage(_ storage: SyncableHistory, records: [Record<HistoryPayload>]) -> Success {
// Skip over at most this many failing records before aborting the sync.
let maskSomeFailures = self.mask(3)
// TODO: it'd be nice to put this in an extension on SyncableHistory. Waiting for Swift 2.0...
func applyRecord(_ rec: Record<HistoryPayload>) -> Success {
let guid = rec.id
let payload = rec.payload
let modified = rec.modified
// We apply deletions immediately. Yes, this will throw away local visits
// that haven't yet been synced. That's how Sync works, alas.
if payload.deleted {
return storage.deleteByGUID(guid, deletedAt: modified).bind(maskSomeFailures)
}
// It's safe to apply other remote records, too -- even if we re-download, we know
// from our local cached server timestamp on each record that we've already seen it.
// We have to reconcile on-the-fly: we're about to overwrite the server record, which
// is our shared parent.
let place = rec.payload.asPlace()
if isIgnoredURL(place.url) {
log.debug("Ignoring incoming record \(guid) because its URL is one we wish to ignore.")
return succeed()
}
let placeThenVisits = storage.insertOrUpdatePlace(place, modified: modified)
>>> { storage.storeRemoteVisits(payload.visits, forGUID: guid) }
return placeThenVisits.map({ result in
if result.isFailure {
let reason = result.failureValue?.description ?? "unknown reason"
log.error("Record application failed: \(reason)")
}
return result
}).bind(maskSomeFailures)
}
return self.applyIncomingRecords(records, apply: applyRecord)
}
fileprivate func uploadModifiedPlaces(_ places: [(Place, [Visit])], lastTimestamp: Timestamp, fromStorage storage: SyncableHistory, withServer storageClient: Sync15CollectionClient<HistoryPayload>) -> DeferredTimestamp {
log.info("Preparing upload…")
// Build sequences of 1000 history items, sequence by sequence
// These will be uploaded in smaller batches by the upload batcher, but we chunk here
// in order to bound peak memory usage when we call makeHistoryRecord below.
let toUpload = chunk(places, by: 1000)
let perChunk: (ArraySlice<(Place, [Visit])>, Timestamp) -> DeferredTimestamp = { (records, timestamp) in
let recs = records.map(makeHistoryRecord)
log.info("Uploading \(recs.count) history items…")
return self.uploadRecords(recs, lastTimestamp: timestamp, storageClient: storageClient) { result, lastModified in
// We don't do anything with failed.
return storage.markAsSynchronized(result.success, modified: lastModified ?? timestamp)
}
}
let start = deferMaybe(lastTimestamp)
return walk(toUpload, start: start, f: perChunk)
}
fileprivate func uploadDeletedPlaces(_ guids: [GUID], lastTimestamp: Timestamp, fromStorage storage: SyncableHistory, withServer storageClient: Sync15CollectionClient<HistoryPayload>) -> DeferredTimestamp {
let records = guids.map(makeDeletedHistoryRecord)
// Deletions are smaller, so upload 100 at a time.
return self.uploadRecords(records, lastTimestamp: lastTimestamp, storageClient: storageClient) { result, lastModified in
storage.markAsDeleted(result.success) >>> always(lastModified ?? lastTimestamp)
}
}
fileprivate func uploadOutgoingFromStorage(_ storage: SyncableHistory, lastTimestamp: Timestamp, withServer storageClient: Sync15CollectionClient<HistoryPayload>) -> Success {
var workWasDone = false
let uploadDeleted: (Timestamp) -> DeferredTimestamp = { timestamp in
storage.getDeletedHistoryToUpload()
>>== { guids in
if !guids.isEmpty {
workWasDone = true
}
log.info("Uploading \(guids.count) deleted places.")
return self.uploadDeletedPlaces(guids, lastTimestamp: timestamp, fromStorage: storage, withServer: storageClient)
}
}
let uploadModified: (Timestamp) -> DeferredTimestamp = { timestamp in
storage.getModifiedHistoryToUpload()
>>== { places in
if !places.isEmpty {
workWasDone = true
}
log.info("Uploading \(places.count) modified places.")
return self.uploadModifiedPlaces(places, lastTimestamp: timestamp, fromStorage: storage, withServer: storageClient)
}
}
// The last clause will checkpoint the DB. But we just checkpointed the DB after downloading records!
// Yes, that's true. Either there will be lots of work to do (e.g., having just marked
// thousands of records as uploaded, or dropping lots of deleted rows), and so it's
// worthwhile… or there won't be much work to do, and the checkpoint will be cheap.
// If we did nothing -- uploaded no deletions, uploaded no modified records -- then we
// don't checkpoint at all.
return deferMaybe(lastTimestamp)
>>== uploadDeleted
>>== uploadModified
>>> effect({ log.debug("Done syncing. Work was done? \(workWasDone)") })
>>> { workWasDone ? storage.doneUpdatingMetadataAfterUpload() : succeed() } // A closure so we eval workWasDone after it's set!
>>> effect({ log.debug("Done.") })
}
/**
* If the green light turns red, we don't want to continue to upload -- doing
* so would cause us to fast-forward our last sync timestamp and skip whatever
* we hadn't yet downloaded.
*/
fileprivate func go(_ info: InfoCollections, greenLight: @escaping () -> Bool, downloader: BatchingDownloader<HistoryPayload>, history: SyncableHistory) -> SyncResult {
if !greenLight() {
log.info("Green light turned red. Stopping history download.")
return deferMaybe(.partial(self.statsSession))
}
func applyBatched() -> Success {
return self.applyIncomingToStorage(history, records: downloader.retrieve())
>>> effect(downloader.advance)
}
func onBatchResult(_ result: Maybe<DownloadEndState>) -> SyncResult {
guard let end = result.successValue else {
log.warning("Got failure: \(result.failureValue!)")
return deferMaybe(completedWithStats)
}
switch end {
case .complete:
log.info("Done with batched mirroring.")
return applyBatched()
>>> history.doneApplyingRecordsAfterDownload
>>> { deferMaybe(self.completedWithStats) }
case .incomplete:
log.debug("Running another batch.")
// This recursion is fine because Deferred always pushes callbacks onto a queue.
return applyBatched()
>>> { self.go(info, greenLight: greenLight, downloader: downloader, history: history) }
case .interrupted:
log.info("Interrupted. Aborting batching this time.")
return deferMaybe(.partial(self.statsSession))
case .noNewData:
log.info("No new data. No need to continue batching.")
downloader.advance()
return deferMaybe(completedWithStats)
}
}
return downloader.go(info, limit: self.batchSize)
.bind(onBatchResult)
}
open func synchronizeLocalHistory(_ history: SyncableHistory, withServer storageClient: Sync15StorageClient, info: InfoCollections, greenLight: @escaping () -> Bool) -> SyncResult {
if let reason = self.reasonToNotSync(storageClient) {
return deferMaybe(.notStarted(reason))
}
let encoder = RecordEncoder<HistoryPayload>(decode: { HistoryPayload($0) }, encode: { $0.json })
guard let historyClient = self.collectionClient(encoder, storageClient: storageClient) else {
log.error("Couldn't make history factory.")
return deferMaybe(FatalError(message: "Couldn't make history factory."))
}
let downloader = BatchingDownloader(collectionClient: historyClient, basePrefs: self.prefs, collection: "history")
// The original version of the history synchronizer tracked its
// own last fetched time. We need to migrate this into the
// batching downloader.
let since: Timestamp = self.lastFetched
if since > downloader.lastModified {
log.debug("Advancing downloader lastModified to synchronizer lastFetched \(since).")
downloader.lastModified = since
self.lastFetched = 0
}
statsSession.start()
return self.go(info, greenLight: greenLight, downloader: downloader, history: history)
>>== { syncResult in
switch syncResult {
case .completed:
// When we're done downloading, we can upload.
return self.uploadOutgoingFromStorage(history,
lastTimestamp: 0,
withServer: historyClient)
>>> { deferMaybe(self.completedWithStats) }
// If we didn't finish downloading, do nothing further -- just pass
// through the download result.
case .notStarted(_):
return deferMaybe(syncResult)
case .partial:
log.debug("Didn't finish downloading history; not uploading yet.")
return deferMaybe(syncResult)
}
}
}
}

147
mobile/ios/Sync/Synchronizers/IndependentRecordSynchronizer.swift Normal file
View file

@ -0,0 +1,147 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
private let log = Logger.syncLogger
class Uploader {
/**
* Upload just about anything that can be turned into something we can upload.
*/
func sequentialPosts<T>(_ items: [T], by: Int, lastTimestamp: Timestamp, storageOp: @escaping ([T], Timestamp) -> DeferredTimestamp) -> DeferredTimestamp {
// This needs to be a real Array, not an ArraySlice,
// for the types to line up.
let chunks = chunk(items, by: by).map { Array($0) }
let start = deferMaybe(lastTimestamp)
let perChunk: ([T], Timestamp) -> DeferredTimestamp = { (records, timestamp) in
// TODO: detect interruptions -- clients uploading records during our sync --
// by using ifUnmodifiedSince. We can detect uploaded records since our download
// (chain the download timestamp into this function), and we can detect uploads
// that race with our own (chain download timestamps across 'walk' steps).
// If we do that, we can also advance our last fetch timestamp after each chunk.
log.debug("Uploading \(records.count) records.")
return storageOp(records, timestamp)
}
return walk(chunks, start: start, f: perChunk)
}
}
open class IndependentRecordSynchronizer: TimestampedSingleCollectionSynchronizer {
private func reportApplyStatsWrap<T>(apply: @escaping (T) -> Success) -> (T) -> Success {
return { record in
return apply(record).bind({ result in
var stats = SyncDownloadStats()
stats.applied = 1
if result.isSuccess {
stats.succeeded = 1
} else {
stats.failed = 1
}
self.statsSession.recordDownload(stats: stats)
return Deferred(value: result)
})
}
}
/**
* Just like the usual applyIncomingToStorage, but doesn't fast-forward the timestamp.
*/
func applyIncomingRecords<T>(_ records: [T], apply: @escaping (T) -> Success) -> Success {
if records.isEmpty {
log.debug("No records; done applying.")
return succeed()
}
return walk(records, f: reportApplyStatsWrap(apply: apply))
}
func applyIncomingToStorage<T>(_ records: [T], fetched: Timestamp, apply: @escaping (T) -> Success) -> Success {
func done() -> Success {
log.debug("Bumping fetch timestamp to \(fetched).")
self.lastFetched = fetched
return succeed()
}
if records.isEmpty {
log.debug("No records; done applying.")
return done()
}
return walk(records, f: reportApplyStatsWrap(apply: apply)) >>> done
}
}
extension TimestampedSingleCollectionSynchronizer {
/**
* On each chunk that we upload, we pass along the server modified timestamp to the next,
* chained through the provided `onUpload` function.
*
* The last chunk passes this modified timestamp out, and we assign it to lastFetched.
*
* The idea of this is twofold:
*
* 1. It does the fast-forwarding that every other Sync client does.
*
* 2. It allows us to (eventually) pass the last collection modified time as If-Unmodified-Since
* on each upload batch, as we do between the download and the upload phase.
* This alone allows us to detect conflicts from racing clients.
*
* In order to implement the latter, we'd need to chain the date from getSince in place of the
* 0 in the call to uploadOutgoingFromStorage in each synchronizer.
*/
func uploadRecords<T>(_ records: [Record<T>], lastTimestamp: Timestamp, storageClient: Sync15CollectionClient<T>, onUpload: @escaping (POSTResult, Timestamp?) -> DeferredTimestamp) -> DeferredTimestamp {
if records.isEmpty {
log.debug("No modified records to upload.")
return deferMaybe(lastTimestamp)
}
func reportUploadStatsWrap(result: POSTResult, timestamp: Timestamp?) -> DeferredTimestamp {
let stats = SyncUploadStats(sent: result.success.count, sentFailed: result.failed.count)
self.statsSession.recordUpload(stats: stats)
return onUpload(result, timestamp)
}
let batch = storageClient.newBatch(ifUnmodifiedSince: (lastTimestamp == 0) ? nil : lastTimestamp, onCollectionUploaded: reportUploadStatsWrap)
return batch.addRecords(records)
>>> batch.endBatch
>>> {
let timestamp = batch.ifUnmodifiedSince ?? lastTimestamp
self.setTimestamp(timestamp)
return deferMaybe(timestamp)
}
}
func uploadRecordsSingleBatch<T>(_ records: [Record<T>], lastTimestamp: Timestamp, storageClient: Sync15CollectionClient<T>) -> Deferred<Maybe<(timestamp: Timestamp, succeeded: [GUID])>> {
if records.isEmpty {
log.debug("No modified records to upload.")
return deferMaybe((timestamp: lastTimestamp, succeeded: []))
}
func reportUploadStatsWrap(result: POSTResult, timestamp: Timestamp?) -> DeferredTimestamp {
let stats = SyncUploadStats(sent: result.success.count, sentFailed: result.failed.count)
self.statsSession.recordUpload(stats: stats)
return deferMaybe(timestamp ?? lastTimestamp)
}
let batch = storageClient.newBatch(ifUnmodifiedSince: (lastTimestamp == 0) ? nil : lastTimestamp, onCollectionUploaded: reportUploadStatsWrap)
return batch.addRecords(records, singleBatch: true)
>>== batch.endSingleBatch
>>== { (succeeded, lastModified) in
guard let timestamp = lastModified else {
return deferMaybe(FatalError(message: "Could not retrieve lastModified from the server response."))
}
self.setTimestamp(timestamp)
return deferMaybe((timestamp: timestamp, succeeded: succeeded))
}
}
}

190
mobile/ios/Sync/Synchronizers/LoginsSynchronizer.swift Normal file
View file

@ -0,0 +1,190 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
import SwiftyJSON
private let log = Logger.syncLogger
let PasswordsStorageVersion = 1
private func makeDeletedLoginRecord(_ guid: GUID) -> Record<LoginPayload> {
// Local modified time is ignored in upload serialization.
let modified: Timestamp = 0
// Arbitrary large number: deletions sync down first.
let sortindex = 5_000_000
let json: JSON = JSON([
"id": guid,
"deleted": true,
])
let payload = LoginPayload(json)
return Record<LoginPayload>(id: guid, payload: payload, modified: modified, sortindex: sortindex)
}
func makeLoginRecord(_ login: Login) -> Record<LoginPayload> {
let id = login.guid
let modified: Timestamp = 0 // Ignored in upload serialization.
let sortindex = 1
let tLU = NSNumber(value: login.timeLastUsed / 1000)
let tPC = NSNumber(value: login.timePasswordChanged / 1000)
let tC = NSNumber(value: login.timeCreated / 1000)
let dict: [String: Any] = [
"id": id,
"hostname": login.hostname,
"httpRealm": login.httpRealm as Any,
"formSubmitURL": login.formSubmitURL as Any,
"username": login.username ?? "",
"password": login.password ,
"usernameField": login.usernameField ?? "",
"passwordField": login.passwordField ?? "",
"timesUsed": login.timesUsed,
"timeLastUsed": tLU,
"timePasswordChanged": tPC,
"timeCreated": tC,
]
let payload = LoginPayload(JSON(dict))
return Record<LoginPayload>(id: id, payload: payload, modified: modified, sortindex: sortindex)
}
/**
* Our current local terminology ("logins") has diverged from the terminology in
* use when Sync was built ("passwords"). I've done my best to draw a reasonable line
* between the server collection/record format/etc. and local stuff: local storage
* works with logins, server records and collection are passwords.
*/
open class LoginsSynchronizer: IndependentRecordSynchronizer, Synchronizer {
public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "passwords")
}
override var storageVersion: Int {
return PasswordsStorageVersion
}
func getLogin(_ record: Record<LoginPayload>) -> ServerLogin {
let guid = record.id
let payload = record.payload
let modified = record.modified
let login = ServerLogin(guid: guid, hostname: payload.hostname, username: payload.username, password: payload.password, modified: modified)
login.formSubmitURL = payload.formSubmitURL
login.httpRealm = payload.httpRealm
login.usernameField = payload.usernameField
login.passwordField = payload.passwordField
// Microseconds locally, milliseconds remotely. We should clean this up.
login.timeCreated = 1000 * (payload.timeCreated ?? 0)
login.timeLastUsed = 1000 * (payload.timeLastUsed ?? 0)
login.timePasswordChanged = 1000 * (payload.timePasswordChanged ?? 0)
login.timesUsed = payload.timesUsed ?? 0
return login
}
func applyIncomingToStorage(_ storage: SyncableLogins, records: [Record<LoginPayload>], fetched: Timestamp) -> Success {
return self.applyIncomingToStorage(records, fetched: fetched) { rec in
let guid = rec.id
let payload = rec.payload
guard payload.isValid() else {
log.warning("Login record \(guid) is invalid. Skipping.")
return succeed()
}
// We apply deletions immediately. That might not be exactly what we want -- perhaps you changed
// a password locally after deleting it remotely -- but it's expedient.
if payload.deleted {
return storage.deleteByGUID(guid, deletedAt: rec.modified)
}
return storage.applyChangedLogin(self.getLogin(rec))
}
}
fileprivate func uploadChangedRecords<T>(_ deleted: Set<GUID>, modified: Set<GUID>, records: [Record<T>], lastTimestamp: Timestamp, storage: SyncableLogins, withServer storageClient: Sync15CollectionClient<T>) -> Success {
let onUpload: (POSTResult, Timestamp?) -> DeferredTimestamp = { result, lastModified in
let uploaded = Set(result.success)
return storage.markAsDeleted(uploaded.intersection(deleted)) >>> { storage.markAsSynchronized(uploaded.intersection(modified), modified: lastModified ?? lastTimestamp) }
}
return uploadRecords(records,
lastTimestamp: lastTimestamp,
storageClient: storageClient,
onUpload: onUpload) >>> succeed
}
// Find any records for which a local overlay exists. If we want to be really precise,
// we can find the original server modified time for each record and use it as
// If-Unmodified-Since on a PUT, or just use the last fetch timestamp, which should
// be equivalent.
// We will already have reconciled any conflicts on download, so this upload phase should
// be as simple as uploading any changed or deleted items.
fileprivate func uploadOutgoingFromStorage(_ storage: SyncableLogins, lastTimestamp: Timestamp, withServer storageClient: Sync15CollectionClient<LoginPayload>) -> Success {
let deleted: () -> Deferred<Maybe<(Set<GUID>, [Record<LoginPayload>])>> = {
return storage.getDeletedLoginsToUpload() >>== { guids in
let records = guids.map(makeDeletedLoginRecord)
return deferMaybe((Set(guids), records))
}
}
let modified: () -> Deferred<Maybe<(Set<GUID>, [Record<LoginPayload>])>> = {
return storage.getModifiedLoginsToUpload() >>== { logins in
let guids = Set(logins.map { $0.guid })
let records = logins.map(makeLoginRecord)
return deferMaybe((guids, records))
}
}
return accumulate([deleted, modified]) >>== { result in
let (deletedGUIDs, deletedRecords) = result[0]
let (modifiedGUIDs, modifiedRecords) = result[1]
let allRecords = deletedRecords + modifiedRecords
return self.uploadChangedRecords(deletedGUIDs, modified: modifiedGUIDs, records: allRecords,
lastTimestamp: lastTimestamp, storage: storage, withServer: storageClient)
}
}
open func synchronizeLocalLogins(_ logins: SyncableLogins, withServer storageClient: Sync15StorageClient, info: InfoCollections) -> SyncResult {
if let reason = self.reasonToNotSync(storageClient) {
return deferMaybe(.notStarted(reason))
}
let encoder = RecordEncoder<LoginPayload>(decode: { LoginPayload($0) }, encode: { $0.json })
guard let passwordsClient = self.collectionClient(encoder, storageClient: storageClient) else {
log.error("Couldn't make logins factory.")
return deferMaybe(FatalError(message: "Couldn't make logins factory."))
}
let since: Timestamp = self.lastFetched
log.debug("Synchronizing \(self.collection). Last fetched: \(since).")
let applyIncomingToStorage: (StorageResponse<[Record<LoginPayload>]>) -> Success = { response in
let ts = response.metadata.timestampMilliseconds
let lm = response.metadata.lastModifiedMilliseconds!
log.debug("Applying incoming password records from response timestamped \(ts), last modified \(lm).")
log.debug("Records header hint: \(response.metadata.records ??? "nil")")
return self.applyIncomingToStorage(logins, records: response.value, fetched: lm) >>> effect {
NotificationCenter.default.post(name: NotificationDataRemoteLoginChangesWereApplied, object: nil)
}
}
statsSession.start()
return passwordsClient.getSince(since)
>>== applyIncomingToStorage
// TODO: If we fetch sorted by date, we can bump the lastFetched timestamp
// to the last successfully applied record timestamp, no matter where we fail.
// There's no need to do the upload before bumping -- the storage of local changes is stable.
>>> { self.uploadOutgoingFromStorage(logins, lastTimestamp: 0, withServer: passwordsClient) }
>>> { return deferMaybe(self.completedWithStats) }
}
}

293
mobile/ios/Sync/Synchronizers/Synchronizer.swift Normal file
View file

@ -0,0 +1,293 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
private let log = Logger.syncLogger
/**
* This exists to pass in external context: e.g., the UIApplication can
* expose notification functionality in this way.
*/
public protocol SyncDelegate {
func displaySentTab(for url: URL, title: String, from deviceName: String?)
// TODO: storage.
}
/**
* We sometimes want to make a synchronizer start from scratch: to throw away any
* metadata and reset storage to match, allowing us to respond to significant server
* changes.
*
* But instantiating a Synchronizer is a lot of work if we simply want to change some
* persistent state. This protocol describes a static func that fits most synchronizers.
*
* When the returned `Deferred` is filled with a success value, the supplied prefs and
* storage are ready to sync from scratch.
*
* Persisted long-term/local data is kept, and will later be reconciled as appropriate.
*/
public protocol ResettableSynchronizer {
static func resetSynchronizerWithStorage(_ storage: ResettableSyncStorage, basePrefs: Prefs, collection: String) -> Success
}
/**
* This is a delegate that allows synchronizers to notify other devices in the Sync account
* that a collection changed.
*/
public protocol CollectionChangedNotifier {
func notify(deviceIDs: [GUID], collectionsChanged collections: [String], reason: String) -> Success
func notifyAll(collectionsChanged collections: [String], reason: String) -> Success
}
// TODO: return values?
/**
* A Synchronizer is (unavoidably) entirely in charge of what it does within a sync.
* For example, it might make incremental progress in building a local cache of remote records, never actually performing an upload or modifying local storage.
* It might only upload data. Etc.
*
* Eventually I envision an intent-like approach, or additional methods, to specify preferences and constraints
* (e.g., "do what you can in a few seconds", or "do a full sync, no matter how long it takes"), but that'll come in time.
*
* A Synchronizer is a two-stage beast. It needs to support synchronization, of course; that
* needs a completely configured client, which can only be obtained from Ready. But it also
* needs to be able to do certain things beforehand:
*
* * Wipe its collections from the server (presumably via a delegate from the state machine).
* * Prepare to sync from scratch ("reset") in response to a changed set of keys, syncID, or node assignment.
* * Wipe local storage ("wipeClient").
*
* Those imply that some kind of 'Synchronizer' exists throughout the state machine. We *could*
* pickle instructions for eventual delivery next time one is made and synchronized…
*/
public protocol Synchronizer {
init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason)
/**
* Return a reason if the current state of this synchronizer -- particularly prefs and scratchpad --
* prevent a routine sync from occurring.
*/
func reasonToNotSync(_: Sync15StorageClient) -> SyncNotStartedReason?
}
/**
* We sometimes wish to return something more nuanced than simple success or failure.
* For example, refusing to sync because the engine was disabled isn't success (nothing was transferred!)
* but it also isn't an error.
*
* To do this we model real failures -- something went wrong -- as failures in the Result, and
* everything else in this status enum. This will grow as we return more details from a sync to allow
* for batch scheduling, success-case backoff and so on.
*/
public enum SyncStatus {
case completed(SyncEngineStatsSession)
case notStarted(SyncNotStartedReason)
case partial(SyncEngineStatsSession)
public var description: String {
switch self {
case .completed:
return "Completed"
case let .notStarted(reason):
return "Not started: \(reason.description)"
case .partial:
return "Partial"
}
}
}
public typealias DeferredTimestamp = Deferred<Maybe<Timestamp>>
public typealias SyncResult = Deferred<Maybe<SyncStatus>>
public typealias EngineIdentifier = String
public typealias EngineStatus = (EngineIdentifier, SyncStatus)
public typealias EngineResults = [EngineStatus]
public typealias SyncOperationResult = (engineResults: Maybe<EngineResults>, stats: SyncOperationStatsSession?)
public enum SyncNotStartedReason {
case noAccount
case offline
case backoff(remainingSeconds: Int)
case engineRemotelyNotEnabled(collection: String)
case engineFormatOutdated(needs: Int)
case engineFormatTooNew(expected: Int) // This'll disappear eventually; we'll wipe the server and upload m/g.
case storageFormatOutdated(needs: Int)
case storageFormatTooNew(expected: Int) // This'll disappear eventually; we'll wipe the server and upload m/g.
case stateMachineNotReady // Because we're not done implementing.
case redLight
case unknown // Likely a programming error.
var telemetryId: String {
switch self {
case .noAccount:
return "sync.not_started.reason.no_account"
case .offline:
return "sync.not_started.reason.offline"
case .backoff(_):
return "sync.not_started.reason.backoff"
case .engineRemotelyNotEnabled(_):
return "sync.not_started.reason.remotely_not_enabled"
case .engineFormatOutdated(_):
return "sync.not_started.reason.format_outdated"
case .engineFormatTooNew(_): // This'll disappear eventually; we'll wipe the server and upload m/g.
return "sync.not_started.reason.format_too_new"
case .storageFormatOutdated(_):
return "sync.not_started.reason.storage_format_outdated"
case .storageFormatTooNew(_): // This'll disappear eventually; we'll wipe the server and upload m/g.
return "sync.not_started.reason.storage_format_too_new"
case .stateMachineNotReady: // Because we're not done implementing.
return "sync.not_started.reason.state_machine_not_ready"
case .redLight:
return "sync.not_started.reason.red_light"
case .unknown: // Likely a programming error
return "sync.not_started.reason.unknown"
}
}
var description: String {
switch self {
case .noAccount:
return "no account"
case let .backoff(remaining):
return "in backoff: \(remaining) seconds remaining"
default:
return "undescribed reason"
}
}
}
open class FatalError: SyncError {
let message: String
init(message: String) {
self.message = message
}
open var description: String {
return self.message
}
}
public protocol SingleCollectionSynchronizer {
func remoteHasChanges(_ info: InfoCollections) -> Bool
}
open class BaseCollectionSynchronizer {
let collection: String
let scratchpad: Scratchpad
let delegate: SyncDelegate
let basePrefs: Prefs
let prefs: Prefs
let why: SyncReason
var statsSession: SyncEngineStatsSession
static func prefsForCollection(_ collection: String, withBasePrefs basePrefs: Prefs) -> Prefs {
let branchName = "synchronizer." + collection + "."
return basePrefs.branch(branchName)
}
init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason, collection: String) {
self.scratchpad = scratchpad
self.delegate = delegate
self.collection = collection
self.basePrefs = basePrefs
self.prefs = BaseCollectionSynchronizer.prefsForCollection(collection, withBasePrefs: basePrefs)
self.statsSession = SyncEngineStatsSession(collection: collection)
self.why = why
log.info("Synchronizer configured with prefs '\(self.prefs.getBranchPrefix()).'")
}
var storageVersion: Int {
assert(false, "Override me!")
return 0
}
// Short-hand for returning .Complete status + recorded stats
var completedWithStats: SyncStatus {
return .completed(statsSession.end())
}
open func reasonToNotSync(_ client: Sync15StorageClient) -> SyncNotStartedReason? {
let now = Date.now()
if let until = client.backoff.isInBackoff(now) {
let remaining = (until - now) / 1000
return .backoff(remainingSeconds: Int(remaining))
}
if let metaGlobal = self.scratchpad.global?.value {
// There's no need to check the global storage format here; the state machine will already have
// done so.
if let engineMeta = metaGlobal.engines[collection] {
if engineMeta.version > self.storageVersion {
return .engineFormatOutdated(needs: engineMeta.version)
}
if engineMeta.version < self.storageVersion {
return .engineFormatTooNew(expected: engineMeta.version)
}
} else {
return .engineRemotelyNotEnabled(collection: self.collection)
}
} else {
// But a missing meta/global is a real problem.
return .stateMachineNotReady
}
// Success!
return nil
}
func encrypter<T>(_ encoder: RecordEncoder<T>) -> RecordEncrypter<T>? {
return self.scratchpad.keys?.value.encrypter(self.collection, encoder: encoder)
}
func collectionClient<T>(_ encoder: RecordEncoder<T>, storageClient: Sync15StorageClient) -> Sync15CollectionClient<T>? {
if let encrypter = self.encrypter(encoder) {
return storageClient.clientForCollection(self.collection, encrypter: encrypter)
}
return nil
}
}
/**
* Tracks a lastFetched timestamp, uses it to decide if there are any
* remote changes, and exposes a method to fast-forward after upload.
*/
open class TimestampedSingleCollectionSynchronizer: BaseCollectionSynchronizer, SingleCollectionSynchronizer {
var lastFetched: Timestamp {
set(value) {
self.prefs.setLong(value, forKey: "lastFetched")
}
get {
return self.prefs.unsignedLongForKey("lastFetched") ?? 0
}
}
func setTimestamp(_ timestamp: Timestamp) {
log.debug("Setting post-upload lastFetched to \(timestamp).")
self.lastFetched = timestamp
}
open func remoteHasChanges(_ info: InfoCollections) -> Bool {
return info.modified(self.collection) ?? 0 > self.lastFetched
}
}
extension BaseCollectionSynchronizer: ResettableSynchronizer {
public static func resetSynchronizerWithStorage(_ storage: ResettableSyncStorage, basePrefs: Prefs, collection: String) -> Success {
let synchronizerPrefs = BaseCollectionSynchronizer.prefsForCollection(collection, withBasePrefs: basePrefs)
synchronizerPrefs.removeObjectForKey("lastFetched")
// Not all synchronizers use a batching downloader, but it's
// convenient to just always reset it here.
return storage.resetClient()
>>> effect({ BatchingDownloader.resetDownloaderWithPrefs(synchronizerPrefs, collection: collection) })
}
}

206
mobile/ios/Sync/Synchronizers/TabsSynchronizer.swift Normal file
View file

@ -0,0 +1,206 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
import Foundation
import Shared
import Storage
import XCGLogger
import Deferred
import SwiftyJSON
private let log = Logger.syncLogger
let TabsStorageVersion = 1
open class TabsSynchronizer: TimestampedSingleCollectionSynchronizer, Synchronizer {
public required init(scratchpad: Scratchpad, delegate: SyncDelegate, basePrefs: Prefs, why: SyncReason) {
super.init(scratchpad: scratchpad, delegate: delegate, basePrefs: basePrefs, why: why, collection: "tabs")
}
override var storageVersion: Int {
return TabsStorageVersion
}
var tabsRecordLastUpload: Timestamp {
set(value) {
self.prefs.setLong(value, forKey: "lastTabsUpload")
}
get {
return self.prefs.unsignedLongForKey("lastTabsUpload") ?? 0
}
}
fileprivate func createOwnTabsRecord(_ tabs: [RemoteTab]) -> Record<TabsPayload> {
let guid = self.scratchpad.clientGUID
let tabsJSON = JSON([
"id": guid,
"clientName": self.scratchpad.clientName,
"tabs": tabs.flatMap { $0.toDictionary() }
])
if Logger.logPII {
log.verbose("Sending tabs JSON \(tabsJSON.stringValue() ?? "nil")")
}
let payload = TabsPayload(tabsJSON)
return Record(id: guid, payload: payload, ttl: ThreeWeeksInSeconds)
}
fileprivate func uploadOurTabs(_ localTabs: RemoteClientsAndTabs, toServer tabsClient: Sync15CollectionClient<TabsPayload>) -> Success {
// check to see if our tabs have changed or we're in a fresh start
let lastUploadTime: Timestamp? = (self.tabsRecordLastUpload == 0) ? nil : self.tabsRecordLastUpload
if let lastUploadTime = lastUploadTime,
lastUploadTime >= (Date.now() - (OneMinuteInMilliseconds)) {
log.debug("Not uploading tabs: already did so at \(lastUploadTime).")
return succeed()
}
return localTabs.getTabsForClientWithGUID(nil) >>== { tabs in
if let lastUploadTime = lastUploadTime {
// TODO: track this in memory so we don't have to hit the disk to figure out when our tabs have
// changed and need to be uploaded.
if tabs.every({ $0.lastUsed < lastUploadTime }) {
return succeed()
}
}
let tabsRecord = self.createOwnTabsRecord(tabs)
log.debug("Uploading our tabs: \(tabs.count).")
var uploadStats = SyncUploadStats()
uploadStats.sent += 1
// We explicitly don't send If-Unmodified-Since, because we always
// want our upload to succeed -- we own the record.
return tabsClient.put(tabsRecord, ifUnmodifiedSince: nil) >>== { resp in
if let ts = resp.metadata.lastModifiedMilliseconds {
// Protocol says this should always be present for success responses.
log.debug("Tabs record upload succeeded. New timestamp: \(ts).")
self.tabsRecordLastUpload = ts
} else {
uploadStats.sentFailed += 1
}
return succeed()
} >>== effect({ self.statsSession.recordUpload(stats: uploadStats) })
}
}
open func synchronizeLocalTabs(_ localTabs: RemoteClientsAndTabs, withServer storageClient: Sync15StorageClient, info: InfoCollections) -> SyncResult {
func onResponseReceived(_ response: StorageResponse<[Record<TabsPayload>]>) -> Success {
func afterWipe() -> Success {
var downloadStats = SyncDownloadStats()
let doInsert: (Record<TabsPayload>) -> Deferred<Maybe<(Int)>> = { record in
let remotes = record.payload.isValid() ? record.payload.remoteTabs : []
let ins = localTabs.insertOrUpdateTabsForClientGUID(record.id, tabs: remotes)
// Since tabs are all sent within a single record, we don't count number of tabs applied
// but number of records. In this case it's just one.
downloadStats.applied += 1
ins.upon() { res in
if let inserted = res.successValue {
if inserted != remotes.count {
log.warning("Only inserted \(inserted) tabs, not \(remotes.count). Malformed or missing client?")
}
downloadStats.applied += 1
} else {
downloadStats.failed += 1
}
}
return ins
}
let ourGUID = self.scratchpad.clientGUID
let records = response.value
let responseTimestamp = response.metadata.lastModifiedMilliseconds
log.debug("Got \(records.count) tab records.")
// We can only insert tabs for clients that we know locally, so
// first we fetch the list of IDs and intersect the two.
// TODO: there's a much more efficient way of doing this.
return localTabs.getClientGUIDs()
>>== { clientGUIDs in
let filtered = records.filter({ $0.id != ourGUID && clientGUIDs.contains($0.id) })
if filtered.count != records.count {
log.debug("Filtered \(records.count) records down to \(filtered.count).")
}
let allDone = all(filtered.map(doInsert))
return allDone.bind { (results) -> Success in
if let failure = results.find({ $0.isFailure }) {
return deferMaybe(failure.failureValue!)
}
self.lastFetched = responseTimestamp!
return succeed()
}
} >>== effect({ self.statsSession.downloadStats })
}
// If this is a fresh start, do a wipe.
if self.lastFetched == 0 {
log.info("Last fetch was 0. Wiping tabs.")
return localTabs.wipeRemoteTabs()
>>== afterWipe
}
return afterWipe()
}
if let reason = self.reasonToNotSync(storageClient) {
return deferMaybe(SyncStatus.notStarted(reason))
}
let keys = self.scratchpad.keys?.value
let encoder = RecordEncoder<TabsPayload>(decode: { TabsPayload($0) }, encode: { $0.json })
if let encrypter = keys?.encrypter(self.collection, encoder: encoder) {
let tabsClient = storageClient.clientForCollection(self.collection, encrypter: encrypter)
statsSession.start()
if !self.remoteHasChanges(info) {
// upload local tabs if they've changed or we're in a fresh start.
return uploadOurTabs(localTabs, toServer: tabsClient)
>>> { deferMaybe(self.completedWithStats) }
}
return tabsClient.getSince(self.lastFetched)
>>== onResponseReceived
>>> { self.uploadOurTabs(localTabs, toServer: tabsClient) }
>>> { deferMaybe(self.completedWithStats) }
}
log.error("Couldn't make tabs factory.")
return deferMaybe(FatalError(message: "Couldn't make tabs factory."))
}
/**
* This is a dedicated resetting interface that does both tabs and clients at the
* same time.
*/
open static func resetClientsAndTabsWithStorage(_ storage: ResettableSyncStorage, basePrefs: Prefs) -> Success {
let clientPrefs = BaseCollectionSynchronizer.prefsForCollection("clients", withBasePrefs: basePrefs)
let tabsPrefs = BaseCollectionSynchronizer.prefsForCollection("tabs", withBasePrefs: basePrefs)
clientPrefs.removeObjectForKey("lastFetched")
tabsPrefs.removeObjectForKey("lastFetched")
return storage.resetClient()
}
}
extension RemoteTab {
public func toDictionary() -> Dictionary<String, Any>? {
let tabHistory = history.flatMap { $0.absoluteString }
if tabHistory.isEmpty {
return nil
}
return [
"title": title,
"icon": icon?.absoluteString as Any? ?? NSNull(),
"urlHistory": tabHistory,
"lastUsed": millisecondsToDecimalSeconds(lastUsed)
]
}
}