/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import Foundation import Shared import Account import Storage import SwiftyJSON import SyncTelemetry import Deferred fileprivate let log = Logger.syncLogger public let PrefKeySyncEvents = "sync.telemetry.events" public enum SyncReason: String { case startup = "startup" case scheduled = "scheduled" case backgrounded = "backgrounded" case user = "user" case syncNow = "syncNow" case didLogin = "didLogin" case push = "push" } public enum SyncPingReason: String { case shutdown = "shutdown" case schedule = "schedule" case idChanged = "idchanged" } public protocol Stats { func hasData() -> Bool } private protocol DictionaryRepresentable { func asDictionary() -> [String: Any] } public struct SyncUploadStats: Stats { var sent: Int = 0 var sentFailed: Int = 0 public func hasData() -> Bool { return sent > 0 || sentFailed > 0 } } extension SyncUploadStats: DictionaryRepresentable { func asDictionary() -> [String: Any] { return [ "sent": sent, "sentFailed": sentFailed ] } } public struct SyncDownloadStats: Stats { var applied: Int = 0 var succeeded: Int = 0 var failed: Int = 0 var newFailed: Int = 0 var reconciled: Int = 0 public func hasData() -> Bool { return applied > 0 || succeeded > 0 || failed > 0 || newFailed > 0 || reconciled > 0 } } extension SyncDownloadStats: DictionaryRepresentable { func asDictionary() -> [String: Any] { return [ "applied": applied, "succeeded": succeeded, "failed": failed, "newFailed": newFailed, "reconciled": reconciled ] } } public struct ValidationStats: Stats, DictionaryRepresentable { let problems: [ValidationProblem] let took: Int64 let checked: Int? public func hasData() -> Bool { return !problems.isEmpty } func asDictionary() -> [String: Any] { var dict: [String: Any] = [ "problems": problems.map { $0.asDictionary() }, "took": took ] if let checked = self.checked { dict["checked"] = checked } return dict } } public struct ValidationProblem: DictionaryRepresentable { let name: String let count: Int func asDictionary() -> [String: Any] { return ["name": name, "count": count] } } public class StatsSession { var took: Int64 = 0 var when: Timestamp? private var startUptimeNanos: UInt64? public func start(when: UInt64 = Date.now()) { self.when = when self.startUptimeNanos = DispatchTime.now().uptimeNanoseconds } public func hasStarted() -> Bool { return startUptimeNanos != nil } public func end() -> Self { guard let startUptime = startUptimeNanos else { assertionFailure("SyncOperationStats called end without first calling start!") return self } // Casting to Int64 should be safe since we're using uptime since boot in both cases. // Convert to milliseconds as stated in the sync ping format took = (Int64(DispatchTime.now().uptimeNanoseconds) - Int64(startUptime)) / 1000000 return self } } // Stats about a single engine's sync. public class SyncEngineStatsSession: StatsSession { public var validationStats: ValidationStats? private(set) var uploadStats: SyncUploadStats private(set) var downloadStats: SyncDownloadStats public init(collection: String) { self.uploadStats = SyncUploadStats() self.downloadStats = SyncDownloadStats() } public func recordDownload(stats: SyncDownloadStats) { self.downloadStats.applied += stats.applied self.downloadStats.succeeded += stats.succeeded self.downloadStats.failed += stats.failed self.downloadStats.newFailed += stats.newFailed self.downloadStats.reconciled += stats.reconciled } public func recordUpload(stats: SyncUploadStats) { self.uploadStats.sent += stats.sent self.uploadStats.sentFailed += stats.sentFailed } } extension SyncEngineStatsSession: DictionaryRepresentable { func asDictionary() -> [String: Any] { var dict: [String: Any] = [ "took": took, ] if downloadStats.hasData() { dict["incoming"] = downloadStats.asDictionary() } if uploadStats.hasData() { dict["outgoing"] = uploadStats.asDictionary() } if let validation = self.validationStats, validation.hasData() { dict["validation"] = validation.asDictionary() } return dict } } // Stats and metadata for a sync operation. public class SyncOperationStatsSession: StatsSession { public let why: SyncReason public var uid: String? public var deviceID: String? fileprivate let didLogin: Bool public init(why: SyncReason, uid: String, deviceID: String?) { self.why = why self.uid = uid self.deviceID = deviceID self.didLogin = (why == .didLogin) } } extension SyncOperationStatsSession: DictionaryRepresentable { func asDictionary() -> [String: Any] { let whenValue = when ?? 0 return [ "when": whenValue, "took": took, "didLogin": didLogin, "why": why.rawValue ] } } public enum SyncPingError: MaybeErrorType { case failedToRestoreScratchpad public var description: String { switch self { case .failedToRestoreScratchpad: return "Failed to restore Scratchpad from prefs" } } } public enum SyncPingFailureReasonName: String { case httpError = "httperror" case unexpectedError = "unexpectederror" case sqlError = "sqlerror" case otherError = "othererror" } public protocol SyncPingFailureFormattable { var failureReasonName: SyncPingFailureReasonName { get } } public struct SyncPing: SyncTelemetryPing { public private(set) var payload: JSON public static func from(result: SyncOperationResult, account: FirefoxAccount, remoteClientsAndTabs: RemoteClientsAndTabs, prefs: Prefs, why: SyncPingReason) -> Deferred> { // Grab our token so we can use the hashed_fxa_uid and clientGUID from our scratchpad for // our ping's identifiers return account.syncAuthState.token(Date.now(), canBeExpired: false) >>== { (token, kB) in let scratchpadPrefs = prefs.branch("sync.scratchpad") guard let scratchpad = Scratchpad.restoreFromPrefs(scratchpadPrefs, syncKeyBundle: KeyBundle.fromKB(kB)) else { return deferMaybe(SyncPingError.failedToRestoreScratchpad) } var ping: [String: Any] = [ "version": 1, "why": why.rawValue, "uid": token.hashedFxAUID, "deviceID": (scratchpad.clientGUID + token.hashedFxAUID).sha256.hexEncodedString ] // TODO: We don't cache our sync pings so if it fails, it fails. Once we add // some kind of caching we'll want to make sure we don't dump the events if // the ping has failed. let pickledEvents = prefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? [] let events = pickledEvents.flatMap(Event.unpickle).map { $0.toArray() } ping["events"] = events prefs.setObject(nil, forKey: PrefKeySyncEvents) return dictionaryFrom(result: result, storage: remoteClientsAndTabs, token: token) >>== { syncDict in // TODO: Split the sync ping metadata from storing a single sync. ping["syncs"] = [syncDict] return deferMaybe(SyncPing(payload: JSON(ping))) } } } // Generates a single sync ping payload that is stored in the 'syncs' list in the sync ping. private static func dictionaryFrom(result: SyncOperationResult, storage: RemoteClientsAndTabs, token: TokenServerToken) -> Deferred> { return connectedDevices(fromStorage: storage, token: token) >>== { devices in guard let stats = result.stats else { return deferMaybe([String: Any]()) } var dict = stats.asDictionary() if let engineResults = result.engineResults.successValue { dict["engines"] = SyncPing.enginePingDataFrom(engineResults: engineResults) } else if let failure = result.engineResults.failureValue { var errorName: SyncPingFailureReasonName if let formattableFailure = failure as? SyncPingFailureFormattable { errorName = formattableFailure.failureReasonName } else { errorName = .unexpectedError } dict["failureReason"] = [ "name": errorName.rawValue, "error": "\(type(of: failure))", ] } dict["devices"] = devices return deferMaybe(dict) } } // Returns a list of connected devices formatted for use in the 'devices' property in the sync ping. private static func connectedDevices(fromStorage storage: RemoteClientsAndTabs, token: TokenServerToken) -> Deferred> { func dictionaryFrom(client: RemoteClient) -> [String: Any]? { var device = [String: Any]() if let os = client.os { device["os"] = os } if let version = client.version { device["version"] = version } if let guid = client.guid { device["id"] = (guid + token.hashedFxAUID).sha256.hexEncodedString } return device } return storage.getClients() >>== { deferMaybe($0.flatMap(dictionaryFrom)) } } private static func enginePingDataFrom(engineResults: EngineResults) -> [[String: Any]] { return engineResults.map { result in let (name, status) = result var engine: [String: Any] = [ "name": name ] // For complete/partial results, extract out the collect stats // and add it to engine information. For syncs that were not able to // start, return why and a reason. switch status { case .completed(let stats): engine.merge(with: stats.asDictionary()) case .partial(let stats): engine.merge(with: stats.asDictionary()) case .notStarted(let reason): engine.merge(with: [ "status": reason.telemetryId ]) } return engine } } }