Dactyloidae iOS initial commit

mobile/ios/Sync/SyncTelemetryUtils.swift

@@ -0,0 +1,356 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import Foundation
+import Shared
+import Account
+import Storage
+import SwiftyJSON
+import SyncTelemetry
+import Deferred
+
+fileprivate let log = Logger.syncLogger
+
+public let PrefKeySyncEvents = "sync.telemetry.events"
+
+public enum SyncReason: String {
+    case startup = "startup"
+    case scheduled = "scheduled"
+    case backgrounded = "backgrounded"
+    case user = "user"
+    case syncNow = "syncNow"
+    case didLogin = "didLogin"
+    case push = "push"
+}
+
+public enum SyncPingReason: String {
+    case shutdown = "shutdown"
+    case schedule = "schedule"
+    case idChanged = "idchanged"
+}
+
+public protocol Stats {
+    func hasData() -> Bool
+}
+
+private protocol DictionaryRepresentable {
+    func asDictionary() -> [String: Any]
+}
+
+public struct SyncUploadStats: Stats {
+    var sent: Int = 0
+    var sentFailed: Int = 0
+
+    public func hasData() -> Bool {
+        return sent > 0 || sentFailed > 0
+    }
+}
+
+extension SyncUploadStats: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        return [
+            "sent": sent,
+            "sentFailed": sentFailed
+        ]
+    }
+}
+
+public struct SyncDownloadStats: Stats {
+    var applied: Int = 0
+    var succeeded: Int = 0
+    var failed: Int = 0
+    var newFailed: Int = 0
+    var reconciled: Int = 0
+
+    public func hasData() -> Bool {
+        return applied > 0 ||
+               succeeded > 0 ||
+               failed > 0 ||
+               newFailed > 0 ||
+               reconciled > 0
+    }
+}
+
+extension SyncDownloadStats: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        return [
+            "applied": applied,
+            "succeeded": succeeded,
+            "failed": failed,
+            "newFailed": newFailed,
+            "reconciled": reconciled
+        ]
+    }
+}
+
+public struct ValidationStats: Stats, DictionaryRepresentable {
+    let problems: [ValidationProblem]
+    let took: Int64
+    let checked: Int?
+
+    public func hasData() -> Bool {
+        return !problems.isEmpty
+    }
+
+    func asDictionary() -> [String: Any] {
+        var dict: [String: Any] = [
+            "problems": problems.map { $0.asDictionary() },
+            "took": took
+        ]
+        if let checked = self.checked {
+            dict["checked"] = checked
+        }
+        return dict
+    }
+}
+
+public struct ValidationProblem: DictionaryRepresentable {
+    let name: String
+    let count: Int
+
+    func asDictionary() -> [String: Any] {
+        return ["name": name, "count": count]
+    }
+}
+
+public class StatsSession {
+    var took: Int64 = 0
+    var when: Timestamp?
+
+    private var startUptimeNanos: UInt64?
+
+    public func start(when: UInt64 = Date.now()) {
+        self.when = when
+        self.startUptimeNanos = DispatchTime.now().uptimeNanoseconds
+    }
+
+    public func hasStarted() -> Bool {
+        return startUptimeNanos != nil
+    }
+
+    public func end() -> Self {
+        guard let startUptime = startUptimeNanos else {
+            assertionFailure("SyncOperationStats called end without first calling start!")
+            return self
+        }
+
+        // Casting to Int64 should be safe since we're using uptime since boot in both cases.
+        // Convert to milliseconds as stated in the sync ping format
+        took = (Int64(DispatchTime.now().uptimeNanoseconds) - Int64(startUptime)) / 1000000
+        return self
+    }
+}
+
+// Stats about a single engine's sync.
+public class SyncEngineStatsSession: StatsSession {
+    public var validationStats: ValidationStats?
+
+    private(set) var uploadStats: SyncUploadStats
+    private(set) var downloadStats: SyncDownloadStats
+
+    public init(collection: String) {
+        self.uploadStats = SyncUploadStats()
+        self.downloadStats = SyncDownloadStats()
+    }
+
+    public func recordDownload(stats: SyncDownloadStats) {
+        self.downloadStats.applied += stats.applied
+        self.downloadStats.succeeded += stats.succeeded
+        self.downloadStats.failed += stats.failed
+        self.downloadStats.newFailed += stats.newFailed
+        self.downloadStats.reconciled += stats.reconciled
+    }
+
+    public func recordUpload(stats: SyncUploadStats) {
+        self.uploadStats.sent += stats.sent
+        self.uploadStats.sentFailed += stats.sentFailed
+    }
+}
+
+extension SyncEngineStatsSession: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        var dict: [String: Any] = [
+            "took": took,
+        ]
+
+        if downloadStats.hasData() {
+            dict["incoming"] = downloadStats.asDictionary()
+        }
+
+        if uploadStats.hasData() {
+            dict["outgoing"] = uploadStats.asDictionary()
+        }
+
+        if let validation = self.validationStats, validation.hasData() {
+            dict["validation"] = validation.asDictionary()
+        }
+
+        return dict
+    }
+}
+
+// Stats and metadata for a sync operation.
+public class SyncOperationStatsSession: StatsSession {
+    public let why: SyncReason
+    public var uid: String?
+    public var deviceID: String?
+
+    fileprivate let didLogin: Bool
+
+    public init(why: SyncReason, uid: String, deviceID: String?) {
+        self.why = why
+        self.uid = uid
+        self.deviceID = deviceID
+        self.didLogin = (why == .didLogin)
+    }
+}
+
+extension SyncOperationStatsSession: DictionaryRepresentable {
+    func asDictionary() -> [String: Any] {
+        let whenValue = when ?? 0
+        return [
+            "when": whenValue,
+            "took": took,
+            "didLogin": didLogin,
+            "why": why.rawValue
+        ]
+    }
+}
+
+public enum SyncPingError: MaybeErrorType {
+    case failedToRestoreScratchpad
+
+    public var description: String {
+        switch self {
+        case .failedToRestoreScratchpad: return "Failed to restore Scratchpad from prefs"
+        }
+    }
+}
+
+public enum SyncPingFailureReasonName: String {
+    case httpError = "httperror"
+    case unexpectedError = "unexpectederror"
+    case sqlError = "sqlerror"
+    case otherError = "othererror"
+}
+
+public protocol SyncPingFailureFormattable {
+    var failureReasonName: SyncPingFailureReasonName { get }
+}
+
+public struct SyncPing: SyncTelemetryPing {
+    public private(set) var payload: JSON
+
+    public static func from(result: SyncOperationResult,
+                            account: FirefoxAccount,
+                            remoteClientsAndTabs: RemoteClientsAndTabs,
+                            prefs: Prefs,
+                            why: SyncPingReason) -> Deferred<Maybe<SyncPing>> {
+        // Grab our token so we can use the hashed_fxa_uid and clientGUID from our scratchpad for
+        // our ping's identifiers
+        return account.syncAuthState.token(Date.now(), canBeExpired: false) >>== { (token, kB) in
+            let scratchpadPrefs = prefs.branch("sync.scratchpad")
+            guard let scratchpad = Scratchpad.restoreFromPrefs(scratchpadPrefs, syncKeyBundle: KeyBundle.fromKB(kB)) else {
+                return deferMaybe(SyncPingError.failedToRestoreScratchpad)
+            }
+
+            var ping: [String: Any] = [
+                "version": 1,
+                "why": why.rawValue,
+                "uid": token.hashedFxAUID,
+                "deviceID": (scratchpad.clientGUID + token.hashedFxAUID).sha256.hexEncodedString
+            ]
+
+            // TODO: We don't cache our sync pings so if it fails, it fails. Once we add
+            // some kind of caching we'll want to make sure we don't dump the events if
+            // the ping has failed.
+            let pickledEvents = prefs.arrayForKey(PrefKeySyncEvents) as? [Data] ?? []
+            let events = pickledEvents.flatMap(Event.unpickle).map { $0.toArray() }
+            ping["events"] = events
+            prefs.setObject(nil, forKey: PrefKeySyncEvents)
+
+            return dictionaryFrom(result: result, storage: remoteClientsAndTabs, token: token) >>== { syncDict in
+                // TODO: Split the sync ping metadata from storing a single sync.
+                ping["syncs"] = [syncDict]
+                return deferMaybe(SyncPing(payload: JSON(ping)))
+            }
+        }
+    }
+
+    // Generates a single sync ping payload that is stored in the 'syncs' list in the sync ping.
+    private static func dictionaryFrom(result: SyncOperationResult,
+                                       storage: RemoteClientsAndTabs,
+                                       token: TokenServerToken) -> Deferred<Maybe<[String: Any]>> {
+        return connectedDevices(fromStorage: storage, token: token) >>== { devices in
+            guard let stats = result.stats else {
+                return deferMaybe([String: Any]())
+            }
+
+            var dict = stats.asDictionary()
+            if let engineResults = result.engineResults.successValue {
+                dict["engines"] = SyncPing.enginePingDataFrom(engineResults: engineResults)
+            } else if let failure = result.engineResults.failureValue {
+                var errorName: SyncPingFailureReasonName
+                if let formattableFailure = failure as? SyncPingFailureFormattable {
+                    errorName = formattableFailure.failureReasonName
+                } else {
+                    errorName = .unexpectedError
+                }
+
+                dict["failureReason"] = [
+                    "name": errorName.rawValue,
+                    "error": "\(type(of: failure))",
+                ]
+            }
+
+            dict["devices"] = devices
+            return deferMaybe(dict)
+        }
+    }
+
+    // Returns a list of connected devices formatted for use in the 'devices' property in the sync ping.
+    private static func connectedDevices(fromStorage storage: RemoteClientsAndTabs,
+                                         token: TokenServerToken) -> Deferred<Maybe<[[String: Any]]>> {
+        func dictionaryFrom(client: RemoteClient) -> [String: Any]? {
+            var device = [String: Any]()
+            if let os = client.os {
+                device["os"] = os
+            }
+            if let version = client.version {
+                device["version"] = version
+            }
+            if let guid = client.guid {
+                device["id"] = (guid + token.hashedFxAUID).sha256.hexEncodedString
+            }
+            return device
+        }
+
+        return storage.getClients() >>== { deferMaybe($0.flatMap(dictionaryFrom)) }
+    }
+
+    private static func enginePingDataFrom(engineResults: EngineResults) -> [[String: Any]] {
+        return engineResults.map { result in
+            let (name, status) = result
+            var engine: [String: Any] = [
+                "name": name
+            ]
+
+            // For complete/partial results, extract out the collect stats
+            // and add it to engine information. For syncs that were not able to
+            // start, return why and a reason.
+            switch status {
+            case .completed(let stats):
+                engine.merge(with: stats.asDictionary())
+            case .partial(let stats):
+                engine.merge(with: stats.asDictionary())
+            case .notStarted(let reason):
+                engine.merge(with: [
+                    "status": reason.telemetryId
+                ])
+            }
+
+            return engine
+        }
+    }
+}