/* This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ import Storage import Shared import Alamofire import XCGLogger import Deferred import SDWebImage import Fuzi import SwiftyJSON private let log = Logger.browserLogger private let queue = DispatchQueue(label: "FaviconFetcher", attributes: DispatchQueue.Attributes.concurrent) class FaviconFetcherErrorType: MaybeErrorType { let description: String init(description: String) { self.description = description } } /* A helper class to find the favicon associated with a URL. * This will load the page and parse any icons it finds out of it. * If that fails, it will attempt to find a favicon.ico in the root host domain. */ open class FaviconFetcher: NSObject, XMLParserDelegate { open static var userAgent: String = "" static let ExpirationTime = TimeInterval(60*60*24*7) // Only check for icons once a week fileprivate static var characterToFaviconCache = [String: UIImage]() static var defaultFavicon: UIImage = { return UIImage(named: "defaultFavicon")! }() static var colors: [String: UIColor] = [:] //An in-Memory data store that stores background colors domains. Stored using url.baseDomain. // Sites can be accessed via their baseDomain. static var defaultIcons: [String: (color: UIColor, url: String)] = { return FaviconFetcher.getDefaultIcons() }() static let multiRegionDomains = ["craigslist", "google", "amazon"] class func getDefaultIconForURL(url: URL) -> (color: UIColor, url: String)? { // Problem: Sites like amazon exist with .ca/.de and many other tlds. // Solution: They are stored in the default icons list as "amazon" instead of "amazon.com" this allows us to have favicons for every tld." // Here, If the site is in the multiRegionDomain array look it up via its second level domain (amazon) instead of its baseDomain (amazon.com) let hostName = url.hostSLD if multiRegionDomains.contains(hostName), let icon = defaultIcons[hostName] { return icon } if let name = url.baseDomain, let icon = defaultIcons[name] { return icon } return nil } class func getForURL(_ url: URL, profile: Profile) -> Deferred> { let f = FaviconFetcher() return f.loadFavicons(url, profile: profile) } fileprivate func loadFavicons(_ url: URL, profile: Profile, oldIcons: [Favicon] = [Favicon]()) -> Deferred> { if isIgnoredURL(url) { return deferMaybe(FaviconFetcherErrorType(description: "Not fetching ignored URL to find favicons.")) } let deferred = Deferred>() var oldIcons: [Favicon] = oldIcons queue.async { _ in self.parseHTMLForFavicons(url).bind({ (result: Maybe<[Favicon]>) -> Deferred<[Maybe]> in var deferreds = [Deferred>]() if let icons = result.successValue { deferreds = icons.map { self.getFavicon(url, icon: $0, profile: profile) } } return all(deferreds) }).bind({ (results: [Maybe]) -> Deferred> in for result in results { if let icon = result.successValue { oldIcons.append(icon) } } oldIcons = oldIcons.sorted { return $0.width! > $1.width! } return deferMaybe(oldIcons) }).upon({ (result: Maybe<[Favicon]>) in deferred.fill(result) return }) } return deferred } lazy fileprivate var alamofire: SessionManager = { let configuration = URLSessionConfiguration.default var defaultHeaders = SessionManager.default.session.configuration.httpAdditionalHeaders ?? [:] defaultHeaders["User-Agent"] = userAgent configuration.httpAdditionalHeaders = defaultHeaders configuration.timeoutIntervalForRequest = 5 return SessionManager(configuration: configuration) }() fileprivate func fetchDataForURL(_ url: URL) -> Deferred> { let deferred = Deferred>() alamofire.request(url).response { response in // Don't cancel requests just because our Manager is deallocated. withExtendedLifetime(self.alamofire) { if response.error == nil { if let data = response.data { deferred.fill(Maybe(success: data)) return } } let errorDescription = (response.error as NSError?)?.description ?? "No content." deferred.fill(Maybe(failure: FaviconFetcherErrorType(description: errorDescription))) } } return deferred } ///Users/fpatel/Documents/firefox-ios/Client/Utils/FaviconFetcher.swift:117:77: Cannot convert value of type 'String' to expected argument type 'String.Index' (aka 'String.CharacterView.Index') // Loads and parses an html document and tries to find any known favicon-type tags for the page fileprivate func parseHTMLForFavicons(_ url: URL) -> Deferred> { return fetchDataForURL(url).bind({ result -> Deferred> in var icons = [Favicon]() guard let data = result.successValue, result.isSuccess, let root = try? HTMLDocument(data: data as Data) else { return deferMaybe([]) } var reloadUrl: URL? = nil for meta in root.xpath("//head/meta") { if let refresh = meta["http-equiv"], refresh == "Refresh", let content = meta["content"], let index = content.range(of: "URL="), let url = NSURL(string: content.substring(from: index.upperBound)) { reloadUrl = url as URL } } if let url = reloadUrl { return self.parseHTMLForFavicons(url) } var bestType = IconType.noneFound for link in root.xpath("//head//link[contains(@rel, 'icon')]") { var iconType: IconType? = nil if let rel = link["rel"] { switch rel { case "shortcut icon": iconType = .icon case "icon": iconType = .icon case "apple-touch-icon": iconType = .appleIcon case "apple-touch-icon-precomposed": iconType = .appleIconPrecomposed default: iconType = nil } } guard let href = link["href"], iconType != nil else { continue //Skip the rest of the loop. But don't stop the loop } if href.endsWith(".ico") { iconType = .guess } if let type = iconType, !bestType.isPreferredTo(type), let iconUrl = NSURL(string: href, relativeTo: url as URL), let absoluteString = iconUrl.absoluteString { let icon = Favicon(url: absoluteString, date: NSDate() as Date, type: type) // If we already have a list of Favicons going already, then add it… if type == bestType { icons.append(icon) } else { // otherwise, this is the first in a new best yet type. icons = [icon] bestType = type } } // If we haven't got any options icons, then use the default at the root of the domain. if let url = NSURL(string: "/favicon.ico", relativeTo: url as URL), icons.isEmpty, let absoluteString = url.absoluteString { let icon = Favicon(url: absoluteString, date: NSDate() as Date, type: .guess) icons = [icon] } } return deferMaybe(icons) }) } func getFavicon(_ siteUrl: URL, icon: Favicon, profile: Profile) -> Deferred> { let deferred = Deferred>() let url = icon.url let manager = SDWebImageManager.shared() let site = Site(url: siteUrl.absoluteString, title: "") var fav = Favicon(url: url, type: icon.type) if let url = url.asURL { var fetch: SDWebImageOperation? fetch = manager.loadImage(with: url, options: .lowPriority, progress: { (receivedSize, expectedSize, _) in if receivedSize > FaviconManager.maximumFaviconSize || expectedSize > FaviconManager.maximumFaviconSize { fetch?.cancel() } }, completed: { (img, _, _, _, _, url) in guard let url = url else { deferred.fill(Maybe(failure: FaviconError())) return } fav = Favicon(url: url.absoluteString, type: icon.type) if let img = img { fav.width = Int(img.size.width) fav.height = Int(img.size.height) profile.favicons.addFavicon(fav, forSite: site) } else { fav.width = 0 fav.height = 0 } deferred.fill(Maybe(success: fav)) }) } else { return deferMaybe(FaviconFetcherErrorType(description: "Invalid URL \(url)")) } return deferred } // Returns a single Favicon UIImage for a given URL class func fetchFavImageForURL(forURL url: URL, profile: Profile) -> Deferred> { let deferred = Deferred>() FaviconFetcher.getForURL(url.domainURL, profile: profile).uponQueue(DispatchQueue.main) { result in var iconURL: URL? if let favicons = result.successValue, favicons.count > 0, let faviconImageURL = favicons.first?.url.asURL { iconURL = faviconImageURL } else { return deferred.fill(Maybe(failure: FaviconError())) } SDWebImageManager.shared().loadImage(with: iconURL, options: .continueInBackground, progress: nil) { (image, _, _, _, _, _) in if let image = image { deferred.fill(Maybe(success: image)) } else { deferred.fill(Maybe(failure: FaviconError())) } } } return deferred } // Returns the default favicon for a site based on the first letter of the site's domain class func getDefaultFavicon(_ url: URL) -> UIImage { guard let character = url.baseDomain?.characters.first else { return defaultFavicon } let faviconLetter = String(character).uppercased() if let cachedFavicon = characterToFaviconCache[faviconLetter] { return cachedFavicon } var faviconImage = UIImage() let faviconLabel = UILabel(frame: CGRect(x: 0, y: 0, width: TwoLineCellUX.ImageSize, height: TwoLineCellUX.ImageSize)) faviconLabel.text = faviconLetter faviconLabel.textAlignment = .center faviconLabel.font = UIFont.systemFont(ofSize: 18, weight: UIFontWeightMedium) faviconLabel.textColor = UIColor.white UIGraphicsBeginImageContextWithOptions(faviconLabel.bounds.size, false, 0.0) faviconLabel.layer.render(in: UIGraphicsGetCurrentContext()!) faviconImage = UIGraphicsGetImageFromCurrentImageContext()! UIGraphicsEndImageContext() characterToFaviconCache[faviconLetter] = faviconImage return faviconImage } // Returns a color based on the url's hash class func getDefaultColor(_ url: URL) -> UIColor { guard let hash = url.baseDomain?.hashValue else { return UIColor.gray } let index = abs(hash) % (UIConstants.DefaultColorStrings.count - 1) let colorHex = UIConstants.DefaultColorStrings[index] return UIColor(colorString: colorHex) } // Default favicons and background colors provided via mozilla/tippy-top-sites class func getDefaultIcons() -> [String: (color: UIColor, url: String)] { let filePath = Bundle.main.path(forResource: "top_sites", ofType: "json") let file = try! Data(contentsOf: URL(fileURLWithPath: filePath!)) let json = JSON(file) var icons: [String: (color: UIColor, url: String)] = [:] json.forEach({ guard let url = $0.1["domain"].string, let color = $0.1["background_color"].string, var path = $0.1["image_url"].string else { return } path = path.replacingOccurrences(of: ".png", with: "") let filePath = Bundle.main.path(forResource: "TopSites/" + path, ofType: "png") if let filePath = filePath { if color == "#fff" || color == "#FFF" { icons[url] = (UIColor.clear, filePath) } else { icons[url] = (UIColor(colorString: color.replacingOccurrences(of: "#", with: "")), filePath) } } }) return icons } }