Dactyloidae iOS initial commit

mobile/ios/Client/Assets/Search/scrape_plugins.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from lxml import html
+from lxml import etree
+import copy
+import json
+import os
+import requests
+import shutil
+import subprocess
+import urllib
+
+# Paths for en-US plugins included in the core Android repo.
+EN_PLUGINS_DIR_URL = "https://hg.mozilla.org/releases/mozilla-aurora/file/default/mobile/locales/en-US/searchplugins"
+EN_PLUGINS_FILE_URL = "https://hg.mozilla.org/releases/mozilla-aurora/raw-file/default/mobile/locales/en-US/searchplugins/%s"
+EN_PREFS_URL = "https://hg.mozilla.org/releases/mozilla-aurora/raw-file/default/mobile/locales/en-US/chrome/region.properties"
+
+# Paths for plugins in the l10n repos.
+L10N_LOCALE_LIST_URL = "https://hg.mozilla.org/releases/mozilla-aurora/raw-file/default/mobile/android/locales/all-locales"
+L10N_PLUGINS_DIR_URL = "https://hg.mozilla.org/releases/l10n/mozilla-aurora/%s/file/default/mobile/searchplugins"
+L10N_PLUGINS_FILE_URL = "https://hg.mozilla.org/releases/l10n/mozilla-aurora/%s/raw-file/default/mobile/searchplugins/%%s"
+L10N_PREFS_URL = "https://hg.mozilla.org/releases/l10n/mozilla-aurora/%s/raw-file/default/mobile/chrome/region.properties"
+
+MOZ_HEADER = """\
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+
+"""
+
+ns = { "search": "http://www.mozilla.org/2006/browser/search/" }
+
+def main():
+    # Remove and recreate the SearchPlugins directory.
+    if os.path.exists("SearchPlugins"):
+        shutil.rmtree("SearchPlugins")
+    os.makedirs("SearchPlugins")
+
+    # Import en-US engines from the core repo.
+    downloadLocale("en", EnScraper())
+
+    # Import engines from the l10n repos.
+    response = requests.get(L10N_LOCALE_LIST_URL)
+    locales = response.text.strip().split("\n")
+    supportedLocales = getSupportedLocales()
+    for locale in locales:
+        if (locale not in supportedLocales):
+            print("skipping unsupported locale: %s" % locale)
+            continue
+
+        downloadLocale(locale, L10nScraper(locale))
+
+    verifyEngines()
+
+def downloadLocale(locale, scraper):
+    print("scraping: %s..." % locale)
+    files = scraper.getFileList()
+    if files == None:
+        print("no files for locale: %s" % locale)
+        return
+
+    print("  found search plugins")
+
+    directory = os.path.join("SearchPlugins", locale)
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+    # Get the default search engine for this locale.
+    default = scraper.getDefault()
+    print("  default: %s" % default)
+    saveDefault(locale, default)
+
+    for file in files:
+        path = os.path.join(directory, file)
+        print("  downloading: %s..." % file)
+        downloadedFile = scraper.getFile(file)
+        name, extension = os.path.splitext(file)
+
+        # Apply iOS-specific overlays for this engine if they are defined.
+        if extension == ".xml":
+            engine = name.split("-")[0]
+            overlay = overlayForEngine(engine)
+            if overlay:
+                plugin = etree.parse(downloadedFile)
+                overlay.apply(plugin)
+                contents = MOZ_HEADER + etree.tostring(plugin.getroot(), encoding="utf-8", pretty_print=True)
+                with open(path, "w") as outfile:
+                    outfile.write(contents)
+                continue
+
+        # Otherwise, just use the downloaded file as is.
+        shutil.move(downloadedFile, path)
+
+def verifyEngines():
+    print("verifying engines...")
+    enDir = os.path.join("SearchPlugins", "en")
+    for locale in os.listdir("SearchPlugins"):
+        if locale.startswith("."): continue
+        localeDir = os.path.join("SearchPlugins", locale)
+        with open(os.path.join(localeDir, "list.txt")) as f:
+            engineList = f.read().splitlines()
+
+        for engine in engineList:
+            if engine.endswith(":hidden"): continue
+            path = os.path.join(localeDir, engine + ".xml")
+            enPath = os.path.join(enDir, engine + ".xml")
+            if not os.path.exists(path) and not os.path.exists(enPath):
+                print("  ERROR: missing engine %s for locale %s" % (engine, locale))
+
+def getSupportedLocales():
+    supportedLocales = subprocess.Popen("./get_supported_locales.swift", stdout=subprocess.PIPE).communicate()[0]
+    return json.loads(supportedLocales.replace("_", "-"))
+
+def overlayForEngine(engine):
+    path = os.path.join("SearchOverlays", "%s.xml" % engine)
+    if not os.path.exists(path):
+        return None
+    return Overlay(path)
+
+def saveDefault(locale, default):
+    directory = os.path.join("SearchPlugins", locale, "default.txt")
+    file = open(directory, "w")
+    file.write(default.encode("UTF-8"))
+
+
+class Scraper:
+    def pluginsDirURL(self): pass
+    def pluginsFileURL(self): pass
+    def prefsURL(self): pass
+    def defaultPrefName(self): pass
+
+    def getFileList(self):
+        response = requests.get(self.pluginsDirURL)
+        if not response.ok:
+            raise Exception("error: could not read plugins directory")
+
+        tree = html.fromstring(response.content)
+        return tree.xpath('//a[@class="list"]/text()')
+
+    def getFile(self, file):
+        result = urllib.urlretrieve(self.pluginsFileURL % file)
+        return result[0]
+
+    def getDefault(self):
+        response = requests.get(self.prefsURL)
+        if not response.ok:
+            raise Exception("error: could not read prefs file")
+
+        lines = response.text.strip().split("\n")
+        for line in lines:
+            values = line.strip().split("=")
+            if len(values) == 2 and values[0].strip() == self.defaultPrefName:
+                default = values[1].strip()
+                return default
+
+        raise Exception("error: no default pref found")
+
+
+class L10nScraper(Scraper):
+    def __init__(self, locale):
+        self.pluginsDirURL = L10N_PLUGINS_DIR_URL % locale
+        self.pluginsFileURL = L10N_PLUGINS_FILE_URL % locale
+        self.prefsURL = L10N_PREFS_URL % locale
+        self.defaultPrefName = "browser.search.defaultenginename"
+
+
+class EnScraper(Scraper):
+    def __init__(self):
+        self.pluginsDirURL = EN_PLUGINS_DIR_URL
+        self.pluginsFileURL = EN_PLUGINS_FILE_URL
+        self.prefsURL = EN_PREFS_URL
+        self.defaultPrefName = "browser.search.defaultenginename.US"
+
+class Overlay:
+    def __init__(self, path):
+        overlay = etree.parse(path)
+        self.actions = overlay.getroot().getchildren()
+
+    def apply(self, doc):
+        for action in self.actions:
+            if action.tag == "replace":
+                self.replace(target=action.get("target"), replacement=action[0], doc=doc)
+            elif action.tag == "append":
+                self.append(parent=action.get("parent"), child=action[0], doc=doc)
+
+    def replace(self, target, replacement, doc):
+        for element in doc.xpath(target, namespaces=ns):
+            replacementCopy = copy.deepcopy(replacement)
+            element.getparent().replace(element, replacementCopy)
+
+            # Try to preserve indentation.
+            replacementCopy.tail = element.tail
+
+    def append(self, parent, child, doc):
+        for element in doc.xpath(parent, namespaces=ns):
+            childCopy = copy.deepcopy(child)
+            element.append(childCopy)
+
+            # Try to preserve indentation.
+            childCopy.tail = "\n"
+            previous = childCopy.getprevious()
+            if previous is not None:
+                childCopy.tail = previous.tail
+                prevPrevious = previous.getprevious()
+                if prevPrevious is not None:
+                    previous.tail = prevPrevious.tail
+
+
+if __name__ == "__main__":
+        main()