import FIREFOX_52_6_0esr_RELEASE from mozilla-esr52 hg repo

testing/mozharness/scripts/release/uptake_monitoring.py

@@ -0,0 +1,188 @@
+#!/usr/bin/env python
+# lint_ignore=E501
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+""" uptake_monitoring.py
+
+A script to replace the old-fashion way of computing the uptake monitoring
+from the scheduler within the slaves.
+"""
+
+import os
+import sys
+import datetime
+import time
+import xml.dom.minidom
+
+sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
+
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
+from mozharness.base.script import BaseScript
+from mozharness.mozilla.buildbot import BuildbotMixin
+
+
+def get_tuxedo_uptake_url(tuxedo_server_url, related_product, os):
+    return '%s/uptake/?product=%s&os=%s' % (tuxedo_server_url,
+                                            related_product, os)
+
+
+class UptakeMonitoring(BaseScript, VirtualenvMixin, BuildbotMixin):
+    config_options = virtualenv_config_options
+
+    def __init__(self, require_config_file=True):
+        super(UptakeMonitoring, self).__init__(
+            config_options=self.config_options,
+            require_config_file=require_config_file,
+            config={
+                "virtualenv_modules": [
+                    "redo",
+                    "requests",
+                ],
+
+                "virtualenv_path": "venv",
+                "credentials_file": "oauth.txt",
+                "buildbot_json_path": "buildprops.json",
+                "poll_interval": 60,
+                "poll_timeout": 20*60,
+                "min_uptake": 10000,
+            },
+            all_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "monitor-uptake",
+            ],
+            default_actions=[
+                "create-virtualenv",
+                "activate-virtualenv",
+                "monitor-uptake",
+            ],
+        )
+
+    def _pre_config_lock(self, rw_config):
+        super(UptakeMonitoring, self)._pre_config_lock(rw_config)
+        # override properties from buildbot properties here as defined by
+        # taskcluster properties
+        self.read_buildbot_config()
+        if not self.buildbot_config:
+            self.warning("Skipping buildbot properties overrides")
+            return
+        props = self.buildbot_config["properties"]
+        for prop in ['tuxedo_server_url', 'version']:
+            if props.get(prop):
+                self.info("Overriding %s with %s" % (prop, props[prop]))
+                self.config[prop] = props.get(prop)
+            else:
+                self.warning("%s could not be found within buildprops" % prop)
+                return
+        partials = [v.strip() for v in props["partial_versions"].split(",")]
+        self.config["partial_versions"] = [v.split("build")[0] for v in partials]
+        self.config["platforms"] = [p.strip() for p in
+                                    props["platforms"].split(",")]
+
+    def _get_product_uptake(self, tuxedo_server_url, auth,
+                            related_product, os):
+        from redo import retry
+        import requests
+
+        url = get_tuxedo_uptake_url(tuxedo_server_url, related_product, os)
+        self.info("Requesting {} from tuxedo".format(url))
+
+        def get_tuxedo_page():
+            r = requests.get(url, auth=auth,
+                             verify=False, timeout=60)
+            r.raise_for_status()
+            return r.content
+
+        def calculateUptake(page):
+            doc = xml.dom.minidom.parseString(page)
+            uptake_values = []
+
+            for element in doc.getElementsByTagName('available'):
+                for node in element.childNodes:
+                    if node.nodeType == xml.dom.minidom.Node.TEXT_NODE and \
+                            node.data.isdigit():
+                        uptake_values.append(int(node.data))
+            if not uptake_values:
+                uptake_values = [0]
+            return min(uptake_values)
+
+        page = retry(get_tuxedo_page)
+        uptake = calculateUptake(page)
+        self.info("Current uptake for {} is {}".format(related_product, uptake))
+        return uptake
+
+    def _get_release_uptake(self, auth):
+        assert isinstance(self.config["platforms"], (list, tuple))
+
+        # handle the products first
+        tuxedo_server_url = self.config["tuxedo_server_url"]
+        version = self.config["version"]
+        dl = []
+
+        for product, info in self.config["products"].iteritems():
+            if info.get("check_uptake"):
+                product_template = info["product-name"]
+                related_product = product_template % {"version": version}
+
+                enUS_platforms = set(self.config["platforms"])
+                paths_platforms = set(info["paths"].keys())
+                platforms = enUS_platforms.intersection(paths_platforms)
+
+                for platform in platforms:
+                    bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
+                    dl.append(self._get_product_uptake(tuxedo_server_url, auth,
+                                                       related_product, bouncer_platform))
+        # handle the partials as well
+        prev_versions = self.config["partial_versions"]
+        for product, info in self.config["partials"].iteritems():
+            if info.get("check_uptake"):
+                product_template = info["product-name"]
+                for prev_version in prev_versions:
+                    subs = {
+                        "version": version,
+                        "prev_version": prev_version
+                    }
+                    related_product = product_template % subs
+
+                    enUS_platforms = set(self.config["platforms"])
+                    paths_platforms = set(info["paths"].keys())
+                    platforms = enUS_platforms.intersection(paths_platforms)
+
+                    for platform in platforms:
+                        bouncer_platform = info["paths"].get(platform).get('bouncer-platform')
+                        dl.append(self._get_product_uptake(tuxedo_server_url, auth,
+                                                           related_product, bouncer_platform))
+        return min(dl)
+
+    def monitor_uptake(self):
+        credentials_file = os.path.join(os.getcwd(),
+                                        self.config["credentials_file"])
+        credentials = {}
+        execfile(credentials_file, credentials)
+        auth = (credentials['tuxedoUsername'], credentials['tuxedoPassword'])
+        self.info("Starting the loop to determine the uptake monitoring ...")
+
+        start_time = datetime.datetime.now()
+        while True:
+            delta = (datetime.datetime.now() - start_time).seconds
+            if delta > self.config["poll_timeout"]:
+                self.error("Uptake monitoring sadly timed-out")
+                raise Exception("Time-out during uptake monitoring")
+
+            uptake = self._get_release_uptake(auth)
+            self.info("Current uptake value to check is {}".format(uptake))
+
+            if uptake >= self.config["min_uptake"]:
+                self.info("Uptake monitoring is complete!")
+                break
+            else:
+                self.info("Mirrors not yet updated, sleeping for a bit ...")
+                time.sleep(self.config["poll_interval"])
+
+
+if __name__ == '__main__':
+    myScript = UptakeMonitoring()
+    myScript.run_and_exit()