PGO and LTO in clang-cl

build/autoconf/clang-cl-pgo.m4

@@ -0,0 +1,30 @@
+dnl This Source Code Form is subject to the terms of the Mozilla Public
+dnl License, v. 2.0. If a copy of the MPL was not distributed with this
+dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+AC_DEFUN([MOZ_CLANG_CL_PGO], [
+if test -n "$CLANG_CL"; then
+    dnl Override the MSVC/GCC flags: clang-cl uses LLVM instrumentation.
+    PROFILE_GEN_CFLAGS="-clang:-fprofile-instr-generate"
+    PROFILE_USE_CFLAGS='-clang:-fprofile-instr-use="$(DEPTH)/merged.profdata"'
+    PROFILE_GEN_LDFLAGS=
+    PROFILE_USE_LDFLAGS=
+
+    if test -n "$MOZ_PGO"; then
+        AC_PATH_PROG(LLVM_PROFDATA, llvm-profdata.exe)
+        if test -z "$LLVM_PROFDATA"; then
+            AC_MSG_ERROR([clang-cl PGO requires llvm-profdata from the compiler's LLVM installation on PATH])
+        fi
+        dnl The training harness uses native Windows Python, not an MSYS shell.
+        LLVM_PROFDATA="$(cd "$(dirname "$LLVM_PROFDATA")" && pwd -W)/$(basename "$LLVM_PROFDATA")"
+        dnl clang-cl emits a default-library directive for its profile runtime.
+        dnl We invoke the linker directly, so supply the runtime search path.
+        clang_resource_dir=`$CC -print-resource-dir | tr '\\' '/'`
+        if test ! -d "$clang_resource_dir/lib/windows"; then
+            AC_MSG_ERROR([clang-cl PGO requires the Windows compiler-rt profile runtime])
+        fi
+        PROFILE_GEN_LDFLAGS="-LIBPATH:\"$clang_resource_dir/lib/windows\""
+    fi
+fi
+AC_SUBST(LLVM_PROFDATA)
+])

build/pgo/llvm_pgo.py

@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Collect LLVM profiles outside the directories cleaned between PGO passes."""
+
+import glob
+import os
+import subprocess
+import tempfile
+
+
+class LLVMProfile(object):
+    def __init__(self, topobjdir, profdata):
+        if not profdata or not os.path.isfile(profdata):
+            raise RuntimeError("clang-cl PGO requires a configured llvm-profdata")
+        self.topobjdir = os.path.abspath(topobjdir)
+        self.profdata = profdata
+        self.output = os.path.join(self.topobjdir, 'merged.profdata')
+        self.directory = None
+
+    def prepare(self, env):
+        # Never reuse a previous training run's merged or raw profiles.
+        if os.path.exists(self.output):
+            os.remove(self.output)
+        self.directory = tempfile.mkdtemp(prefix='pgo-profiles-', dir=self.topobjdir)
+        # %m distinguishes instrumented DLLs; %p distinguishes child processes.
+        env['LLVM_PROFILE_FILE'] = os.path.join(self.directory, '%m-%p.profraw')
+
+    def merge(self):
+        profiles = sorted(glob.glob(os.path.join(self.directory, '*.profraw')))
+        if not profiles or not any(os.path.getsize(p) for p in profiles):
+            raise RuntimeError("PGO training produced no LLVM profile data")
+        # An input list avoids Windows command-line length limits. Keep profiles
+        # on failure so that corrupt or incompatible data can be diagnosed.
+        inputs = os.path.join(self.directory, 'profiles.list')
+        with open(inputs, 'w') as stream:
+            for profile in profiles:
+                stream.write(profile + '\n')
+        subprocess.check_call([self.profdata, 'merge', '-o', self.output,
+                               '-f', inputs])
+        if not os.path.isfile(self.output) or not os.path.getsize(self.output):
+            raise RuntimeError("llvm-profdata did not produce a merged profile")

build/pgo/profileserver.py

@@ -18,9 +18,20 @@ import tempfile
 from datetime import datetime
 from mozbuild.base import MozbuildObject
 from buildconfig import substs
+from llvm_pgo import LLVMProfile
 
 PORT = 8888
 
+
+def wait_for_training(runner, timeout):
+  result = runner.wait(timeout=timeout)
+  if result is None:
+    runner.stop()
+    raise RuntimeError("PGO training browser timed out after %s seconds" % timeout)
+  if result != 0:
+    raise RuntimeError("PGO training browser exited unsuccessfully")
+
+
 if __name__ == '__main__':
   cli = CLI()
   debug_args, interactive = cli.debugger_arguments()
@@ -54,6 +65,15 @@ if __name__ == '__main__':
 
     env = os.environ.copy()
     env["XPCOM_DEBUG_BREAK"] = "warn"
+    # Keep the training workload in the parent process. In particular, the
+    # initialization javascript: URL must work without a content subprocess.
+    # This environment belongs only to the temporary training browser.
+    env["MOZ_FORCE_DISABLE_E10S"] = "1"
+
+    llvm_profile = None
+    if substs.get("CLANG_CL") and substs.get("MOZ_PGO"):
+      llvm_profile = LLVMProfile(build.topobjdir, substs.get("LLVM_PROFDATA"))
+      llvm_profile.prepare(env)
 
     # For VC12+, make sure we can find the right bitness of pgort1x0.dll
     if not substs.get('HAVE_64BIT_BUILD'):
@@ -72,7 +92,7 @@ if __name__ == '__main__':
                            cmdargs=['javascript:Quitter.quit()'],
                            env=env)
     runner.start()
-    runner.wait()
+    wait_for_training(runner, 180)
 
     jarlog = os.getenv("JARLOG_FILE")
     if jarlog:
@@ -85,7 +105,9 @@ if __name__ == '__main__':
                            cmdargs=cmdargs,
                            env=env)
     runner.start(debug_args=debug_args, interactive=interactive)
-    runner.wait()
-    httpd.stop()
+    wait_for_training(runner, 600)
+    if llvm_profile:
+      llvm_profile.merge()
   finally:
+    httpd.stop()
     shutil.rmtree(profilePath)

build/pgo/test_llvm_pgo.py

@@ -0,0 +1,139 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from distutils.spawn import find_executable
+
+from llvm_pgo import LLVMProfile
+
+
+class LLVMProfileTest(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.mkdtemp(prefix='llvm pgo test ')
+        self.profile = LLVMProfile(self.directory, sys.executable)
+        self.check_call = subprocess.check_call
+
+    def tearDown(self):
+        subprocess.check_call = self.check_call
+        shutil.rmtree(self.directory)
+
+    def write(self, path, data):
+        with open(path, 'w') as stream:
+            stream.write(data)
+
+    def test_missing_tool(self):
+        with self.assertRaises(RuntimeError):
+            LLVMProfile(self.directory, None)
+
+    def test_prepare_isolates_runs(self):
+        self.write(self.profile.output, 'stale merged profile')
+        env = {}
+        self.profile.prepare(env)
+        first = self.profile.directory
+        self.write(os.path.join(first, 'stale.profraw'), 'stale raw profile')
+        self.profile.prepare(env)
+        self.assertNotEqual(first, self.profile.directory)
+        self.assertFalse(os.path.exists(self.profile.output))
+        self.assertEqual(env['LLVM_PROFILE_FILE'],
+                         os.path.join(self.profile.directory, '%m-%p.profraw'))
+        with self.assertRaises(RuntimeError):
+            self.profile.merge()
+
+    def test_empty_profiles(self):
+        self.profile.prepare({})
+        self.write(os.path.join(self.profile.directory, 'empty.profraw'), '')
+        with self.assertRaises(RuntimeError):
+            self.profile.merge()
+
+    def test_merge_uses_input_list(self):
+        self.profile.prepare({})
+        raw = os.path.join(self.profile.directory, 'module-process.profraw')
+        self.write(raw, 'raw profile')
+        def merge(args):
+            self.assertEqual(args[:4], [sys.executable, 'merge', '-o',
+                                       self.profile.output])
+            self.assertEqual(args[4], '-f')
+            with open(args[5]) as stream:
+                self.assertEqual(stream.read().splitlines(), [raw])
+            self.write(self.profile.output, 'merged profile')
+        subprocess.check_call = merge
+        self.profile.merge()
+
+    def test_merge_failure_propagates(self):
+        self.profile.prepare({})
+        self.write(os.path.join(self.profile.directory, 'bad.profraw'), 'bad')
+        def fail(args):
+            raise subprocess.CalledProcessError(1, args)
+        subprocess.check_call = fail
+        with self.assertRaises(subprocess.CalledProcessError):
+            self.profile.merge()
+
+    def test_merge_requires_output(self):
+        self.profile.prepare({})
+        self.write(os.path.join(self.profile.directory, 'data.profraw'), 'raw')
+        subprocess.check_call = lambda args: None
+        with self.assertRaises(RuntimeError):
+            self.profile.merge()
+
+
+class PGOConfigurationTest(unittest.TestCase):
+    def setUp(self):
+        self.make = (os.environ.get('MAKE') or find_executable('mozmake') or
+                     find_executable('make'))
+        if not self.make:
+            self.skipTest('GNU make is required')
+        self.directory = tempfile.mkdtemp(prefix='pgo configuration ')
+        config = os.path.join(os.path.dirname(__file__), '..', '..',
+                              'config', 'config.mk')
+        with open(config) as stream:
+            source = stream.read()
+        start = source.index('# Reject stale Windows PGO configuration')
+        end = source.index('# Enable profile-based feedback', start)
+        self.makefile = os.path.join(self.directory, 'Makefile')
+        with open(self.makefile, 'w') as stream:
+            stream.write(source[start:end] + '\nall:;\n')
+
+    def tearDown(self):
+        if hasattr(self, 'directory'):
+            shutil.rmtree(self.directory)
+
+    def run_make(self, *variables):
+        process = subprocess.Popen([self.make, '-f', self.makefile] +
+                                   list(variables), stdout=subprocess.PIPE,
+                                   stderr=subprocess.STDOUT)
+        output = process.communicate()[0].decode('utf-8')
+        return process.returncode, output
+
+    def test_missing_profdata_fails_before_build(self):
+        code, output = self.run_make('MOZ_PGO=1', 'CLANG_CL=1', 'LLVM_PROFDATA=')
+        self.assertNotEqual(code, 0)
+        self.assertIn('requires LLVM_PROFDATA', output)
+
+    def test_msvc_flags_rejected_for_clang(self):
+        code, output = self.run_make('MOZ_PGO=1', 'CLANG_CL=1',
+                                    'LLVM_PROFDATA=llvm-profdata.exe',
+                                    'PROFILE_GEN_CFLAGS=-GL')
+        self.assertNotEqual(code, 0)
+        self.assertIn('requires LLVM instrumentation flags', output)
+
+    def test_llvm_configuration_accepted(self):
+        code, output = self.run_make('MOZ_PGO=1', 'CLANG_CL=1',
+                                    'LLVM_PROFDATA=llvm-profdata.exe',
+                                    'PROFILE_GEN_CFLAGS=-clang:-fprofile-instr-generate')
+        self.assertEqual(code, 0, output)
+
+    def test_non_pgo_and_msvc_unaffected(self):
+        for variables in [('MOZ_PGO=', 'CLANG_CL=1'),
+                          ('MOZ_PGO=1', 'CLANG_CL=')]:
+            code, output = self.run_make(*variables)
+            self.assertEqual(code, 0, output)
+
+
+if __name__ == '__main__':
+    unittest.main()