PGO and LTO in clang-cl

Makefile.in

@@ -220,14 +220,14 @@ ifneq ($(filter-out maybe_clobber_profiledbuild,$(MAKECMDGOALS)),)
 GARBAGE_DIRS += dist _tests
 endif
 
-# Windows PGO builds don't perform a clean before the 2nd pass. So, we want
+# MSVC PGO builds don't perform a clean before the 2nd pass. So, we want
 # to preserve content for the 2nd pass on Windows. Everywhere else, we always
 # process the install manifests as part of export.
 # For the binaries rule, not all the install manifests matter, so force only
 # the interesting ones to be done.
 ifdef MOZ_PROFILE_USE
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
-ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
+ifneq ($(OS_ARCH)_$(GNU_CC)_$(CLANG_CL), WINNT__)
 recurse_pre-export:: install-manifests
 binaries::
 	@$(MAKE) install-manifests install_manifests=dist/include
@@ -247,6 +247,10 @@ recurse_artifact:
 	$(topsrcdir)/mach --log-no-times artifact install
 
 ifndef JS_STANDALONE
+# PGO training is also needed in release builds configured without tests.
+pgo-profile-run:
+	$(PYTHON) $(topsrcdir)/build/pgo/profileserver.py $(EXTRA_TEST_ARGS)
+
 ifdef ENABLE_TESTS
 # Additional makefile targets to call automated test suites
 include $(topsrcdir)/testing/testsuite-targets.mk
@@ -356,7 +360,7 @@ pretty-installer:
 
 #XXX: this is a hack, since we don't want to clobber for MSVC
 # PGO support, but we can't do this test in client.mk
-ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
+ifneq ($(OS_ARCH)_$(GNU_CC)_$(CLANG_CL), WINNT__)
 # No point in clobbering if PGO has been explicitly disabled.
 ifndef NO_PROFILE_GUIDED_OPTIMIZE
 maybe_clobber_profiledbuild: clean

aclocal.m4

@@ -17,6 +17,7 @@ builtin(include, build/autoconf/altoptions.m4)dnl
 builtin(include, build/autoconf/mozprog.m4)dnl
 builtin(include, build/autoconf/mozheader.m4)dnl
 builtin(include, build/autoconf/lto.m4)dnl
+builtin(include, build/autoconf/clang-cl-pgo.m4)dnl
 builtin(include, build/autoconf/frameptr.m4)dnl
 builtin(include, build/autoconf/compiler-opts.m4)dnl
 builtin(include, build/autoconf/expandlibs.m4)dnl

build/autoconf/clang-cl-pgo.m4

@@ -0,0 +1,30 @@
+dnl This Source Code Form is subject to the terms of the Mozilla Public
+dnl License, v. 2.0. If a copy of the MPL was not distributed with this
+dnl file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+AC_DEFUN([MOZ_CLANG_CL_PGO], [
+if test -n "$CLANG_CL"; then
+    dnl Override the MSVC/GCC flags: clang-cl uses LLVM instrumentation.
+    PROFILE_GEN_CFLAGS="-clang:-fprofile-instr-generate"
+    PROFILE_USE_CFLAGS='-clang:-fprofile-instr-use="$(DEPTH)/merged.profdata"'
+    PROFILE_GEN_LDFLAGS=
+    PROFILE_USE_LDFLAGS=
+
+    if test -n "$MOZ_PGO"; then
+        AC_PATH_PROG(LLVM_PROFDATA, llvm-profdata.exe)
+        if test -z "$LLVM_PROFDATA"; then
+            AC_MSG_ERROR([clang-cl PGO requires llvm-profdata from the compiler's LLVM installation on PATH])
+        fi
+        dnl The training harness uses native Windows Python, not an MSYS shell.
+        LLVM_PROFDATA="$(cd "$(dirname "$LLVM_PROFDATA")" && pwd -W)/$(basename "$LLVM_PROFDATA")"
+        dnl clang-cl emits a default-library directive for its profile runtime.
+        dnl We invoke the linker directly, so supply the runtime search path.
+        clang_resource_dir=`$CC -print-resource-dir | tr '\\' '/'`
+        if test ! -d "$clang_resource_dir/lib/windows"; then
+            AC_MSG_ERROR([clang-cl PGO requires the Windows compiler-rt profile runtime])
+        fi
+        PROFILE_GEN_LDFLAGS="-LIBPATH:\"$clang_resource_dir/lib/windows\""
+    fi
+fi
+AC_SUBST(LLVM_PROFDATA)
+])

build/pgo/llvm_pgo.py

@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""Collect LLVM profiles outside the directories cleaned between PGO passes."""
+
+import glob
+import os
+import subprocess
+import tempfile
+
+
+class LLVMProfile(object):
+    def __init__(self, topobjdir, profdata):
+        if not profdata or not os.path.isfile(profdata):
+            raise RuntimeError("clang-cl PGO requires a configured llvm-profdata")
+        self.topobjdir = os.path.abspath(topobjdir)
+        self.profdata = profdata
+        self.output = os.path.join(self.topobjdir, 'merged.profdata')
+        self.directory = None
+
+    def prepare(self, env):
+        # Never reuse a previous training run's merged or raw profiles.
+        if os.path.exists(self.output):
+            os.remove(self.output)
+        self.directory = tempfile.mkdtemp(prefix='pgo-profiles-', dir=self.topobjdir)
+        # %m distinguishes instrumented DLLs; %p distinguishes child processes.
+        env['LLVM_PROFILE_FILE'] = os.path.join(self.directory, '%m-%p.profraw')
+
+    def merge(self):
+        profiles = sorted(glob.glob(os.path.join(self.directory, '*.profraw')))
+        if not profiles or not any(os.path.getsize(p) for p in profiles):
+            raise RuntimeError("PGO training produced no LLVM profile data")
+        # An input list avoids Windows command-line length limits. Keep profiles
+        # on failure so that corrupt or incompatible data can be diagnosed.
+        inputs = os.path.join(self.directory, 'profiles.list')
+        with open(inputs, 'w') as stream:
+            for profile in profiles:
+                stream.write(profile + '\n')
+        subprocess.check_call([self.profdata, 'merge', '-o', self.output,
+                               '-f', inputs])
+        if not os.path.isfile(self.output) or not os.path.getsize(self.output):
+            raise RuntimeError("llvm-profdata did not produce a merged profile")

build/pgo/profileserver.py

@@ -18,9 +18,20 @@ import tempfile
 from datetime import datetime
 from mozbuild.base import MozbuildObject
 from buildconfig import substs
+from llvm_pgo import LLVMProfile
 
 PORT = 8888
 
+
+def wait_for_training(runner, timeout):
+  result = runner.wait(timeout=timeout)
+  if result is None:
+    runner.stop()
+    raise RuntimeError("PGO training browser timed out after %s seconds" % timeout)
+  if result != 0:
+    raise RuntimeError("PGO training browser exited unsuccessfully")
+
+
 if __name__ == '__main__':
   cli = CLI()
   debug_args, interactive = cli.debugger_arguments()
@@ -54,6 +65,15 @@ if __name__ == '__main__':
 
     env = os.environ.copy()
     env["XPCOM_DEBUG_BREAK"] = "warn"
+    # Keep the training workload in the parent process. In particular, the
+    # initialization javascript: URL must work without a content subprocess.
+    # This environment belongs only to the temporary training browser.
+    env["MOZ_FORCE_DISABLE_E10S"] = "1"
+
+    llvm_profile = None
+    if substs.get("CLANG_CL") and substs.get("MOZ_PGO"):
+      llvm_profile = LLVMProfile(build.topobjdir, substs.get("LLVM_PROFDATA"))
+      llvm_profile.prepare(env)
 
     # For VC12+, make sure we can find the right bitness of pgort1x0.dll
     if not substs.get('HAVE_64BIT_BUILD'):
@@ -72,7 +92,7 @@ if __name__ == '__main__':
                            cmdargs=['javascript:Quitter.quit()'],
                            env=env)
     runner.start()
-    runner.wait()
+    wait_for_training(runner, 180)
 
     jarlog = os.getenv("JARLOG_FILE")
     if jarlog:
@@ -85,7 +105,9 @@ if __name__ == '__main__':
                            cmdargs=cmdargs,
                            env=env)
     runner.start(debug_args=debug_args, interactive=interactive)
-    runner.wait()
+    wait_for_training(runner, 600)
-    httpd.stop()
+    if llvm_profile:
+      llvm_profile.merge()
   finally:
+    httpd.stop()
     shutil.rmtree(profilePath)

build/pgo/test_llvm_pgo.py

@@ -0,0 +1,139 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+import unittest
+from distutils.spawn import find_executable
+
+from llvm_pgo import LLVMProfile
+
+
+class LLVMProfileTest(unittest.TestCase):
+    def setUp(self):
+        self.directory = tempfile.mkdtemp(prefix='llvm pgo test ')
+        self.profile = LLVMProfile(self.directory, sys.executable)
+        self.check_call = subprocess.check_call
+
+    def tearDown(self):
+        subprocess.check_call = self.check_call
+        shutil.rmtree(self.directory)
+
+    def write(self, path, data):
+        with open(path, 'w') as stream:
+            stream.write(data)
+
+    def test_missing_tool(self):
+        with self.assertRaises(RuntimeError):
+            LLVMProfile(self.directory, None)
+
+    def test_prepare_isolates_runs(self):
+        self.write(self.profile.output, 'stale merged profile')
+        env = {}
+        self.profile.prepare(env)
+        first = self.profile.directory
+        self.write(os.path.join(first, 'stale.profraw'), 'stale raw profile')
+        self.profile.prepare(env)
+        self.assertNotEqual(first, self.profile.directory)
+        self.assertFalse(os.path.exists(self.profile.output))
+        self.assertEqual(env['LLVM_PROFILE_FILE'],
+                         os.path.join(self.profile.directory, '%m-%p.profraw'))
+        with self.assertRaises(RuntimeError):
+            self.profile.merge()
+
+    def test_empty_profiles(self):
+        self.profile.prepare({})
+        self.write(os.path.join(self.profile.directory, 'empty.profraw'), '')
+        with self.assertRaises(RuntimeError):
+            self.profile.merge()
+
+    def test_merge_uses_input_list(self):
+        self.profile.prepare({})
+        raw = os.path.join(self.profile.directory, 'module-process.profraw')
+        self.write(raw, 'raw profile')
+        def merge(args):
+            self.assertEqual(args[:4], [sys.executable, 'merge', '-o',
+                                       self.profile.output])
+            self.assertEqual(args[4], '-f')
+            with open(args[5]) as stream:
+                self.assertEqual(stream.read().splitlines(), [raw])
+            self.write(self.profile.output, 'merged profile')
+        subprocess.check_call = merge
+        self.profile.merge()
+
+    def test_merge_failure_propagates(self):
+        self.profile.prepare({})
+        self.write(os.path.join(self.profile.directory, 'bad.profraw'), 'bad')
+        def fail(args):
+            raise subprocess.CalledProcessError(1, args)
+        subprocess.check_call = fail
+        with self.assertRaises(subprocess.CalledProcessError):
+            self.profile.merge()
+
+    def test_merge_requires_output(self):
+        self.profile.prepare({})
+        self.write(os.path.join(self.profile.directory, 'data.profraw'), 'raw')
+        subprocess.check_call = lambda args: None
+        with self.assertRaises(RuntimeError):
+            self.profile.merge()
+
+
+class PGOConfigurationTest(unittest.TestCase):
+    def setUp(self):
+        self.make = (os.environ.get('MAKE') or find_executable('mozmake') or
+                     find_executable('make'))
+        if not self.make:
+            self.skipTest('GNU make is required')
+        self.directory = tempfile.mkdtemp(prefix='pgo configuration ')
+        config = os.path.join(os.path.dirname(__file__), '..', '..',
+                              'config', 'config.mk')
+        with open(config) as stream:
+            source = stream.read()
+        start = source.index('# Reject stale Windows PGO configuration')
+        end = source.index('# Enable profile-based feedback', start)
+        self.makefile = os.path.join(self.directory, 'Makefile')
+        with open(self.makefile, 'w') as stream:
+            stream.write(source[start:end] + '\nall:;\n')
+
+    def tearDown(self):
+        if hasattr(self, 'directory'):
+            shutil.rmtree(self.directory)
+
+    def run_make(self, *variables):
+        process = subprocess.Popen([self.make, '-f', self.makefile] +
+                                   list(variables), stdout=subprocess.PIPE,
+                                   stderr=subprocess.STDOUT)
+        output = process.communicate()[0].decode('utf-8')
+        return process.returncode, output
+
+    def test_missing_profdata_fails_before_build(self):
+        code, output = self.run_make('MOZ_PGO=1', 'CLANG_CL=1', 'LLVM_PROFDATA=')
+        self.assertNotEqual(code, 0)
+        self.assertIn('requires LLVM_PROFDATA', output)
+
+    def test_msvc_flags_rejected_for_clang(self):
+        code, output = self.run_make('MOZ_PGO=1', 'CLANG_CL=1',
+                                    'LLVM_PROFDATA=llvm-profdata.exe',
+                                    'PROFILE_GEN_CFLAGS=-GL')
+        self.assertNotEqual(code, 0)
+        self.assertIn('requires LLVM instrumentation flags', output)
+
+    def test_llvm_configuration_accepted(self):
+        code, output = self.run_make('MOZ_PGO=1', 'CLANG_CL=1',
+                                    'LLVM_PROFDATA=llvm-profdata.exe',
+                                    'PROFILE_GEN_CFLAGS=-clang:-fprofile-instr-generate')
+        self.assertEqual(code, 0, output)
+
+    def test_non_pgo_and_msvc_unaffected(self):
+        for variables in [('MOZ_PGO=', 'CLANG_CL=1'),
+                          ('MOZ_PGO=1', 'CLANG_CL=')]:
+            code, output = self.run_make(*variables)
+            self.assertEqual(code, 0, output)
+
+
+if __name__ == '__main__':
+    unittest.main()

config/config.mk

@@ -210,13 +210,25 @@ ifdef CPP_UNIT_TESTS
 NO_PROFILE_GUIDED_OPTIMIZE = 1
 endif
 
+# Reject stale Windows PGO configuration before compiling the first pass.
+ifdef MOZ_PGO
+ifdef CLANG_CL
+ifeq ($(strip $(LLVM_PROFDATA)),)
+$(error clang-cl PGO requires LLVM_PROFDATA; rerun ./mach configure)
+endif
+ifeq ($(filter -clang:-fprofile-instr-generate,$(PROFILE_GEN_CFLAGS)),)
+$(error clang-cl PGO requires LLVM instrumentation flags; rerun ./mach configure)
+endif
+endif
+endif
+
 # Enable profile-based feedback
 ifneq (1,$(NO_PROFILE_GUIDED_OPTIMIZE))
 ifdef MOZ_PROFILE_GENERATE
 OS_CFLAGS += $(if $(filter $(notdir $<),$(notdir $(NO_PROFILE_GUIDED_OPTIMIZE))),,$(PROFILE_GEN_CFLAGS))
 OS_CXXFLAGS += $(if $(filter $(notdir $<),$(notdir $(NO_PROFILE_GUIDED_OPTIMIZE))),,$(PROFILE_GEN_CFLAGS))
 OS_LDFLAGS += $(PROFILE_GEN_LDFLAGS)
-ifeq (WINNT,$(OS_ARCH))
+ifeq ($(OS_ARCH)_$(CLANG_CL),WINNT_)
 AR_FLAGS += -LTCG
 endif
 endif # MOZ_PROFILE_GENERATE
@@ -225,7 +237,7 @@ ifdef MOZ_PROFILE_USE
 OS_CFLAGS += $(if $(filter $(notdir $<),$(notdir $(NO_PROFILE_GUIDED_OPTIMIZE))),,$(PROFILE_USE_CFLAGS))
 OS_CXXFLAGS += $(if $(filter $(notdir $<),$(notdir $(NO_PROFILE_GUIDED_OPTIMIZE))),,$(PROFILE_USE_CFLAGS))
 OS_LDFLAGS += $(PROFILE_USE_LDFLAGS)
-ifeq (WINNT,$(OS_ARCH))
+ifeq ($(OS_ARCH)_$(CLANG_CL),WINNT_)
 AR_FLAGS += -LTCG
 endif
 endif # MOZ_PROFILE_USE

config/rules.mk

@@ -531,7 +531,7 @@ endif
 ##############################################
 ifneq (1,$(NO_PROFILE_GUIDED_OPTIMIZE))
 ifdef MOZ_PROFILE_USE
-ifeq ($(OS_ARCH)_$(GNU_CC), WINNT_)
+ifeq ($(OS_ARCH)_$(GNU_CC)_$(CLANG_CL), WINNT__)
 # When building with PGO, we have to make sure to re-link
 # in the MOZ_PROFILE_USE phase if we linked in the
 # MOZ_PROFILE_GENERATE phase. We'll touch this pgo.relink
@@ -563,7 +563,7 @@ endif # MOZ_PROFILE_USE
 ifdef MOZ_PROFILE_GENERATE
 # Clean up profiling data during PROFILE_GENERATE phase
 export::
-ifeq ($(OS_ARCH)_$(GNU_CC), WINNT_)
+ifeq ($(OS_ARCH)_$(GNU_CC)_$(CLANG_CL), WINNT__)
 	$(foreach pgd,$(wildcard *.pgd),pgomgr -clear $(pgd);)
 else
 ifdef GNU_CC

js/src/aclocal.m4

@@ -16,6 +16,7 @@ builtin(include, ../../build/autoconf/altoptions.m4)dnl
 builtin(include, ../../build/autoconf/mozprog.m4)dnl
 builtin(include, ../../build/autoconf/mozheader.m4)dnl
 builtin(include, ../../build/autoconf/lto.m4)dnl
+builtin(include, ../../build/autoconf/clang-cl-pgo.m4)dnl
 builtin(include, ../../build/autoconf/frameptr.m4)dnl
 builtin(include, ../../build/autoconf/compiler-opts.m4)dnl
 builtin(include, ../../build/autoconf/expandlibs.m4)dnl

js/src/old-configure.in

@@ -711,7 +711,17 @@ case "$target" in
         if test "$AS_BIN"; then
             AS="$(basename "$AS_BIN")"
         fi
-        AR='lib'
+        if test -n "$CLANG_CL"; then
+            dnl Microsoft lib.exe cannot archive LLVM bitcode from ThinLTO.
+            AC_PATH_PROG(LLVM_LIB, llvm-lib.exe)
+            if test -z "$LLVM_LIB"; then
+                AC_MSG_ERROR([clang-cl requires llvm-lib.exe on PATH])
+            fi
+            dnl expandlibs invokes AR from native Windows Python.
+            AR="$(cd "$(dirname "$LLVM_LIB")" && pwd -W)/$(basename "$LLVM_LIB")"
+        else
+            AR='lib'
+        fi
         AR_FLAGS='-NOLOGO -OUT:$@'
         AR_EXTRACT=
         RANLIB='echo not_ranlib'
@@ -2005,6 +2015,9 @@ if test "$ac_cv_struct_tm_zone_tm_gmtoff" = "yes" ; then
 fi
 fi # ! SKIP_COMPILER_CHECKS
 
+dnl Windows skips the GCC probes above, but still needs LLVM PGO setup.
+MOZ_CLANG_CL_PGO
+
 AC_DEFINE(CPP_THROW_NEW, [throw()])
 AC_LANG_C
 

old-configure.in

@@ -876,7 +876,17 @@ case "$target" in
         if test "$AS_BIN"; then
             AS="$(basename "$AS_BIN")"
         fi
-        AR='lib'
+        if test -n "$CLANG_CL"; then
+            dnl Microsoft lib.exe cannot archive LLVM bitcode from ThinLTO.
+            AC_PATH_PROG(LLVM_LIB, llvm-lib.exe)
+            if test -z "$LLVM_LIB"; then
+                AC_MSG_ERROR([clang-cl requires llvm-lib.exe on PATH])
+            fi
+            dnl expandlibs invokes AR from native Windows Python.
+            AR="$(cd "$(dirname "$LLVM_LIB")" && pwd -W)/$(basename "$LLVM_LIB")"
+        else
+            AR='lib'
+        fi
         AR_FLAGS='-NOLOGO -OUT:$@'
         AR_EXTRACT=
         RANLIB='echo not_ranlib'
@@ -4547,6 +4557,9 @@ AC_SUBST(PROFILE_USE_LDFLAGS)
 
 fi # ! SKIP_COMPILER_CHECKS
 
+dnl Windows skips the GCC probes above, but still needs LLVM PGO setup.
+MOZ_CLANG_CL_PGO
+
 AC_DEFINE(CPP_THROW_NEW, [throw()])
 AC_LANG_C
 

testing/testsuite-targets.mk

@@ -143,9 +143,6 @@ cppunittests-remote:
 jetpack-tests:
 	cd $(topsrcdir)/addon-sdk/source && $(PYTHON) bin/cfx -b $(abspath $(browser_path)) --parseable testpkgs
 
-pgo-profile-run:
-	$(PYTHON) $(topsrcdir)/build/pgo/profileserver.py $(EXTRA_TEST_ARGS)
-
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk