Issue #2065 - Part 3: Process install manifests with --track in the recursive make backend

This excludes parts that remove support for building the Mozilla SDK.
Based on https://bugzilla.mozilla.org/show_bug.cgi?id=1390916

Makefile.in

@@ -187,16 +187,7 @@ tup:
 	@$(TUP) $(if $(findstring s,$(filter-out --%,$(MAKEFLAGS))),,--verbose)
 	$(call BUILDSTATUS,TIER_FINISH tup)
 
-# process_install_manifest needs to be invoked with --no-remove when building
+.PHONY: $(addprefix install-,$(install_manifests))
-# js as standalone because automated builds are building nspr separately and
-# that would remove the resulting files.
-# Eventually, a standalone js build would just be able to build nspr itself,
-# removing the need for the former.
-ifdef JS_STANDALONE
-NO_REMOVE=1
-endif
-
-.PHONY: $(addprefix install-,$(subst /,_,$(install_manifests)))
 $(addprefix install-,$(install_manifests)): install-%: $(install_manifest_depends)
 ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 	@# If we're using the hybrid FasterMake/RecursiveMake backend, we want
@@ -204,7 +195,7 @@ ifneq (,$(filter FasterMake+RecursiveMake,$(BUILD_BACKENDS)))
 	@# same directory, because that would blow up
 	$(if $(wildcard _build_manifests/install/$(subst /,_,$*)),$(if $(wildcard faster/install_$(subst /,_,$*)*),$(error FasterMake and RecursiveMake ends of the hybrid build system want to handle $*)))
 endif
-	$(addprefix $(call py_action,process_install_manifest,$(if $(NO_REMOVE),--no-remove )$*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
+	$(addprefix $(call py_action,process_install_manifest,--track install_$(subst /,_,$*).track $*) ,$(wildcard _build_manifests/install/$(subst /,_,$*)))
 
 # Dummy wrapper rule to allow the faster backend to piggy back
 $(addprefix install-,$(subst /,_,$(filter dist/%,$(install_manifests)))): install-dist_%: install-dist/% ;
@@ -217,10 +208,9 @@ install-tests: install-test-files
 .PHONY: run-tests-deps
 run-tests-deps: $(install_manifest_depends)
 
-# Force --no-remove, because $objdir/_tests is handled by multiple manifests.
 .PHONY: install-test-files
 install-test-files:
-	$(call py_action,process_install_manifest,--no-remove _tests _build_manifests/install/_test_files)
+	$(call py_action,process_install_manifest,--track install__test_files.track _tests _build_manifests/install/_test_files)
 
 include $(topsrcdir)/build/moz-automation.mk
 
@@ -240,13 +230,13 @@ ifndef NO_PROFILE_GUIDED_OPTIMIZE
 ifneq ($(OS_ARCH)_$(GNU_CC), WINNT_)
 recurse_pre-export:: install-manifests
 binaries::
-	@$(MAKE) install-manifests NO_REMOVE=1 install_manifests=dist/include
+	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 endif
 else # !MOZ_PROFILE_USE (normal build)
 recurse_pre-export:: install-manifests
 binaries::
-	@$(MAKE) install-manifests NO_REMOVE=1 install_manifests=dist/include
+	@$(MAKE) install-manifests install_manifests=dist/include
 endif
 
 # For historical reasons that are unknown, $(DIST)/sdk is always blown away

js/src/Makefile.in

@@ -185,7 +185,7 @@ install:: js-config.h
 #
 
 install::
-	$(call py_action,process_install_manifest,--no-remove --no-symlinks $(DESTDIR)$(includedir) $(DEPTH)/_build_manifests/install/dist_include)
+	$(call py_action,process_install_manifest,--track install_dist_include.track --no-symlinks $(DESTDIR)$(includedir) $(DEPTH)/_build_manifests/install/dist_include)
 
 #
 # END SpiderMonkey header installation

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -29,33 +29,31 @@ COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, paths, track=None,
+def process_manifest(destdir, paths, track,
-        remove_unaccounted=True,
-        remove_all_directory_symlinks=True,
-        remove_empty_directories=True,
         no_symlinks=False,
         defines={}):
 
-    if track:
+    if os.path.exists(track):
-        if os.path.exists(track):
+        # We use the same format as install manifests for the tracking
-            # We use the same format as install manifests for the tracking
+        # data.
-            # data.
+        manifest = InstallManifest(path=track)
-            manifest = InstallManifest(path=track)
+        remove_unaccounted = FileRegistry()
-            remove_unaccounted = FileRegistry()
+        dummy_file = BaseFile()
-            dummy_file = BaseFile()
 
-            finder = FileFinder(destdir, find_executables=False,
+        finder = FileFinder(destdir, find_executables=False,
-                                find_dotfiles=True)
+                            find_dotfiles=True)
-            for dest in manifest._dests:
+        for dest in manifest._dests:
-                for p, f in finder.find(dest):
+            for p, f in finder.find(dest):
-                    remove_unaccounted.add(p, dummy_file)
+                remove_unaccounted.add(p, dummy_file)
 
-        else:
+        remove_empty_directories=True
-            # If tracking is enabled and there is no file, we don't want to
+        remove_all_directory_symlinks=True
-            # be removing anything.
+    else:
-            remove_unaccounted=False
+        # If tracking is enabled and there is no file, we don't want to
-            remove_empty_directories=False
+        # be removing anything.
-            remove_all_directory_symlinks=False
+        remove_unaccounted=False
+        remove_empty_directories=False
+        remove_all_directory_symlinks=False
 
     manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest
     manifest = manifest_cls()
@@ -83,15 +81,9 @@ def main(argv):
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
-    parser.add_argument('--no-remove', action='store_true',
-        help='Do not remove unaccounted files from destination.')
-    parser.add_argument('--no-remove-all-directory-symlinks', action='store_true',
-        help='Do not remove all directory symlinks from destination.')
-    parser.add_argument('--no-remove-empty-directories', action='store_true',
-        help='Do not remove empty directories from destination.')
     parser.add_argument('--no-symlinks', action='store_true',
         help='Do not install symbolic links. Always copy files')
-    parser.add_argument('--track', metavar="PATH",
+    parser.add_argument('--track', metavar="PATH", required=True,
         help='Use installed files tracking information from the given path.')
     parser.add_argument('-D', action=DefinesAction,
         dest='defines', metavar="VAR[=VAL]",
@@ -101,11 +93,11 @@ def main(argv):
 
     start = time.time()
 
-    result = process_manifest(args.destdir, args.manifests,
+    result = process_manifest(
-        track=args.track, remove_unaccounted=not args.no_remove,
+        args.destdir,
-        remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
+        args.manifests,
-        remove_empty_directories=not args.no_remove_empty_directories,
         no_symlinks=args.no_symlinks,
+        track=args.track,
         defines=args.defines)
 
     elapsed = time.time() - start

xpcom/xpidl/Makefile.in

@@ -3,7 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 export::
-	$(call py_action,process_install_manifest,$(DIST)/idl $(DEPTH)/_build_manifests/install/dist_idl)
+	$(call py_action,process_install_manifest,--track install-xpidl.track $(DIST)/idl $(DEPTH)/_build_manifests/install/dist_idl)
 	$(call SUBMAKE,xpidl,$(DEPTH)/config/makefiles/xpidl)
 
 clean clobber realclean clobber_all distclean::