Merge remote-tracking branch 'origin/tracking' into custom

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -29,33 +29,31 @@ COMPLETE = 'Elapsed: {elapsed:.2f}s; From {dest}: Kept {existing} existing; ' \
     'Removed {rm_files} files and {rm_dirs} directories.'
 
 
-def process_manifest(destdir, paths, track=None,
-        remove_unaccounted=True,
-        remove_all_directory_symlinks=True,
-        remove_empty_directories=True,
+def process_manifest(destdir, paths, track,
         no_symlinks=False,
         defines={}):
 
-    if track:
-        if os.path.exists(track):
-            # We use the same format as install manifests for the tracking
-            # data.
-            manifest = InstallManifest(path=track)
-            remove_unaccounted = FileRegistry()
-            dummy_file = BaseFile()
+    if os.path.exists(track):
+        # We use the same format as install manifests for the tracking
+        # data.
+        manifest = InstallManifest(path=track)
+        remove_unaccounted = FileRegistry()
+        dummy_file = BaseFile()
 
-            finder = FileFinder(destdir, find_executables=False,
-                                find_dotfiles=True)
-            for dest in manifest._dests:
-                for p, f in finder.find(dest):
-                    remove_unaccounted.add(p, dummy_file)
+        finder = FileFinder(destdir, find_executables=False,
+                            find_dotfiles=True)
+        for dest in manifest._dests:
+            for p, f in finder.find(dest):
+                remove_unaccounted.add(p, dummy_file)
 
-        else:
-            # If tracking is enabled and there is no file, we don't want to
-            # be removing anything.
-            remove_unaccounted=False
-            remove_empty_directories=False
-            remove_all_directory_symlinks=False
+        remove_empty_directories=True
+        remove_all_directory_symlinks=True
+    else:
+        # If tracking is enabled and there is no file, we don't want to
+        # be removing anything.
+        remove_unaccounted=False
+        remove_empty_directories=False
+        remove_all_directory_symlinks=False
 
     manifest_cls = InstallManifestNoSymlinks if no_symlinks else InstallManifest
     manifest = manifest_cls()
@@ -70,7 +68,9 @@ def process_manifest(destdir, paths, track=None,
         remove_empty_directories=remove_empty_directories)
 
     if track:
-        manifest.write(path=track)
+        # We should record files that we actually copied.
+        # It is too late to expand wildcards when the track file is read.
+        manifest.write(path=track, expand_pattern=True)
 
     return result
 
@@ -81,15 +81,9 @@ def main(argv):
 
     parser.add_argument('destdir', help='Destination directory.')
     parser.add_argument('manifests', nargs='+', help='Path to manifest file(s).')
-    parser.add_argument('--no-remove', action='store_true',
-        help='Do not remove unaccounted files from destination.')
-    parser.add_argument('--no-remove-all-directory-symlinks', action='store_true',
-        help='Do not remove all directory symlinks from destination.')
-    parser.add_argument('--no-remove-empty-directories', action='store_true',
-        help='Do not remove empty directories from destination.')
     parser.add_argument('--no-symlinks', action='store_true',
         help='Do not install symbolic links. Always copy files')
-    parser.add_argument('--track', metavar="PATH",
+    parser.add_argument('--track', metavar="PATH", required=True,
         help='Use installed files tracking information from the given path.')
     parser.add_argument('-D', action=DefinesAction,
         dest='defines', metavar="VAR[=VAL]",
@@ -99,11 +93,11 @@ def main(argv):
 
     start = time.time()
 
-    result = process_manifest(args.destdir, args.manifests,
-        track=args.track, remove_unaccounted=not args.no_remove,
-        remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
-        remove_empty_directories=not args.no_remove_empty_directories,
+    result = process_manifest(
+        args.destdir,
+        args.manifests,
         no_symlinks=args.no_symlinks,
+        track=args.track,
         defines=args.defines)
 
     elapsed = time.time() - start

python/mozbuild/mozbuild/backend/fastermake.py

@@ -79,7 +79,7 @@ class FasterMakeBackend(CommonBackend, PartialBackend):
                             .add_pattern_symlink(
                                 prefix,
                                 f.full_path[len(prefix):],
-                                mozpath.join(path, f.target_basename))
+                                path)
                     else:
                         self._install_manifests[obj.install_target].add_symlink(
                             f.full_path,

python/mozbuild/mozpack/manifests.py

@@ -228,7 +228,7 @@ class InstallManifest(object):
         """
         return json.loads(data)
 
-    def write(self, path=None, fileobj=None):
+    def write(self, path=None, fileobj=None, expand_pattern=False):
         """Serialize this manifest to a file or file object.
 
         If path is specified, that file will be written to. If fileobj is specified,
@@ -242,10 +242,21 @@ class InstallManifest(object):
             for dest in sorted(self._dests):
                 entry = self._dests[dest]
 
-                parts = ['%d' % entry[0], dest]
-                parts.extend(entry[1:])
-                fh.write('%s\n' % self.FIELD_SEPARATOR.join(
-                    p.encode('utf-8') for p in parts))
+                if expand_pattern and entry[0] in (self.PATTERN_SYMLINK, self.PATTERN_COPY):
+                    type, base, pattern, dest = entry
+                    type = self.SYMLINK if type == self.PATTERN_SYMLINK else self.COPY
+                    finder = FileFinder(base)
+                    paths = [f[0] for f in finder.find(pattern)]
+                    for path in paths:
+                        source = mozpath.join(base, path)
+                        parts = ['%d' % type, mozpath.join(dest, path), source]
+                        fh.write('%s\n' % self.FIELD_SEPARATOR.join(
+                            p.encode('utf-8') for p in parts))
+                else:
+                    parts = ['%d' % entry[0], dest]
+                    parts.extend(entry[1:])
+                    fh.write('%s\n' % self.FIELD_SEPARATOR.join(
+                        p.encode('utf-8') for p in parts))
 
     def add_symlink(self, source, dest):
         """Add a symlink to this manifest.
@@ -289,7 +300,7 @@ class InstallManifest(object):
 
            <base>/foo/bar.h -> <dest>/foo/bar.h
         """
-        self._add_entry(mozpath.join(base, pattern, dest),
+        self._add_entry(mozpath.join(dest, pattern),
             (self.PATTERN_SYMLINK, base, pattern, dest))
 
     def add_pattern_copy(self, base, pattern, dest):
@@ -297,7 +308,7 @@ class InstallManifest(object):
 
         See ``add_pattern_symlink()`` for usage.
         """
-        self._add_entry(mozpath.join(base, pattern, dest),
+        self._add_entry(mozpath.join(dest, pattern),
             (self.PATTERN_COPY, base, pattern, dest))
 
     def add_preprocess(self, source, dest, deps, marker='#', defines={},

python/mozbuild/mozpack/test/test_manifests.py

@@ -156,6 +156,29 @@ class TestInstallManifest(TestWithTmpDir):
         self.assertIn('s_dest2', m1)
         self.assertIn('c_dest2', m1)
 
+    def test_write_expand_pattern(self):
+        source = self.tmppath('source')
+        os.mkdir(source)
+        os.mkdir('%s/base' % source)
+        os.mkdir('%s/base/foo' % source)
+
+        with open('%s/base/foo/file1' % source, 'a'):
+            pass
+
+        with open('%s/base/foo/file2' % source, 'a'):
+            pass
+
+        m = InstallManifest()
+        m.add_pattern_link('%s/base' % source, '**', 'dest')
+
+        track = self.tmppath('track')
+        m.write(path=track, expand_pattern=True)
+
+        m = InstallManifest(path=track)
+        self.assertEqual([dest for dest in m._dests],
+                         ['dest/foo/file1', 'dest/foo/file2'])
+
+
     def test_copier_application(self):
         dest = self.tmppath('dest')
         os.mkdir(dest)