Bug 1274518 - Add IPDL preprocessing support

Co-authored-by: Kai-Zhen Li <seinlin.maung@gmail.com>

python/mozbuild/mozbuild/backend/common.py

@@ -29,6 +29,7 @@ from mozbuild.frontend.data import (
     FinalTargetFiles,
     GeneratedEventWebIDLFile,
     GeneratedWebIDLFile,
+    PreprocessedIPDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     SharedLibrary,
@@ -215,6 +216,21 @@ class BinariesCollection(object):
         self.shared_libraries = []
         self.programs = []
 
+class IPDLCollection(object):
+    """Collects IPDL files during the build."""
+
+    def __init__(self):
+        self.sources = set()
+        self.preprocessed_sources = set()
+
+    def all_sources(self):
+        return self.sources | self.preprocessed_sources
+
+    def all_regular_sources(self):
+        return self.sources
+
+    def all_preprocessed_sources(self):
+        return self.preprocessed_sources
 
 class CommonBackend(BuildBackend):
     """Holds logic common to all build backends."""
@@ -225,7 +241,7 @@ class CommonBackend(BuildBackend):
         self._webidls = WebIDLCollection()
         self._binaries = BinariesCollection()
         self._configs = set()
-        self._ipdl_sources = set()
+        self._ipdls = IPDLCollection()
 
     def consume_object(self, obj):
         self._configs.add(obj.config)
@@ -290,6 +306,10 @@ class CommonBackend(BuildBackend):
             self._webidls.generated_sources.add(mozpath.join(obj.srcdir,
                 obj.basename))
 
+        elif isinstance(obj, PreprocessedIPDLFile):
+            self._ipdls.preprocessed_sources.add(mozpath.join(
+                obj.srcdir, obj.basename))
+
         elif isinstance(obj, PreprocessedWebIDLFile):
             # WebIDL isn't relevant to artifact builds.
             if self.environment.is_artifact_build:
@@ -310,7 +330,7 @@ class CommonBackend(BuildBackend):
             if self.environment.is_artifact_build:
                 return True
 
-            self._ipdl_sources.add(mozpath.join(obj.srcdir, obj.basename))
+            self._ipdls.sources.add(mozpath.join(obj.srcdir, obj.basename))
 
         elif isinstance(obj, UnifiedSources):
             # Unified sources aren't relevant to artifact builds.
@@ -341,7 +361,9 @@ class CommonBackend(BuildBackend):
 
         self._handle_webidl_collection(self._webidls)
 
-        sorted_ipdl_sources = list(sorted(self._ipdl_sources))
+        sorted_ipdl_sources = list(sorted(self._ipdls.all_sources()))
+        sorted_nonstatic_ipdl_sources = list(sorted(self._ipdls.all_preprocessed_sources()))
+        sorted_static_ipdl_sources = list(sorted(self._ipdls.all_regular_sources()))
 
         def files_from(ipdl):
             base = mozpath.basename(ipdl)
@@ -364,7 +386,8 @@ class CommonBackend(BuildBackend):
                                                           files_per_unified_file=16))
 
         self._write_unified_files(unified_source_mapping, ipdl_dir, poison_windows_h=False)
-        self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, unified_source_mapping)
+        self._handle_ipdl_sources(ipdl_dir, sorted_ipdl_sources, sorted_nonstatic_ipdl_sources,
+                                  sorted_static_ipdl_sources, unified_source_mapping)
 
         for config in self._configs:
             self.backend_input_files.add(config.source)

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -1407,18 +1407,32 @@ class RecursiveMakeBackend(CommonBackend):
 
         self._makefile_out_count += 1
 
-    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, sorted_nonstatic_ipdl_sources,
-                             unified_ipdl_cppsrcs_mapping):
+                             sorted_static_ipdl_sources, unified_ipdl_cppsrcs_mapping):
         # Write out a master list of all IPDL source files.
         mk = Makefile()
 
-        mk.add_statement('ALL_IPDLSRCS := %s' % ' '.join(sorted_ipdl_sources))
+        sorted_nonstatic_ipdl_basenames = list()
+        for source in sorted_nonstatic_ipdl_sources:
+            basename = os.path.basename(source)
+            sorted_nonstatic_ipdl_basenames.append(basename)
+            rule = mk.create_rule([basename])
+            rule.add_dependencies([source])
+            rule.add_commands([
+                '$(RM) $@',
+                '$(call py_action,preprocessor,$(DEFINES) $(ACDEFINES) '
+                    '$< -o $@)'
+            ])
+
+        mk.add_statement('ALL_IPDLSRCS := %s %s' % (' '.join(sorted_nonstatic_ipdl_basenames),
+                         ' '.join(sorted_static_ipdl_sources)))
 
         self._add_unified_build_rules(mk, unified_ipdl_cppsrcs_mapping,
                                       unified_files_makefile_variable='CPPSRCS')
 
-        mk.add_statement('IPDLDIRS := %s' % ' '.join(sorted(set(mozpath.dirname(p)
+        # Preprocessed ipdl files are generated in ipdl_dir.
-            for p in self._ipdl_sources))))
+        mk.add_statement('IPDLDIRS := %s %s' % (ipdl_dir, ' '.join(sorted(set(mozpath.dirname(p)
+            for p in sorted_static_ipdl_sources)))))
 
         with self._write_file(mozpath.join(ipdl_dir, 'ipdlsrcs.mk')) as ipdls:
             mk.dump(ipdls, removal_guard=False)

python/mozbuild/mozbuild/backend/tup.py

@@ -298,8 +298,8 @@ class TupOnly(CommonBackend, PartialBackend):
             outputs=[output],
         )
 
-    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, sorted_nonstatic_ipdl_sources,
-                             unified_ipdl_cppsrcs_mapping):
+                             sorted_static_ipdl_sources, unified_ipdl_cppsrcs_mapping):
         # TODO: This isn't implemented yet in the tup backend, but it is called
         # by the CommonBackend.
         pass

python/mozbuild/mozbuild/compilation/database.py

@@ -177,8 +177,8 @@ class CompileDBBackend(CommonBackend):
     def _handle_idl_manager(self, idl_manager):
         pass
 
-    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources,
+    def _handle_ipdl_sources(self, ipdl_dir, sorted_ipdl_sources, sorted_nonstatic_ipdl_sources,
-                             unified_ipdl_cppsrcs_mapping):
+                             sorted_static_ipdl_sources, unified_ipdl_cppsrcs_mapping):
         for f in unified_ipdl_cppsrcs_mapping:
             self._build_db_line(ipdl_dir, None, self.environment, f[0],
                                 '.cpp')

python/mozbuild/mozbuild/frontend/context.py

@@ -1441,6 +1441,13 @@ VARIABLES = {
         not use this flag.
         """),
 
+    'PREPROCESSED_IPDL_SOURCES': (StrictOrderingOnAppendList, list,
+        """Preprocessed IPDL source files.
+
+        These files will be preprocessed, then parsed and converted to
+        ``.cpp`` files.
+        """),
+
     'IPDL_SOURCES': (StrictOrderingOnAppendList, list,
         """IPDL source files.
 

python/mozbuild/mozbuild/frontend/data.py

@@ -218,6 +218,18 @@ class IPDLFile(ContextDerived):
 
         self.basename = path
 
+class PreprocessedIPDLFile(ContextDerived):
+    """Describes an individual .ipdl source file that requires preprocessing."""
+
+    __slots__ = (
+        'basename',
+    )
+
+    def __init__(self, context, path):
+        ContextDerived.__init__(self, context)
+
+        self.basename = path
+
 class WebIDLFile(ContextDerived):
     """Describes an individual .webidl source file."""
 

python/mozbuild/mozbuild/frontend/emitter.py

@@ -58,6 +58,7 @@ from .data import (
     ObjdirFiles,
     ObjdirPreprocessedFiles,
     PerSourceFlag,
+    PreprocessedIPDLFile,
     PreprocessedTestWebIDLFile,
     PreprocessedWebIDLFile,
     Program,
@@ -872,6 +873,7 @@ class TreeMetadataEmitter(LoggingMixin):
             ('GENERATED_EVENTS_WEBIDL_FILES', GeneratedEventWebIDLFile),
             ('GENERATED_WEBIDL_FILES', GeneratedWebIDLFile),
             ('IPDL_SOURCES', IPDLFile),
+            ('PREPROCESSED_IPDL_SOURCES', PreprocessedIPDLFile),
             ('PREPROCESSED_TEST_WEBIDL_FILES', PreprocessedTestWebIDLFile),
             ('PREPROCESSED_WEBIDL_FILES', PreprocessedWebIDLFile),
             ('TEST_WEBIDL_FILES', TestWebIDLFile),

python/mozbuild/mozbuild/test/backend/data/ipdl_sources/bar/moz.build

@@ -3,6 +3,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+PREPROCESSED_IPDL_SOURCES += [
+    'bar1.ipdl',
+]
+
 IPDL_SOURCES += [
     'bar.ipdl',
     'bar2.ipdlh',

python/mozbuild/mozbuild/test/backend/data/ipdl_sources/foo/moz.build

@@ -3,6 +3,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+PREPROCESSED_IPDL_SOURCES += [
+    'foo1.ipdl',
+]
+
 IPDL_SOURCES += [
     'foo.ipdl',
     'foo2.ipdlh',

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -656,7 +656,7 @@ class TestRecursiveMakeBackend(BackendTester):
             self.assertEqual(m, m2)
 
     def test_ipdl_sources(self):
-        """Test that IPDL_SOURCES are written to ipdlsrcs.mk correctly."""
+        """Test that PREPROCESSED_IPDL_SOURCES and IPDL_SOURCES are written to ipdlsrcs.mk correctly."""
         env = self._consume('ipdl_sources', RecursiveMakeBackend)
 
         manifest_path = mozpath.join(env.topobjdir,
@@ -667,9 +667,9 @@ class TestRecursiveMakeBackend(BackendTester):
         topsrcdir = env.topsrcdir.replace(os.sep, '/')
 
         expected = [
-            "ALL_IPDLSRCS := %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4),
+            "ALL_IPDLSRCS := bar1.ipdl foo1.ipdl %s/bar/bar.ipdl %s/bar/bar2.ipdlh %s/foo/foo.ipdl %s/foo/foo2.ipdlh" % tuple([topsrcdir] * 4),
             "CPPSRCS := UnifiedProtocols0.cpp",
-            "IPDLDIRS := %s/bar %s/foo" % (topsrcdir, topsrcdir),
+            "IPDLDIRS := %s/ipc/ipdl %s/bar %s/foo" % (env.topobjdir, topsrcdir, topsrcdir),
         ]
 
         found = [str for str in lines if str.startswith(('ALL_IPDLSRCS',

python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/bar/moz.build

@@ -3,6 +3,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+PREPROCESSED_IPDL_SOURCES += [
+    'bar1.ipdl',
+]
+
 IPDL_SOURCES += [
     'bar.ipdl',
     'bar2.ipdlh',

python/mozbuild/mozbuild/test/frontend/data/ipdl_sources/foo/moz.build

@@ -3,6 +3,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+PREPROCESSED_IPDL_SOURCES += [
+    'foo1.ipdl',
+]
+
 IPDL_SOURCES += [
     'foo.ipdl',
     'foo2.ipdlh',

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -30,6 +30,7 @@ from mozbuild.frontend.data import (
     JARManifest,
     LinkageMultipleRustLibrariesError,
     LocalInclude,
+    PreprocessedIPDLFile,
     Program,
     SdkFiles,
     SharedLibrary,
@@ -688,9 +689,12 @@ class TestEmitterBasic(unittest.TestCase):
         objs = self.read_topsrcdir(reader)
 
         ipdls = []
+        nonstatic_ipdls = []
         for o in objs:
             if isinstance(o, IPDLFile):
                 ipdls.append('%s/%s' % (o.relativedir, o.basename))
+            elif isinstance(o, PreprocessedIPDLFile):
+                nonstatic_ipdls.append('%s/%s' % (o.relativedir, o.basename))
 
         expected = [
             'bar/bar.ipdl',
@@ -699,7 +703,12 @@ class TestEmitterBasic(unittest.TestCase):
             'foo/foo2.ipdlh',
         ]
 
-        self.assertEqual(ipdls, expected)
+        expected = [
+            'bar/bar1.ipdl',
+            'foo/foo1.ipdl',
+        ]
+
+        self.assertEqual(nonstatic_ipdls, expected)
 
     def test_local_includes(self):
         """Test that LOCAL_INCLUDES is emitted correctly."""