[GR-79617] Ship bytecode files in resources JAR

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -53,10 +53,6 @@
         '_weakrefset',
         'types',
         'enum',
-        # GraalPy change: don't freeze these, they are deprecated, CPython probably just forgot to remove them from here
-        # 'sre_constants',
-        # 'sre_parse',
-        # 'sre_compile',
         'operator',
         'keyword',
         'heapq',
@@ -81,7 +77,6 @@
         'datetime',
         'contextlib',
         'warnings',
-        'inspect',
         ]),
     ('runpy - run module with -m', [
         "importlib.util",

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java

@@ -84,7 +84,6 @@ private static final class Map {
         private static final PythonFrozenModule DATETIME = new PythonFrozenModule("DATETIME", "datetime", false);
         private static final PythonFrozenModule CONTEXTLIB = new PythonFrozenModule("CONTEXTLIB", "contextlib", false);
         private static final PythonFrozenModule WARNINGS = new PythonFrozenModule("WARNINGS", "warnings", false);
-        private static final PythonFrozenModule INSPECT = new PythonFrozenModule("INSPECT", "inspect", false);
         private static final PythonFrozenModule IMPORTLIB_UTIL = new PythonFrozenModule("IMPORTLIB_UTIL", "importlib.util", false);
         private static final PythonFrozenModule IMPORTLIB_MACHINERY = new PythonFrozenModule("IMPORTLIB_MACHINERY", "importlib.machinery", false);
         private static final PythonFrozenModule RUNPY = new PythonFrozenModule("RUNPY", "runpy", false);
@@ -185,8 +184,6 @@ public static final PythonFrozenModule lookup(String name) {
                 return Map.CONTEXTLIB;
             case "warnings":
                 return Map.WARNINGS;
-            case "inspect":
-                return Map.INSPECT;
             case "importlib.util":
                 return Map.IMPORTLIB_UTIL;
             case "importlib.machinery":

mx.graalpython/mx_graalpython.py

@@ -2382,6 +2382,134 @@ def getResults(self):
             return results
 
 
+class PythonResourceBuildTask(mx.ArchivableBuildTask):
+    def _marker(self):
+        return self.subject.get_output_root() + ".pyc.stamp"
+
+    @staticmethod
+    def _cache_files(source):
+        basename = glob.escape(os.path.basename(source)[:-3])
+        return glob.glob(os.path.join(os.path.dirname(source), "__pycache__", f"{basename}.graalpy*.pyc"))
+
+    def _all_cache_files(self):
+        for root, _, files in os.walk(self.subject.output_dir()):
+            for filename in files:
+                if filename.endswith(".pyc"):
+                    yield os.path.join(root, filename)
+
+    def __str__(self):
+        return f'Copying and compiling Python resources in {self.subject.name}'
+
+    def needsBuild(self, newestInput):
+        if self.args.force:
+            return True, 'forced build'
+        marker = mx.TimeStampFile(self._marker())
+        if not marker.exists():
+            return True, 'Python resource bytecode marker does not exist'
+        if newestInput and marker.isOlderThan(newestInput):
+            return True, f'{marker} is older than {newestInput}'
+        project = cast(PythonResourceProject, self.subject)
+        expected_sources = {relative: source for source, relative in project.getSourceFiles()}
+        actual_sources = {}
+        for root, _, files in os.walk(project.output_dir()):
+            for filename in files:
+                if not filename.endswith(".pyc"):
+                    path = os.path.join(root, filename)
+                    actual_sources[os.path.relpath(path, project.output_dir())] = path
+        if expected_sources.keys() != actual_sources.keys():
+            return True, 'Python resource output does not match its sources'
+        for relative, source in expected_sources.items():
+            if mx.TimeStampFile(actual_sources[relative]).isOlderThan(source):
+                return True, f'Python resource output is older than {source}'
+        expected_caches = set()
+        for relative in expected_sources:
+            if not relative.endswith(".py"):
+                continue
+            output_source = actual_sources[relative]
+            caches = self._cache_files(output_source)
+            if not caches:
+                return True, f'no bytecode cache for {output_source}'
+            if len(caches) > 1:
+                return True, f'multiple bytecode caches for {output_source}'
+            expected_caches.update(caches)
+            if mx.TimeStampFile(caches[0]).isOlderThan(output_source):
+                return True, f'bytecode cache is older than {output_source}'
+        if unexpected_caches := set(self._all_cache_files()) - expected_caches:
+            return True, f'unexpected bytecode cache {next(iter(unexpected_caches))}'
+        return False, 'all Python resource bytecode caches are up to date'
+
+    def newestOutput(self):
+        return mx.TimeStampFile(self._marker())
+
+    def build(self):
+        project = cast(PythonResourceProject, self.subject)
+        output_dir = project.output_dir()
+        if os.path.exists(output_dir):
+            shutil.rmtree(output_dir)
+        for source, relative in project.getSourceFiles():
+            output = os.path.join(output_dir, relative)
+            mx_util.ensure_dir_exists(os.path.dirname(output))
+            shutil.copy2(source, output)
+        args = [
+            "--PosixModuleBackend=java",
+            "--CompressionModulesBackend=java",
+            "--DisableFrozenModules",
+            "-B",
+            "-S",
+            "-m",
+            "compileall",
+            "-f",
+            "-q",
+            "--invalidation-mode",
+            "checked-hash",
+            "-s",
+            output_dir,
+            output_dir,
+        ]
+        if do_run_python(args, jdk=mx.get_jdk(), minimal=True, cwd=self.subject.suite.dir):
+            return True
+        pathlib.Path(self._marker()).touch()
+        return True
+
+    def clean(self, forBuild=False):
+        changed = False
+        if os.path.exists(self.subject.output_dir()):
+            shutil.rmtree(self.subject.output_dir())
+            changed = True
+        if os.path.exists(self._marker()):
+            os.remove(self._marker())
+            changed = True
+        return changed
+
+
+class PythonResourceProject(ArchiveProject):
+    def __init__(self, suite, name, deps, workingSets, theLicense, **kwargs):
+        context = 'project ' + name
+        self.buildDependencies = mx.Suite._pop_list(kwargs, 'buildDependencies', context)
+        self.sourceDir = kwargs.pop('sourceDir')
+        super().__init__(suite, name, deps, workingSets, theLicense, **kwargs)
+
+    def source_dir(self):
+        source_dir = mx_subst.path_substitutions.substitute(self.sourceDir)
+        return source_dir if os.path.isabs(source_dir) else os.path.join(self.dir, source_dir)
+
+    def output_dir(self):
+        return self.get_output_root()
+
+    def getSourceFiles(self):
+        ignore_regexps = [re.compile(s) for s in getattr(self, "ignorePatterns", [])]
+        source_dir = self.source_dir()
+        for root, dirs, files in os.walk(source_dir):
+            dirs[:] = [d for d in dirs if d != "__pycache__"]
+            for filename in files:
+                source = os.path.join(root, filename)
+                if not filename.endswith(".pyc") and not any(r.search(source) for r in ignore_regexps):
+                    yield source, os.path.relpath(source, source_dir)
+
+    def getBuildTask(self, args):
+        return PythonResourceBuildTask(self, args, 1)
+
+
 def deploy_binary_if_main(args):
     """if the active branch is the main branch, deploy binaries for the primary suite to remote maven repository."""
     active_branch = SUITE.vc.active_branch(SUITE.dir)

mx.graalpython/suite.py

@@ -828,30 +828,32 @@
         },
 
         "python-lib": {
-            "class": "ArchiveProject",
-            "outputDir": "graalpython/lib-python/3",
+            "class": "PythonResourceProject",
+            "sourceDir": "graalpython/lib-python/3",
             "type": "dir",
             "prefix": "",
             "ignorePatterns": [
-                ".pyc",
-                "\\/__pycache__\\/",
-                "\\/test\\/",
-                "\\/tests\\/",
-                "\\/idle_test\\/",
-                "\\\\__pycache__\\\\",
-                "\\\\test\\\\",
-                "\\\\tests\\\\",
-                "\\\\idle_test\\\\",
+                "[/\\\\]test[/\\\\](?!support[/\\\\])",
+                "[/\\\\]tests[/\\\\]",
+                "[/\\\\]idle_test[/\\\\]",
+            ],
+            "buildDependencies": [
+                "GRAALPYTHON",
+                "GRAALPYTHON-LAUNCHER",
             ],
             "license": ["PSF-License"],
         },
 
-        "python-test-support-lib": {
-            "class": "ArchiveProject",
-            "outputDir": "graalpython/lib-python/3/test/support",
-            "prefix": "test/support",
+        "python-libgraalpy": {
+            "class": "PythonResourceProject",
+            "sourceDir": "graalpython/lib-graalpython",
+            "prefix": "",
             "ignorePatterns": [],
-            "license": ["PSF-License"],
+            "buildDependencies": [
+                "GRAALPYTHON",
+                "GRAALPYTHON-LAUNCHER",
+            ],
+            "license": ["UPL", "MIT"],
         },
 
         "graalpy_licenses": {
@@ -1302,7 +1304,6 @@
             "layout": {
                 "./META-INF/resources/libpython/": [
                     "dependency:graalpython:python-lib/*",
-                    "dependency:graalpython:python-test-support-lib/*",
                 ],
             },
             "maven": False,
@@ -1319,7 +1320,7 @@
             "description": "GraalVM Python lib-graalpython resources",
             "layout": {
                 "./META-INF/resources/libgraalpy/": [
-                    "file:graalpython/lib-graalpython/*",
+                    "dependency:graalpython:python-libgraalpy/*",
                 ],
             },
             "maven": False,