From 3e6a4e9b95adba7a88ea2b9ae0fbf9d9260781a2 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 17 Aug 2026 19:42:24 +0200 Subject: [PATCH 1/2] Stop freezing inspect, it has too many non-frozen deps --- graalpython/com.oracle.graal.python.frozen/freeze_modules.py | 5 ----- .../graal/python/builtins/objects/module/FrozenModules.java | 3 --- 2 files changed, 8 deletions(-) diff --git a/graalpython/com.oracle.graal.python.frozen/freeze_modules.py b/graalpython/com.oracle.graal.python.frozen/freeze_modules.py index 7f75af6ee8..8903c8b4fe 100644 --- a/graalpython/com.oracle.graal.python.frozen/freeze_modules.py +++ b/graalpython/com.oracle.graal.python.frozen/freeze_modules.py @@ -53,10 +53,6 @@ '_weakrefset', 'types', 'enum', - # GraalPy change: don't freeze these, they are deprecated, CPython probably just forgot to remove them from here - # 'sre_constants', - # 'sre_parse', - # 'sre_compile', 'operator', 'keyword', 'heapq', @@ -81,7 +77,6 @@ 'datetime', 'contextlib', 'warnings', - 'inspect', ]), ('runpy - run module with -m', [ "importlib.util", diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java index b77815b744..e42226e466 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/module/FrozenModules.java @@ -84,7 +84,6 @@ private static final class Map { private static final PythonFrozenModule DATETIME = new PythonFrozenModule("DATETIME", "datetime", false); private static final PythonFrozenModule CONTEXTLIB = new PythonFrozenModule("CONTEXTLIB", "contextlib", false); private static final PythonFrozenModule WARNINGS = new PythonFrozenModule("WARNINGS", "warnings", false); - private static final PythonFrozenModule INSPECT = new PythonFrozenModule("INSPECT", "inspect", false); private static final PythonFrozenModule IMPORTLIB_UTIL = new PythonFrozenModule("IMPORTLIB_UTIL", "importlib.util", false); private static final PythonFrozenModule IMPORTLIB_MACHINERY = new PythonFrozenModule("IMPORTLIB_MACHINERY", "importlib.machinery", false); private static final PythonFrozenModule RUNPY = new PythonFrozenModule("RUNPY", "runpy", false); @@ -185,8 +184,6 @@ public static final PythonFrozenModule lookup(String name) { return Map.CONTEXTLIB; case "warnings": return Map.WARNINGS; - case "inspect": - return Map.INSPECT; case "importlib.util": return Map.IMPORTLIB_UTIL; case "importlib.machinery": From 61d13eed4527e126117cc1aa4a3c24f949c02190 Mon Sep 17 00:00:00 2001 From: Michael Simacek Date: Mon, 14 Sep 2026 21:49:03 +0200 Subject: [PATCH 2/2] Build bytecode files into resources jar --- mx.graalpython/mx_graalpython.py | 128 +++++++++++++++++++++++++++++++ mx.graalpython/suite.py | 37 ++++----- 2 files changed, 147 insertions(+), 18 deletions(-) diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index 9fc8d22642..63ea902fc9 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -2382,6 +2382,134 @@ def getResults(self): return results +class PythonResourceBuildTask(mx.ArchivableBuildTask): + def _marker(self): + return self.subject.get_output_root() + ".pyc.stamp" + + @staticmethod + def _cache_files(source): + basename = glob.escape(os.path.basename(source)[:-3]) + return glob.glob(os.path.join(os.path.dirname(source), "__pycache__", f"{basename}.graalpy*.pyc")) + + def _all_cache_files(self): + for root, _, files in os.walk(self.subject.output_dir()): + for filename in files: + if filename.endswith(".pyc"): + yield os.path.join(root, filename) + + def __str__(self): + return f'Copying and compiling Python resources in {self.subject.name}' + + def needsBuild(self, newestInput): + if self.args.force: + return True, 'forced build' + marker = mx.TimeStampFile(self._marker()) + if not marker.exists(): + return True, 'Python resource bytecode marker does not exist' + if newestInput and marker.isOlderThan(newestInput): + return True, f'{marker} is older than {newestInput}' + project = cast(PythonResourceProject, self.subject) + expected_sources = {relative: source for source, relative in project.getSourceFiles()} + actual_sources = {} + for root, _, files in os.walk(project.output_dir()): + for filename in files: + if not filename.endswith(".pyc"): + path = os.path.join(root, filename) + actual_sources[os.path.relpath(path, project.output_dir())] = path + if expected_sources.keys() != actual_sources.keys(): + return True, 'Python resource output does not match its sources' + for relative, source in expected_sources.items(): + if mx.TimeStampFile(actual_sources[relative]).isOlderThan(source): + return True, f'Python resource output is older than {source}' + expected_caches = set() + for relative in expected_sources: + if not relative.endswith(".py"): + continue + output_source = actual_sources[relative] + caches = self._cache_files(output_source) + if not caches: + return True, f'no bytecode cache for {output_source}' + if len(caches) > 1: + return True, f'multiple bytecode caches for {output_source}' + expected_caches.update(caches) + if mx.TimeStampFile(caches[0]).isOlderThan(output_source): + return True, f'bytecode cache is older than {output_source}' + if unexpected_caches := set(self._all_cache_files()) - expected_caches: + return True, f'unexpected bytecode cache {next(iter(unexpected_caches))}' + return False, 'all Python resource bytecode caches are up to date' + + def newestOutput(self): + return mx.TimeStampFile(self._marker()) + + def build(self): + project = cast(PythonResourceProject, self.subject) + output_dir = project.output_dir() + if os.path.exists(output_dir): + shutil.rmtree(output_dir) + for source, relative in project.getSourceFiles(): + output = os.path.join(output_dir, relative) + mx_util.ensure_dir_exists(os.path.dirname(output)) + shutil.copy2(source, output) + args = [ + "--PosixModuleBackend=java", + "--CompressionModulesBackend=java", + "--DisableFrozenModules", + "-B", + "-S", + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "checked-hash", + "-s", + output_dir, + output_dir, + ] + if do_run_python(args, jdk=mx.get_jdk(), minimal=True, cwd=self.subject.suite.dir): + return True + pathlib.Path(self._marker()).touch() + return True + + def clean(self, forBuild=False): + changed = False + if os.path.exists(self.subject.output_dir()): + shutil.rmtree(self.subject.output_dir()) + changed = True + if os.path.exists(self._marker()): + os.remove(self._marker()) + changed = True + return changed + + +class PythonResourceProject(ArchiveProject): + def __init__(self, suite, name, deps, workingSets, theLicense, **kwargs): + context = 'project ' + name + self.buildDependencies = mx.Suite._pop_list(kwargs, 'buildDependencies', context) + self.sourceDir = kwargs.pop('sourceDir') + super().__init__(suite, name, deps, workingSets, theLicense, **kwargs) + + def source_dir(self): + source_dir = mx_subst.path_substitutions.substitute(self.sourceDir) + return source_dir if os.path.isabs(source_dir) else os.path.join(self.dir, source_dir) + + def output_dir(self): + return self.get_output_root() + + def getSourceFiles(self): + ignore_regexps = [re.compile(s) for s in getattr(self, "ignorePatterns", [])] + source_dir = self.source_dir() + for root, dirs, files in os.walk(source_dir): + dirs[:] = [d for d in dirs if d != "__pycache__"] + for filename in files: + source = os.path.join(root, filename) + if not filename.endswith(".pyc") and not any(r.search(source) for r in ignore_regexps): + yield source, os.path.relpath(source, source_dir) + + def getBuildTask(self, args): + return PythonResourceBuildTask(self, args, 1) + + def deploy_binary_if_main(args): """if the active branch is the main branch, deploy binaries for the primary suite to remote maven repository.""" active_branch = SUITE.vc.active_branch(SUITE.dir) diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 6cdebace18..68374df264 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -828,30 +828,32 @@ }, "python-lib": { - "class": "ArchiveProject", - "outputDir": "graalpython/lib-python/3", + "class": "PythonResourceProject", + "sourceDir": "graalpython/lib-python/3", "type": "dir", "prefix": "", "ignorePatterns": [ - ".pyc", - "\\/__pycache__\\/", - "\\/test\\/", - "\\/tests\\/", - "\\/idle_test\\/", - "\\\\__pycache__\\\\", - "\\\\test\\\\", - "\\\\tests\\\\", - "\\\\idle_test\\\\", + "[/\\\\]test[/\\\\](?!support[/\\\\])", + "[/\\\\]tests[/\\\\]", + "[/\\\\]idle_test[/\\\\]", + ], + "buildDependencies": [ + "GRAALPYTHON", + "GRAALPYTHON-LAUNCHER", ], "license": ["PSF-License"], }, - "python-test-support-lib": { - "class": "ArchiveProject", - "outputDir": "graalpython/lib-python/3/test/support", - "prefix": "test/support", + "python-libgraalpy": { + "class": "PythonResourceProject", + "sourceDir": "graalpython/lib-graalpython", + "prefix": "", "ignorePatterns": [], - "license": ["PSF-License"], + "buildDependencies": [ + "GRAALPYTHON", + "GRAALPYTHON-LAUNCHER", + ], + "license": ["UPL", "MIT"], }, "graalpy_licenses": { @@ -1302,7 +1304,6 @@ "layout": { "./META-INF/resources/libpython/": [ "dependency:graalpython:python-lib/*", - "dependency:graalpython:python-test-support-lib/*", ], }, "maven": False, @@ -1319,7 +1320,7 @@ "description": "GraalVM Python lib-graalpython resources", "layout": { "./META-INF/resources/libgraalpy/": [ - "file:graalpython/lib-graalpython/*", + "dependency:graalpython:python-libgraalpy/*", ], }, "maven": False,