Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@
import org.graalvm.shadowed.org.jline.utils.Signals;

public class JLineConsoleHandler extends ConsoleHandler {
private static final String[] SIGNALS = {"INT", "QUIT", "TSTP", "CONT", "WINCH"};
private static final Terminal.Signal[] SIGNALS = Terminal.Signal.values();

private final InputStream inputStream;
private final OutputStream outputStream;
Expand Down Expand Up @@ -252,14 +252,14 @@ public String readLine(String prompt) {
private static Object[] stashSignalHandlers() {
Object[] handlers = new Object[SIGNALS.length];
for (int i = 0; i < SIGNALS.length; i++) {
handlers[i] = Signals.registerDefault(SIGNALS[i]);
handlers[i] = Signals.registerDefault(SIGNALS[i].name());
}
return handlers;
}

private static void restoreSignalHandlers(Object[] handlers) {
for (int i = 0; i < SIGNALS.length; i++) {
Signals.unregister(SIGNALS[i], handlers[i]);
Signals.unregister(SIGNALS[i].name(), handlers[i]);
}
}

Expand Down
18 changes: 14 additions & 4 deletions graalpython/com.oracle.graal.python.test/src/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -554,7 +554,7 @@ def interrupt_process(process: subprocess.Popen):
process.send_signal(signal.SIGINT)
process.wait(3)
return
except (OSError, subprocess.TimeoutExpired):
except (OSError, ValueError, subprocess.TimeoutExpired):
pass
process.terminate()
try:
Expand Down Expand Up @@ -589,7 +589,7 @@ def partition_tests_into_processes(self, suites: list['TestSuite']) -> list[list
else:
per_file_suites, unpartitioned = partition_list(
suites,
lambda suite: suite.test_file.config.new_worker_per_file,
lambda suite: suite.test_file.config.new_worker_per_file or suite.test_file.test_config.subprocess_args,
)
partitions = [suite.collected_tests for suite in per_file_suites]

Expand All @@ -613,7 +613,7 @@ def partition_tests_into_processes(self, suites: list['TestSuite']) -> list[list
timed_files.sort(reverse=True, key=lambda x: x[0])

# Greedily assign to balance by timing sum
process_loads = [[] for _ in range(self.num_processes)]
process_loads = [[] for _ in range(min(self.num_processes, len(timed_files)))]
process_times = [0.0] * self.num_processes
for t, suite in timed_files:
i = process_times.index(min(process_times))
Expand Down Expand Up @@ -697,6 +697,8 @@ class SubprocessWorker:
def __init__(self, worker_id: int, runner: ParallelTestRunner, tests: list['Test']):
self.prefix = f'[worker-{worker_id + 1}] '
self.runner = runner
self.subprocess_args = tests[0].test_file.test_config.subprocess_args
assert all(test.test_file.test_config.subprocess_args == self.subprocess_args for test in tests)
self.stop_event = runner.stop_event
self.lock = threading.RLock()
self.remaining_test_ids = [test.test_id for test in tests]
Expand Down Expand Up @@ -805,8 +807,9 @@ def run_in_subprocess_and_watch(self):
self.last_started_time = time.time()
cmd = [
sys.executable,
'-u',
*self.runner.subprocess_args,
*self.subprocess_args,
'-u',
__file__,
'worker',
'--port', str(port),
Expand Down Expand Up @@ -922,14 +925,20 @@ class TestFileConfig:
serial: bool | None = None
partial_splits: bool | None = None
per_test_timeout: float | None = None
subprocess_args: tuple[str, ...] = ()
exclude: bool = False

@classmethod
def from_dict(cls, config: dict):
subprocess_args = tuple(config.get('subprocess_args', ()))
subprocess_args_on = config.get('subprocess_args_on')
if subprocess_args and subprocess_args_on is not None and not platform_keys_match(subprocess_args_on):
subprocess_args = ()
return cls(
serial=config.get('serial', cls.serial),
partial_splits=config.get('partial_splits_individual_tests', cls.partial_splits),
per_test_timeout=config.get('per_test_timeout', cls.per_test_timeout),
subprocess_args=subprocess_args,
exclude=platform_keys_match(config.get('exclude_on', ())),
)

Expand All @@ -938,6 +947,7 @@ def combine(self, other: 'TestFileConfig'):
serial=(self.serial if other.serial is None else other.serial),
partial_splits=(self.partial_splits if other.partial_splits is None else other.partial_splits),
per_test_timeout=(self.per_test_timeout if other.per_test_timeout is None else other.per_test_timeout),
subprocess_args=self.subprocess_args + other.subprocess_args,
exclude=self.exclude or other.exclude,
)

Expand Down
13 changes: 9 additions & 4 deletions graalpython/com.oracle.graal.python.test/src/tests/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def _package_present(site_packages_dir: Path, package: str, version: str) -> boo
return os.path.isdir(site_packages_dir / normalized)


def ensure_packages(**package_specs):
def ensure_packages(*, use_current_python=False, **package_specs):
import site
package_names = "-".join(package_specs.keys())
venv_dir = find_rootdir() / f'{sys.implementation.name}-{package_names}-venv'
Expand All @@ -125,14 +125,15 @@ def ensure_packages(**package_specs):
import subprocess
package_specs = [f'{p}=={v}' for p, v in package_specs.items()]
print(f'installing {package_specs} in {venv_dir}')
system_python = install_venv(venv_dir)
install_venv(venv_dir, use_current_python=use_current_python)
site_packages_dir = _venv_site_packages(venv_dir, py_executable)
subprocess.run([py_executable, "-m", "pip", "install", *package_specs], check=True)
print(f'{package_specs} installed in {venv_dir}')

pyvenv_site = str(site_packages_dir)
if os.path.normcase(os.path.normpath(pyvenv_site)) not in {os.path.normcase(os.path.normpath(entry)) for entry in sys.path}:
site.addsitedir(pyvenv_site)
return venv_dir


def get_setuptools(setuptools='67.6.1'):
Expand Down Expand Up @@ -169,9 +170,13 @@ def _system_python_for_venv():
return python


def install_venv(venv_path: Path) -> bool:
def install_venv(venv_path: Path, use_current_python=False) -> bool:
"""Installs a virtual environment at the given path."""
if not sys.executable or (sys.platform.startswith('win32') and sys.implementation.name == "graalpy"):
if use_current_python:
import subprocess
subprocess.run([sys.executable, "-m", "venv", str(venv_path)], check=True)
return False
elif not sys.executable or (sys.platform.startswith('win32') and sys.implementation.name == "graalpy"):
# When running in a PolyBench benchmark context sys.executable is unset
# And thus we must defer to the system's python
# Deferring to the system's python is fine as it will only be used to install setuptools
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,13 @@

if sys.implementation.name == "graalpy":
import autopatch_cargo
from tests.util import _is_sandboxed, skip_if_sandboxed

if os.name == "nt" and not _is_sandboxed():
from tests import ensure_packages

paatch_venv = ensure_packages(use_current_python=True, paatch="1.20.3")
os.environ["PATH"] = os.pathsep.join((str(paatch_venv / "Scripts"), os.environ.get("PATH", "")))


class AutoPatchCargoTest(unittest.TestCase):
Expand Down Expand Up @@ -187,6 +194,7 @@ def prepare_workspace(self, checksum=None):
""".format(checksum=checksum),
)

@skip_if_sandboxed("requires an external patch executable")
def test_patches_locked_crate_and_adds_cargo_override(self):
archive, checksum = self.prepare_crate_archive()
cached_crate = self.cargo_home / "registry" / "src" / "made-up-index" / "made-up-crate-1.2.3"
Expand Down Expand Up @@ -231,6 +239,7 @@ def test_patches_locked_crate_and_adds_cargo_override(self):

assert autopatch_cargo.auto_patch_tree(self.workspace, repository) == 0

@skip_if_sandboxed("requires an external patch executable")
def test_downloads_and_verifies_uncached_crate(self):
archive, checksum = self.prepare_crate_archive(cached=False)
repository = self.prepare_repository()
Expand All @@ -244,6 +253,7 @@ def test_downloads_and_verifies_uncached_crate(self):
patched_crate = self.workspace / ".graalpy" / "crates" / "made-up-crate-1.2.3"
assert '"patched"' in (patched_crate / "src" / "lib.rs").read_text()

@skip_if_sandboxed("requires an external patch executable")
def test_accepts_patch_already_applied_to_cached_archive(self):
_, checksum = self.prepare_crate_archive(message="patched")
repository = self.prepare_repository()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,13 +45,15 @@ def posix_module_backend(self):
return 'cpython'
__graalpython__ = GP()

import fcntl
import os
import subprocess
import sys
import tempfile
import time
import unittest
import sys

if sys.platform != 'win32':
import fcntl

PREFIX = 'select_graalpython_test'
TEMP_DIR = tempfile.gettempdir()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,13 @@

PAGESIZE = mmap.PAGESIZE
FIND_BUFFER_SIZE = 1024 # keep in sync with FindNode#BUFFER_SIZE
NAMED_MMAP_SUPPORTED = (
sys.platform == "win32"
and (sys.implementation.name != "graalpy" or __graalpython__.posix_module_backend() == "native")
)


@unittest.skipUnless(sys.platform == "win32", "requires Windows named mmap support")
@unittest.skipUnless(NAMED_MMAP_SUPPORTED, "requires native Windows named mmap support")
def test_named_mmap_clears_windows_last_error():
import ctypes
import _winapi
Expand All @@ -66,7 +70,7 @@ def test_named_mmap_clears_windows_last_error():
ctypes.set_last_error(0)


@unittest.skipUnless(sys.platform == "win32", "requires Windows named mmap support")
@unittest.skipUnless(NAMED_MMAP_SUPPORTED, "requires native Windows named mmap support")
def test_windows_tagname_as_third_positional_argument():
data = b"named mmap"
tagname = f"graalpy-mmap-test-{os.getpid()}-{time.time_ns()}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def __index__(self):
return 1

v = MyVal()
if sys.platform == 'win32':
if sys.platform == 'win32' and __graalpython__.posix_module_backend() != 'java':
with self.assertRaises(OSError):
select.select([], [], [], v)
else:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,9 @@ test.test_lzma.OpenTestCase.test_bad_params @ darwin-arm64,linux-aarch64,linux-a
test.test_lzma.OpenTestCase.test_binary_modes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_encoding @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_encoding_error_handler @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github
test.test_lzma.OpenTestCase.test_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_format_and_filters @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github
test.test_lzma.OpenTestCase.test_newline @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_text_modes @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_with_pathlike_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github
test.test_lzma.OpenTestCase.test_with_pathlike_filename @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
test.test_lzma.OpenTestCase.test_x_mode @ darwin-arm64,linux-aarch64,linux-aarch64-github,linux-x86_64,linux-x86_64-github,win32-AMD64,win32-AMD64-github
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2017, 2025, Oracle and/or its affiliates.
* Copyright (c) 2017, 2026, Oracle and/or its affiliates.
* Copyright (c) 2013, Regents of the University of California
*
* All rights reserved.
Expand Down Expand Up @@ -40,7 +40,7 @@
* Most builtins are not OS specific. If specified, the builtin is included only if the os
* matches
*/
PythonOS os() default PythonOS.PLATFORM_ANY;
PythonOS[] os() default PythonOS.PLATFORM_ANY;

PythonBuiltinClassType[] extendClasses() default {};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -467,7 +467,8 @@ private static void filterBuiltins(List<PythonBuiltins> builtins) {
} else {
CoreFunctions annotation = builtin.getClass().getAnnotation(CoreFunctions.class);
builtin.setNeedsPostInitialize(annotation.isEager() || annotation.extendClasses().length != 0);
if (annotation.os() != PythonOS.PLATFORM_ANY && annotation.os() != currentOs) {
List<PythonOS> supportedOS = Arrays.asList(annotation.os());
if (!supportedOS.contains(PythonOS.PLATFORM_ANY) && !supportedOS.contains(currentOs)) {
toRemove.add(builtin);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@
*/
package com.oracle.graal.python.builtins.modules;

import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_DARWIN;
import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_LINUX;
import static com.oracle.graal.python.builtins.PythonBuiltinClassType.ValueError;
import static com.oracle.graal.python.runtime.PosixConstants.F_RDLCK;
import static com.oracle.graal.python.runtime.PosixConstants.F_UNLCK;
Expand Down Expand Up @@ -93,7 +95,7 @@
import com.oracle.truffle.api.nodes.Node;
import com.oracle.truffle.api.strings.TruffleString;

@CoreFunctions(defineModule = "fcntl")
@CoreFunctions(defineModule = "fcntl", os = {PLATFORM_LINUX, PLATFORM_DARWIN})
public final class FcntlModuleBuiltins extends PythonBuiltins {
private static final TruffleString T_FCNTL_FLOCK = tsLiteral("fcntl.flock");
private static final TruffleString T_FCNTL_LOCKF = tsLiteral("fcntl.lockf");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@
import com.oracle.graal.python.nodes.util.CastToJavaIntExactNode;
import com.oracle.graal.python.runtime.PythonContext;
import com.oracle.graal.python.runtime.object.PFactory;
import com.oracle.graal.python.util.PythonUtils;
import com.oracle.truffle.api.dsl.Bind;
import com.oracle.truffle.api.dsl.Cached;
import com.oracle.truffle.api.dsl.Cached.Exclusive;
Expand Down Expand Up @@ -230,6 +231,10 @@ static PBytes doBytes(LZMADecompressor self, PBytesLike data, int maxLength,
@Exclusive @Cached LZMANodes.DecompressNode decompress) {
byte[] bytes = toBytes.execute(inliningTarget, data.getSequenceStorage());
int len = data.getSequenceStorage().length();
// Incremental decompression uses the backing array length to detect appended input.
if (bytes.length != len) {
bytes = PythonUtils.arrayCopyOf(bytes, len);
}
return PFactory.createBytes(language, decompress.execute(inliningTarget, self, bytes, len, maxLength));

}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@
import com.oracle.graal.python.runtime.PosixSupportLibrary;
import com.oracle.graal.python.runtime.PosixSupportLibrary.PosixException;
import com.oracle.graal.python.runtime.PythonContext;
import com.oracle.graal.python.runtime.nativeaccess.NativeAccessSupport;
import com.oracle.graal.python.runtime.nativeaccess.NativeContext;
import com.oracle.graal.python.runtime.object.PFactory;
import com.oracle.graal.python.runtime.sequence.storage.ByteSequenceStorage;
Expand Down Expand Up @@ -312,7 +313,7 @@ static PMMap doFile(VirtualFrame frame, Object clazz, int fd, long lengthIn, Obj
}
PythonContext context = PythonContext.get(inliningTarget);
PMMap mmap = PFactory.createMMap(context, clazz, getInstanceShape.execute(clazz), mmapHandle, dupFd, length, access);
if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32) {
if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && NativeAccessSupport.isAvailable()) {
NativeContext.setLastError(0);
}
return mmap;
Expand Down
8 changes: 8 additions & 0 deletions graalpython/lib-python/3/test/conftest.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,14 @@
tags_dir = '../../../com.oracle.graal.python.test/src/tests/unittest_tags'
new_worker_per_file = true

[[test_rules]]
# The pure-Python JSON recursion tests rely on the VM stack overflow being converted to
# RecursionError. The standalone's normal 16 MiB worker stack is too large on Windows.
# The runner collects test_json as a package, so this is the narrowest configurable scope.
selector = ['test_json']
subprocess_args = ['--vm.Xss1m']
subprocess_args_on = ['win32', 'win32-github']

[[test_rules]]
# A list of tests that cannot run in parallel with other tests
serial = true
Expand Down
Loading