Improves test suite performance by 8.5-25x [experiment!]

.github/workflows/test-suite.yml

@@ -857,7 +857,7 @@ jobs:
       PLATFORM: ${{ matrix.x64 && 'x64' || 'x86' }}
       THREAD_SAFE: "${{ matrix.zts && '1' || '0' }}"
       INTRINSICS: "${{ matrix.zts && 'AVX2' || '' }}"
-      PARALLEL: -j2
+      PARALLEL: ${{ matrix.asan && '-j2' || '' }}
       OPCACHE: "${{ matrix.opcache && '1' || '0' }}"
       ASAN: "${{ matrix.asan && '1' || '0' }}"
       CLANG_TOOLSET: "${{ matrix.clang && '1' || '0' }}"

README.md

@@ -97,15 +97,15 @@ can be determined using `nproc`.
 PHP ships with an extensive test suite, the command `make test` is used after
 successful compilation of the sources to run this test suite.
 
-It is possible to run tests using multiple cores by setting `-jN` in
-`TEST_PHP_ARGS` or `TESTS`:
+Tests run in parallel by default, using up to 10 detected logical processors.
+Set `-jN` in `TEST_PHP_ARGS` or `TESTS` to override the worker count:
 
 ```shell
 make TEST_PHP_ARGS=-j4 test
 ```
 
-Shall run `make test` with a maximum of 4 concurrent jobs: Generally the maximum
-number of jobs should not exceed the number of cores available.
+This runs `make test` with a maximum of 4 concurrent jobs. Alternatively,
+use `-j1` to run tests sequentially.
 
 Use the `TEST_PHP_ARGS` or `TESTS` variable to test only specific directories:
 

Zend/tests/arginfo_zpp_mismatch.phpt

@@ -10,6 +10,36 @@ if (getenv('SKIP_MSAN')) die("skip msan misses interceptors for some functions")
 
 require __DIR__ . "/arginfo_zpp_mismatch.inc";
 
+function testWeakSysvIpcFunction($function): bool {
+    if (!in_array($function, ['msg_queue_exists', 'msg_get_queue', 'sem_get', 'shm_attach'], true)) {
+        return false;
+    }
+
+    for ($argumentCount = 0; $argumentCount <= 8; $argumentCount++) {
+        $arguments = array_fill(0, $argumentCount, null);
+        if ($function === 'msg_queue_exists' && $argumentCount >= 1) {
+            $arguments[0] = 1;
+        }
+        if (($function === 'sem_get' || $function === 'shm_attach') && $argumentCount >= 3) {
+            $arguments[2] = 0600;
+        }
+
+        try {
+            $result = @$function(...$arguments);
+            if ($result instanceof SysvMessageQueue) {
+                msg_remove_queue($result);
+            } elseif ($result instanceof SysvSemaphore) {
+                sem_remove($result);
+            } elseif ($result instanceof SysvSharedMemory) {
+                shm_remove($result);
+            }
+        } catch (Throwable) {
+        }
+    }
+
+    return true;
+}
+
 function test($function) {
     if (skipFunction($function)) {
         return;
@@ -21,6 +51,10 @@ function test($function) {
     } else {
         echo "Testing " . get_class($function[0]) . "::$function[1]\n";
     }
+    if (testWeakSysvIpcFunction($function)) {
+        ob_end_clean();
+        return;
+    }
     try {
         @$function();
     } catch (Throwable) {

Zend/tests/new_oom.phpt

@@ -1,5 +1,5 @@
 --TEST--
-Test OOM on new of each class
+Test OOM on new of each instantiable class
 --SKIPIF--
 <?php
 if (getenv("USE_ZEND_ALLOC") === "0") die("skip requires zmm");
@@ -8,14 +8,102 @@ if (getenv("SKIP_SLOW_TESTS")) die('skip slow test');
 --FILE--
 <?php
 
+function getOomProcessCount(): int
+{
+    $processCount = PHP_OS_FAMILY === 'Windows'
+        ? getenv('NUMBER_OF_PROCESSORS')
+        : shell_exec('getconf _NPROCESSORS_ONLN 2>/dev/null');
+    if (!is_string($processCount)) {
+        return 1;
+    }
+
+    $processCount = filter_var(trim($processCount), FILTER_VALIDATE_INT, [
+        'options' => ['min_range' => 1],
+    ]);
+
+    return $processCount === false ? 1 : min($processCount, 4);
+}
+
+function startOomTest(string $php, string $file, string $class): ?array
+{
+    $output = tmpfile();
+    if ($output === false) {
+        echo "Class $class failed\nUnable to create output file\n";
+        return null;
+    }
+
+    $process = proc_open(
+        [$php, '--no-php-ini', $file, $class],
+        [
+            0 => ['null'],
+            1 => $output,
+            2 => ['redirect', 1],
+        ],
+        $pipes,
+    );
+    if (!is_resource($process)) {
+        fclose($output);
+        echo "Class $class failed\nUnable to start process\n";
+        return null;
+    }
+
+    return [
+        'class' => $class,
+        'output' => $output,
+        'process' => $process,
+    ];
+}
+
+function finishOomTest(array $test): bool
+{
+    $status = proc_get_status($test['process']);
+    if ($status['running']) {
+        return false;
+    }
+
+    proc_close($test['process']);
+    rewind($test['output']);
+    $output = stream_get_contents($test['output']);
+    fclose($test['output']);
+
+    if ($status['signaled']) {
+        echo "Class {$test['class']} failed\n";
+        echo "Process terminated by signal {$status['termsig']}\n";
+    } elseif ($output && preg_match('(^\nFatal error: Allowed memory size of [0-9]+ bytes exhausted[^\r\n]* \(tried to allocate [0-9]+ bytes\) in [^\r\n]+ on line [0-9]+\nStack trace:\n(#[0-9]+ [^\r\n]+\n)+$)', $output) !== 1) {
+        echo "Class {$test['class']} failed\n";
+        echo $output, "\n";
+    }
+
+    return true;
+}
+
 $file = __DIR__ . '/new_oom.inc';
 $php = PHP_BINARY;
+$classes = array_filter(
+    get_declared_classes(),
+    static fn(string $class): bool => (new ReflectionClass($class))->isInstantiable(),
+);
+$tests = [];
+$processCount = getOomProcessCount();
 
-foreach (get_declared_classes() as $class) {
-    $output = shell_exec("$php --no-php-ini $file $class 2>&1");
-    if ($output && preg_match('(^\nFatal error: Allowed memory size of [0-9]+ bytes exhausted[^\r\n]* \(tried to allocate [0-9]+ bytes\) in [^\r\n]+ on line [0-9]+\nStack trace:\n(#[0-9]+ [^\r\n]+\n)+$)', $output) !== 1) {
-        echo "Class $class failed\n";
-        echo $output, "\n";
+while ($classes || $tests) {
+    while ($classes && count($tests) < $processCount) {
+        $class = array_shift($classes);
+        $test = startOomTest($php, $file, $class);
+        if ($test !== null) {
+            $tests[] = $test;
+        }
+    }
+
+    $testFinished = false;
+    foreach ($tests as $index => $test) {
+        if (finishOomTest($test)) {
+            unset($tests[$index]);
+            $testFinished = true;
+        }
+    }
+    if (!$testFinished) {
+        usleep(1000);
     }
 }
 

docs/source/miscellaneous/writing-tests.rst

@@ -192,6 +192,11 @@ When you are testing your test case it's really important to make sure that you
 temporary resources (eg files) that you used in the test. There is a special ``--CLEAN--`` section
 to help you do this — see `here <#clean>`_.
 
+Tests run in parallel by default. Mutable resources such as files, directories, ports, database
+objects, and IPC identifiers must therefore be unique to each test. Read-only fixtures may be
+shared. If a resource cannot be isolated, declare the narrowest applicable conflict using
+``--CONFLICTS--`` or a ``SCOPED_CONFLICTS`` file.
+
 Another good check is to look at what lines of code in the PHP source your test case covers. This is
 easy to do, there are some instructions on the `PHP Wiki
 <https://wiki.php.net/doc/articles/writing-tests>`_.
@@ -683,6 +688,9 @@ An alternative to have a ``--CONFLICTS--`` section is to add a file named ``CONF
 directory containing the tests. The contents of the ``CONFLICTS`` file must have the same format as
 the contents of the ``--CONFLICTS--`` section.
 
+Unlike a ``SCOPED_CONFLICTS`` file, a ``CONFLICTS`` file also prevents tests in its own directory
+from running concurrently, regardless of whether the directory has a ``MAX_CONCURRENCY`` file.
+
 **Required:** No.
 
 **Format:** One conflict key per line. Comment lines starting with # are also allowed.
@@ -696,6 +704,33 @@ Example 1 (snippet):
 
 Example 1 (full): :ref:`conflicts_1.phpt`
 
+``SCOPED_CONFLICTS`` file
+-------------------------
+
+**Description:** This file is only relevant for parallel test execution. It specifies directory
+conflict keys that tests in the same directory may share. Tests outside the directory that use the
+same key remain mutually exclusive with the directory. Use this when a directory may run internally
+in parallel but must not overlap another user of the same shared resource.
+
+This does not limit concurrency within the directory. Add a ``MAX_CONCURRENCY`` file separately if a
+numerical cap is also required.
+
+**Required:** No.
+
+**Format:** One conflict key per line. Comment lines starting with # are also allowed.
+
+``MAX_CONCURRENCY`` file
+------------------------
+
+**Description:** This file is only relevant for parallel test execution. It limits how many workers
+may simultaneously execute tests from its directory. It does not change the behavior of
+``CONFLICTS`` or ``SCOPED_CONFLICTS``. This allows a bounded amount of parallelism for tests that
+are safe to overlap but would otherwise contend excessively for a shared resource.
+
+**Required:** No.
+
+**Format:** A positive integer. Comment lines starting with # are also allowed.
+
 ``--WHITESPACE_SENSITIVE--``
 ----------------------------
 

ext/curl/tests/MAX_CONCURRENCY

@@ -0,0 +1 @@
+4

ext/curl/tests/bug48203_multi.phpt

@@ -13,7 +13,7 @@ if (curl_version()['version_number'] === 0x080a00) {
 <?php
 include 'server.inc';
 function checkForClosedFilePointer($target_url, $curl_option, $description) {
-    $fp = fopen(__DIR__ . '/bug48203.tmp', 'w');
+    $fp = fopen(__DIR__ . '/bug48203_multi.tmp', 'w');
 
     $ch1 = curl_init();
     $ch2 = curl_init();
@@ -70,7 +70,7 @@ foreach($options_to_check as $option) {
 
 ?>
 --CLEAN--
-<?php @unlink(__DIR__ . '/bug48203.tmp'); ?>
+<?php @unlink(__DIR__ . '/bug48203_multi.tmp'); ?>
 --EXPECTF--
 Warning: curl_multi_add_handle(): CURLOPT_STDERR resource has gone away, resetting to stderr in %s on line %d
 %A

ext/curl/tests/bug54798-unix.phpt

@@ -12,7 +12,7 @@ if(substr(PHP_OS, 0, 3) == 'WIN' ) {
 <?php
 
 function checkForClosedFilePointer($host, $curl_option, $description) {
-    $fp = fopen(__DIR__ . '/bug54798.tmp', 'w+');
+    $fp = fopen(__DIR__ . '/bug54798-unix.tmp', 'w+');
 
     $ch = curl_init();
 
@@ -52,7 +52,7 @@ foreach($options_to_check as $option) {
 
 ?>
 --CLEAN--
-<?php @unlink(__DIR__ . '/bug54798.tmp'); ?>
+<?php @unlink(__DIR__ . '/bug54798-unix.tmp'); ?>
 --EXPECTF--
 %a
 %aOk for CURLOPT_STDERR

ext/gd/tests/createfromwbmp2.phpt

@@ -8,7 +8,7 @@ gd
 ?>
 --FILE--
 <?php
-$filename = __DIR__ . '/_tmp.wbmp';
+$filename = __DIR__ . '/_tmp_createfromwbmp2.wbmp';
 $fp = fopen($filename,"wb");
 if (!$fp) {
     exit("Failed to create <$filename>");

ext/gd/tests/createfromwbmp2_extern.phpt

@@ -4,7 +4,7 @@ imagecreatefromwbmp with invalid wbmp
 gd
 --FILE--
 <?php
-$filename = __DIR__ . '/_tmp.wbmp';
+$filename = __DIR__ . '/_tmp_createfromwbmp2_extern.wbmp';
 $fp = fopen($filename,"wb");
 if (!$fp) {
     exit("Failed to create <$filename>");
@@ -41,4 +41,4 @@ unlink($filename);
 --EXPECTF--
 Warning: imagecreatefromwbmp(): %croduct of memory allocation multiplication would exceed INT_MAX, failing operation gracefully%win %s on line %d
 
-Warning: imagecreatefromwbmp(): "%s_tmp.wbmp" is not a valid WBMP file in %s on line %d
+Warning: imagecreatefromwbmp(): "%s_tmp_createfromwbmp2_extern.wbmp" is not a valid WBMP file in %s on line %d

ext/opcache/tests/issue0140.phpt

@@ -16,16 +16,16 @@ file_put_contents(FILENAME, "1\n");
 
 var_dump(is_readable(FILENAME));
 include(FILENAME);
-var_dump(filemtime(FILENAME));
+$mtime = filemtime(FILENAME);
+var_dump($mtime);
 
-sleep(2);
 file_put_contents(FILENAME, "2\n");
+touch(FILENAME, $mtime + 2);
 
 var_dump(is_readable(FILENAME));
 include(FILENAME);
 var_dump(filemtime(FILENAME));
 
-sleep(2);
 unlink(FILENAME);
 
 var_dump(is_readable(FILENAME));

ext/openssl/tests/MAX_CONCURRENCY

@@ -0,0 +1 @@
+10

ext/openssl/tests/ServerClientTestCase.inc

@@ -24,7 +24,7 @@ function phpt_has_sslv3() {
     if (!is_null($result)) {
         return $result;
     }
-    $server = @stream_socket_server('sslv3://127.0.0.1:10013');
+    $server = @stream_socket_server('sslv3://127.0.0.1:0');
     if ($result = !!$server) {
         fclose($server);
     }

ext/openssl/tests/gh20802.phpt

@@ -26,7 +26,7 @@ $serverCode = <<<'CODE'
 		    ]
 	    ]
     ]);
-    $server = stream_socket_server('tls://127.0.0.1:12443', $errno, $errstr, $flags, $ctx);
+    $server = stream_socket_server('tls://127.0.0.1:0', $errno, $errstr, $flags, $ctx);
     phpt_notify_server_start($server);
     stream_socket_accept($server, 3);
 CODE;
@@ -42,7 +42,7 @@ $ctx = stream_context_create([
 	    'verify_peer'  => false
 	]
     ]);
-    @stream_socket_client("tls://127.0.0.1:12443", $errno, $errstr, 1, $flags, $ctx);
+    @stream_socket_client("tls://{{ ADDR }}", $errno, $errstr, 1, $flags, $ctx);
 CODE;
 
 include 'CertificateGenerator.inc';

ext/openssl/tests/openssl_cms_verify_der.phpt

@@ -14,7 +14,7 @@ if ($contentfile === false) {
     die("failed to get a temporary filename!");
 }
 
-$pkcsfile = __DIR__ . "/openssl_cms_verify__pkcsfile.tmp";
+$pkcsfile = __DIR__ . "/openssl_cms_verify_der__pkcsfile.tmp";
 $eml = __DIR__ . "/signed.eml";
 $wrong = "wrong";
 $empty = "";
@@ -45,7 +45,7 @@ if (file_exists($contentfile)) {
 ?>
 --CLEAN--
 <?php
-unlink(__DIR__ . DIRECTORY_SEPARATOR . '/openssl_cms_verify__pkcsfile.tmp');
+unlink(__DIR__ . DIRECTORY_SEPARATOR . '/openssl_cms_verify_der__pkcsfile.tmp');
 ?>
 --EXPECT--
 bool(false)

ext/openssl/tests/openssl_pkcs12_export_to_file_error.phpt

@@ -4,7 +4,7 @@ openssl_pkcs12_export_to_file() error tests
 openssl
 --FILE--
 <?php
-$pkcsfile = __DIR__ . "/openssl_pkcs12_export_to_file__pkcsfile.tmp";
+$pkcsfile = __DIR__ . "/openssl_pkcs12_export_to_file_error.tmp";
 
 $cert_file = __DIR__ . "/public.crt";
 $cert = file_get_contents($cert_file);
@@ -21,7 +21,7 @@ var_dump(openssl_pkcs12_export_to_file($cert, '.', $priv, $pass));
 ?>
 --CLEAN--
 <?php
-$pkcsfile = __DIR__ . "/openssl_pkcs12_export_to_file__pkcsfile.tmp";
+$pkcsfile = __DIR__ . "/openssl_pkcs12_export_to_file_error.tmp";
 if (file_exists($pkcsfile)) {
     unlink($pkcsfile);
 }