From b04019aca9b2174808c9dedbc513caff84d1871d Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 29 Aug 2026 14:14:15 +0530 Subject: [PATCH] Parallelize release system tests via per-tutorial matrix (#789) Add prepare-matrix job that reads release from tests.yaml and runs one GitHub Actions job per tutorial with per-case suite names. --- .github/workflows/system-tests-manual.yml | 26 ++- .github/workflows/system-tests-nightly.yml | 4 +- .../workflows/system-tests-release-matrix.yml | 61 +++++++ changelog-entries/789.md | 1 + tests/README.md | 7 +- tests/parallel_matrix.py | 152 ++++++++++++++++++ tests/prepare_matrix.py | 68 ++++++++ 7 files changed, 312 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/system-tests-release-matrix.yml create mode 100644 changelog-entries/789.md create mode 100644 tests/parallel_matrix.py create mode 100644 tests/prepare_matrix.py diff --git a/.github/workflows/system-tests-manual.yml b/.github/workflows/system-tests-manual.yml index 2d4fee4a3..52d37fa76 100644 --- a/.github/workflows/system-tests-manual.yml +++ b/.github/workflows/system-tests-manual.yml @@ -53,11 +53,33 @@ on: type: string jobs: - run-system-tests: + run-system-tests-parallel: name: Trigger system tests + if: ${{ inputs.suites == 'release' }} + uses: ./.github/workflows/system-tests-release-matrix.yml + with: + build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ + PRECICE_REF:${{ inputs.ref-precice }},\ + PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ + CALCULIX_ADAPTER_REF:${{ inputs.ref-calculix-adapter }},\ + DEALII_ADAPTER_REF:${{ inputs.ref-dealii-adapter }},\ + DUMUX_ADAPTER_REF:${{ inputs.ref-dumux-adapter }},\ + DUNE_ADAPTER_REF:${{ inputs.ref-dune-adapter }},\ + FENICS_ADAPTER_REF:${{ inputs.ref-fenics-adapter }},\ + FENICSX_ADAPTER_REF:${{ inputs.ref-fenicsx-adapter }},\ + FMI_RUNNER_REF:${{ inputs.ref-fmi-runner }},\ + MICRO_MANAGER_REF:${{ inputs.ref-micro-manager }},\ + OPENFOAM_ADAPTER_REF:${{ inputs.ref-openfoam-adapter }},\ + SU2_ADAPTER_REF:${{ inputs.ref-su2-adapter }}" + system_tests_branch: ${{ inputs.system_tests_branch || 'develop' }} + log_level: "INFO" + + run-system-tests-custom: + name: Trigger system tests + if: ${{ inputs.suites != 'release' }} uses: ./.github/workflows/system-tests.yml with: - suites: ${{ inputs.suites || 'release' }} + suites: ${{ inputs.suites }} build_args: "TUTORIALS_REF:${{ inputs.ref-tutorials }},\ PRECICE_REF:${{ inputs.ref-precice }},\ PYTHON_BINDINGS_REF:${{ inputs.ref-python-bindings }},\ diff --git a/.github/workflows/system-tests-nightly.yml b/.github/workflows/system-tests-nightly.yml index 2f1e973e2..133e42070 100644 --- a/.github/workflows/system-tests-nightly.yml +++ b/.github/workflows/system-tests-nightly.yml @@ -8,6 +8,4 @@ on: jobs: run-system-tests: name: Trigger system tests - uses: ./.github/workflows/system-tests.yml - with: - suites: 'release' + uses: ./.github/workflows/system-tests-release-matrix.yml diff --git a/.github/workflows/system-tests-release-matrix.yml b/.github/workflows/system-tests-release-matrix.yml new file mode 100644 index 000000000..569754b9e --- /dev/null +++ b/.github/workflows/system-tests-release-matrix.yml @@ -0,0 +1,61 @@ +name: System tests (release matrix) + +on: + workflow_call: + inputs: + build_args: + description: 'Build arguments' + type: string + default: '' + system_tests_branch: + description: 'Branch to take the system tests from' + default: 'develop' + type: string + log_level: + description: 'Logging verbosity level used for the systemtests' + default: 'INFO' + type: string + upload_artifacts: + description: 'TRUE or FALSE (FALSE -> Upload only when the workflow fails)' + default: 'TRUE' + type: string + max_parallel: + description: 'Maximum number of parallel matrix jobs' + default: '4' + type: string + +jobs: + prepare-matrix: + name: Prepare release matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.matrix.outputs.matrix }} + steps: + - name: Check out tutorials + uses: actions/checkout@v7 + with: + ref: ${{ inputs.system_tests_branch }} + - name: Install Python dependencies + run: pip install --user -r tests/requirements.txt + - name: Build matrix from release suite + id: matrix + working-directory: tests + run: | + python3 prepare_matrix.py --source-suite release --format=github > matrix.json + echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT" + + run-system-tests: + name: Trigger system tests (${{ matrix.tutorial }}) + needs: prepare-matrix + strategy: + fail-fast: false + max-parallel: ${{ fromJSON(inputs.max_parallel) }} + matrix: + include: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + uses: ./.github/workflows/system-tests.yml + with: + suites: ${{ matrix.suites }} + build_args: ${{ inputs.build_args }} + system_tests_branch: ${{ inputs.system_tests_branch }} + log_level: ${{ inputs.log_level }} + upload_artifacts: ${{ inputs.upload_artifacts }} diff --git a/changelog-entries/789.md b/changelog-entries/789.md new file mode 100644 index 000000000..17749a997 --- /dev/null +++ b/changelog-entries/789.md @@ -0,0 +1 @@ +- Parallelize scheduled and manual `release` system tests via a per-tutorial GitHub Actions matrix derived from `tests.yaml` [#789](https://github.com/precice/tutorials/issues/789) diff --git a/tests/README.md b/tests/README.md index 0b932319a..c1986098b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -32,7 +32,9 @@ The available test suites are found in [`tests.yaml`](https://github.com/precice The `Use workflow from` is a default option of GitHub Actions that concerns the GHA workflow file itself. -The [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) executes the `release` test suite every night. The [System tests (weekly)](https://github.com/precice/tutorials/actions/workflows/system-tests-weekly.yml) executes the `extra` test suite once per week. +The [System tests (nightly)](https://github.com/precice/tutorials/actions/workflows/system-tests-nightly.yml) executes the `release` test suite every night as a GitHub Actions matrix with one job per tutorial. The [System tests (weekly)](https://github.com/precice/tutorials/actions/workflows/system-tests-weekly.yml) executes the `extra` test suite once per week. + +When the manual workflow runs `release`, it uses the same per-tutorial matrix. Other suite selections still run as a single job. If a matrix job fails, re-run only that tutorial job from the Actions run page. ### Running from a pull request @@ -77,10 +79,11 @@ This will build and connect Docker containers and run tutorials in the `runs/` d To clean up at the end, you might want to run a `docker system prune -a` and remove the `runs/` directory to save space. -There are also some auxiliary scripts (run without arguments): +There are also some auxiliary scripts: - `print_test_suites.py`: Print all test suites defined in `tests.yaml` - `print_case_combinations.py`: Print all possible combinations of participants in a tutorial, using its `metadata.yaml`. +- `prepare_matrix.py`: Build the per-tutorial matrix used for parallel `release` runs. ## Understanding the logs diff --git a/tests/parallel_matrix.py b/tests/parallel_matrix.py new file mode 100644 index 000000000..81abf90ee --- /dev/null +++ b/tests/parallel_matrix.py @@ -0,0 +1,152 @@ +"""Build a per-tutorial parallel matrix from a meta test suite in tests.yaml.""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List, Sequence, Tuple + +import yaml + +from paths import PRECICE_TESTS_DIR + +CaseKey = Tuple[str, Tuple[str, ...]] + + +@dataclass(frozen=True) +class MatrixJob: + tutorial: str + suites: str + case_count: int + + +def _tests_yaml_path(tests_yaml: Path | None = None) -> Path: + return tests_yaml if tests_yaml is not None else PRECICE_TESTS_DIR / "tests.yaml" + + +def load_test_suites(tests_yaml: Path | None = None) -> dict: + with open(_tests_yaml_path(tests_yaml), "r", encoding="utf-8") as handle: + data = yaml.safe_load(handle) + return data["test_suites"] + + +def build_case_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: + """Map (path, case_combination) to the runnable per-case suite name.""" + candidates: Dict[CaseKey, set[str]] = defaultdict(set) + for suite_name, suite_def in test_suites.items(): + tutorials = suite_def.get("tutorials", []) + if len(tutorials) != 1: + continue + entry = tutorials[0] + key = (entry["path"], tuple(entry["case_combination"])) + candidates[key].add(suite_name) + + lookup: Dict[CaseKey, str] = {} + for key, suite_names in candidates.items(): + path, combo = key + canonical = f"{path}_{'_'.join(combo)}" + if canonical in suite_names: + lookup[key] = canonical + else: + lookup[key] = sorted(suite_names)[0] + return lookup + + +def build_external_suite_lookup(test_suites: dict) -> Dict[CaseKey, str]: + """Map external cases to the suite that defines them (e.g. mixedbc).""" + lookup: Dict[CaseKey, str] = {} + for suite_name, suite_def in test_suites.items(): + external = suite_def.get("external", []) + if len(external) != 1: + continue + entry = external[0] + lookup[(entry["path"], tuple(entry["case_combination"]))] = suite_name + return lookup + + +def _release_case_entries( + source_suite: dict, + case_lookup: Dict[CaseKey, str], + external_lookup: Dict[CaseKey, str], +) -> List[Tuple[str, str]]: + """Return (tutorial_path, case_suite_name) pairs for a meta suite definition.""" + entries: List[Tuple[str, str]] = [] + + def add_entry(case_def: dict) -> None: + case_key: CaseKey = (case_def["path"], tuple(case_def["case_combination"])) + tutorial = case_def["path"] + if case_key in case_lookup: + suite_name = case_lookup[case_key] + elif case_key in external_lookup: + suite_name = external_lookup[case_key] + else: + raise KeyError( + f"No runnable suite found for {tutorial} " + f"{case_def['case_combination']}") + entries.append((tutorial, suite_name)) + + for case_def in source_suite.get("tutorials", []): + add_entry(case_def) + for case_def in source_suite.get("external", []): + add_entry(case_def) + + return entries + + +def build_parallel_matrix( + source_suite_name: str = "release", + tests_yaml: Path | None = None, +) -> List[MatrixJob]: + test_suites = load_test_suites(tests_yaml) + if source_suite_name not in test_suites: + raise KeyError(f"Unknown test suite: {source_suite_name}") + + case_lookup = build_case_suite_lookup(test_suites) + external_lookup = build_external_suite_lookup(test_suites) + release_entries = _release_case_entries( + test_suites[source_suite_name], + case_lookup, + external_lookup, + ) + + grouped: Dict[str, List[str]] = defaultdict(list) + for tutorial, case_suite in release_entries: + grouped[tutorial].append(case_suite) + + matrix_jobs: List[MatrixJob] = [] + for tutorial in sorted(grouped): + case_suites = grouped[tutorial] + matrix_jobs.append( + MatrixJob( + tutorial=tutorial, + suites=",".join(case_suites), + case_count=len(case_suites), + ) + ) + return matrix_jobs + + +def matrix_jobs_for_github( + source_suite_name: str = "release", + tests_yaml: Path | None = None, +) -> List[dict]: + return [ + {"tutorial": job.tutorial, "suites": job.suites} + for job in build_parallel_matrix(source_suite_name, tests_yaml) + ] + + +def release_case_suite_names( + source_suite_name: str = "release", + tests_yaml: Path | None = None, +) -> List[str]: + test_suites = load_test_suites(tests_yaml) + case_lookup = build_case_suite_lookup(test_suites) + external_lookup = build_external_suite_lookup(test_suites) + entries = _release_case_entries( + test_suites[source_suite_name], + case_lookup, + external_lookup, + ) + return sorted(suite for _, suite in entries) diff --git a/tests/prepare_matrix.py b/tests/prepare_matrix.py new file mode 100644 index 000000000..f6c273777 --- /dev/null +++ b/tests/prepare_matrix.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Emit a GitHub Actions matrix for parallel release system tests.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys + +from parallel_matrix import build_parallel_matrix, matrix_jobs_for_github + + +def _write_github_step_summary(matrix_jobs) -> None: + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + + total_cases = sum(job.case_count for job in matrix_jobs) + with open(summary_path, "w", encoding="utf-8") as summary_file: + print("## Parallel test matrix\n", file=summary_file) + print(f"- Source suite: `release`", file=summary_file) + print(f"- Tutorial jobs: **{len(matrix_jobs)}**", file=summary_file) + print(f"- Total cases: **{total_cases}**\n", file=summary_file) + print("| Tutorial | Cases |", file=summary_file) + print("| --- | ---: |", file=summary_file) + for job in matrix_jobs: + print(f"| `{job.tutorial}` | {job.case_count} |", file=summary_file) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build a per-tutorial matrix from a meta test suite.") + parser.add_argument( + "--source-suite", + default="release", + help="Meta suite to expand (default: release).", + ) + parser.add_argument( + "--format", + choices=["github", "pretty"], + default="pretty", + help="Output format (github: JSON for fromJson()).", + ) + args = parser.parse_args() + + try: + matrix_jobs = build_parallel_matrix(args.source_suite) + except (KeyError, FileNotFoundError) as error: + print(error, file=sys.stderr) + return 1 + + _write_github_step_summary(matrix_jobs) + + if args.format == "github": + print(json.dumps(matrix_jobs_for_github(args.source_suite))) + else: + total_cases = sum(job.case_count for job in matrix_jobs) + print( + f"Matrix for {args.source_suite}: " + f"{len(matrix_jobs)} tutorial jobs, {total_cases} cases") + for job in matrix_jobs: + print(f" {job.tutorial}: {job.case_count} case(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())