ci: scope sample and test execution

.circleci/config.yml

@@ -1,30 +1,19 @@
 version: 2.1
+parameters:
+  full_run:
+    type: boolean
+    default: false
+  selection_mode:
+    type: string
+    default: shadow
 commands: # a reusable command with parameters
   command_build_and_test:
     parameters:
       nodeNo:
         default: "0"
         type: string
     steps:
-      # Restore the dependency cache
-      - restore_cache:
-          keys:
-            # Default branch if not
-            - source-v2-{{ .Branch }}-{{ .Revision }}
-            - source-v2-{{ .Branch }}-
-            - source-v2-
-      # Machine Setup
-      #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
-      - run:
-          name: Install Headless Chrome dependencies
-          command: |
-            sudo apt-get update && sudo apt-get install -yq \
-            gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 \
-            libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \
-            libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 \
-            libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates \
-            fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libgbm1
-      # Avoid intermittent SSH checkout failures from CircleCI workers.
+      # Pin the event revision: a PR head ref may have advanced since scheduling.
       - run:
           name: Checkout source over HTTPS with retry
           command: |
@@ -35,15 +24,47 @@ commands: # a reusable command with parameters
             else
               ref="$CIRCLE_SHA1"
             fi
+            [[ "$CIRCLE_SHA1" =~ ^[0-9a-fA-F]{40}$ ]] || exit 1
             git init .
             git remote add origin https://github.com/OpenAPITools/openapi-generator.git
             for i in $(seq 1 5); do
-              if git fetch --depth=1 origin "$ref" && git checkout --detach FETCH_HEAD; then
-                exit 0
+              if git fetch --no-tags --depth=1 origin "$CIRCLE_SHA1" ||
+                 git fetch --no-tags --depth=1 origin "$ref"; then
+                if [ "$(git rev-parse FETCH_HEAD)" = "$CIRCLE_SHA1" ] &&
+                   git checkout --detach "$CIRCLE_SHA1"; then
+                  exit 0
+                fi
               fi
               sleep 5
             done
+            echo "Unable to check out the exact CircleCI event revision" >&2
             exit 1
+      - run:
+          name: Select relevant CircleCI scope before setup
+          environment:
+            CIRCLE_NODE_INDEX: "<<parameters.nodeNo>>"
+            CI_FORCE_FULL: "<<pipeline.parameters.full_run>>"
+            CI_SELECTION_MODE: "<<pipeline.parameters.selection_mode>>"
+          command: bash CI/circle_select.sh
+      # Restore the dependency cache
+      - restore_cache:
+          keys:
+            # Do not restore legacy source-v2 caches containing .git over the
+            # already verified checkout. Future caches must be dependency-only.
+            - dependencies-v3-{{ .Branch }}-{{ .Revision }}
+            - dependencies-v3-{{ .Branch }}-
+            - dependencies-v3-
+      # Machine Setup
+      #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+      - run:
+          name: Install Headless Chrome dependencies
+          command: |
+            sudo apt-get update && sudo apt-get install -yq \
+            gconf-service libasound2 libatk1.0-0 libatk-bridge2.0-0 libc6 libcairo2 libcups2 libdbus-1-3 \
+            libexpat1 libfontconfig1 libgcc1 libgconf-2-4 libgdk-pixbuf2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 \
+            libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 \
+            libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 libxrender1 libxss1 libxtst6 ca-certificates \
+            fonts-liberation libappindicator1 libnss3 lsb-release xdg-utils wget libgbm1
       # Prepare for artifact and test results  collection equivalent to how it was done on 1.0.
       # In many cases you can simplify this from what is generated here.
       # 'See docs on artifact collection here https://circleci.com/docs/2.0/artifacts/'

.github/.test/ci-scopes.js

@@ -0,0 +1,140 @@
+'use strict';
+
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const {execFileSync} = require('child_process');
+const yaml = require('./js-yaml.js');
+
+const root = path.resolve(__dirname, '..', '..');
+const manifest = JSON.parse(fs.readFileSync(path.join(root, 'CI', 'change-scopes.json'), 'utf8'));
+assert.strictEqual(manifest.version, 1, 'Unsupported CI scope manifest version');
+assert(manifest.suites && typeof manifest.suites === 'object', 'Missing CI suites');
+
+function matches(file, pattern) {
+    pattern = pattern.replace(/\/+$/, '');
+    if (!/[?*]/.test(pattern)) return file === pattern || file.startsWith(`${pattern}/`);
+    const expression = pattern.split(/(\*\*|\*|\?)/).map(part => {
+        if (part === '**') return '.*';
+        if (part === '*') return '[^/]*';
+        if (part === '?') return '[^/]';
+        return part.replace(/[\\^$+.()|[\]{}]/g, '\\$&');
+    }).join('');
+    return new RegExp(`^${expression}$`).test(file) ||
+        (pattern.endsWith('/**') && file === pattern.slice(0, -3));
+}
+
+function sampleRoots(suite) {
+    return [...new Set([
+        ...(suite.matrix?.sample || []),
+        ...(suite.matrix?.include || []).filter(row => row.sample).map(row => row.sample)
+    ])].map(dir => dir.replace(/\/+$/, ''));
+}
+
+const tracked = execFileSync('git', ['ls-files', '-z', '--', 'samples'], {
+    cwd: root, encoding: 'utf8', maxBuffer: 32 * 1024 * 1024
+}).split('\0').filter(Boolean);
+const trackedSet = new Set(tracked);
+const typescriptRoots = tracked.filter(file => file.endsWith('/tsconfig.json'))
+    .map(file => path.posix.dirname(file))
+    .filter(dir => trackedSet.has(`${dir}/.openapi-generator-ignore`) && trackedSet.has(`${dir}/package.json`));
+const owned = Object.entries(manifest.suites).filter(([id]) => id.startsWith('samples-') ||
+    id.startsWith('gradle-test.') || id.startsWith('circle.'));
+
+function owners(dir) {
+    return owned.filter(([, suite]) => {
+        const roots = sampleRoots(suite);
+        if (suite.discovery === 'typescript') roots.push(...typescriptRoots);
+        return roots.some(candidate => matches(dir, candidate)) ||
+            (suite.paths || []).some(pattern => matches(dir, pattern));
+    }).map(([id]) => id);
+}
+
+// Config outputs and tracked generator markers reveal newly added samples even
+// when no workflow path filter or matrix has been updated.
+const candidates = new Set(tracked.filter(file => file.endsWith('/.openapi-generator-ignore'))
+    .map(file => path.posix.dirname(file)));
+const buildFiles = new Set(['pom.xml', 'package.json', 'pyproject.toml', 'Cargo.toml',
+    'pubspec.yaml', 'go.mod', 'mix.exs', 'build.gradle', 'build.gradle.kts', 'Project.toml']);
+for (const file of tracked) {
+    const name = path.posix.basename(file);
+    const directory = path.posix.dirname(file);
+    if (directory !== 'samples' && (buildFiles.has(name) || name.endsWith('.csproj'))) candidates.add(directory);
+}
+for (const file of fs.readdirSync(path.join(root, 'bin', 'configs')).filter(file => file.endsWith('.yaml'))) {
+    const text = fs.readFileSync(path.join(root, 'bin', 'configs', file), 'utf8');
+    const output = text.match(/^outputDir:\s*(.+)$/m);
+    if (output) {
+        const dir = yaml.safeLoad(output[1]);
+        if (typeof dir === 'string' && dir.startsWith('samples/')) candidates.add(dir.replace(/\/+$/, ''));
+    }
+}
+const unowned = [...candidates].filter(dir => owners(dir).length === 0).sort();
+if (process.argv.includes('--list-unowned')) {
+    process.stdout.write(JSON.stringify(unowned, null, 2) + '\n');
+    process.exit(0);
+}
+
+const exclusions = new Map((manifest.exclusions || []).map(entry => {
+    assert(typeof entry.path === 'string' && entry.path.startsWith('samples/') &&
+        !/[?*]/.test(entry.path), 'Exclusions must name exact sample roots');
+    assert(typeof entry.reason === 'string' && entry.reason.trim(), `Missing exclusion reason: ${entry.path}`);
+    return [entry.path, entry.reason];
+}));
+for (const dir of unowned) {
+    assert(exclusions.has(dir), `Unowned sample ${dir}: register a CI suite or document an explicit exclusion`);
+}
+
+const workflowDir = path.join(root, '.github', 'workflows');
+const workflows = new Map();
+for (const file of fs.readdirSync(workflowDir).filter(file => /\.ya?ml$/.test(file))) {
+    workflows.set(`.github/workflows/${file}`, yaml.safeLoad(fs.readFileSync(path.join(workflowDir, file), 'utf8')));
+}
+for (const [id, suite] of Object.entries(manifest.suites)) {
+    for (const dir of sampleRoots(suite)) {
+        assert(!dir.includes('..') && !path.posix.isAbsolute(dir), `Invalid sample root in ${id}: ${dir}`);
+        assert(fs.statSync(path.join(root, ...dir.split('/'))).isDirectory(), `Missing sample target ${id}: ${dir}`);
+        const packageFile = path.join(root, ...dir.split('/'), 'package.json');
+        if (fs.existsSync(packageFile)) {
+            const pkg = JSON.parse(fs.readFileSync(packageFile, 'utf8'));
+            for (const reference of Object.values({...pkg.dependencies, ...pkg.devDependencies})) {
+                if (typeof reference !== 'string' || !reference.startsWith('file:')) continue;
+                const dependency = path.posix.normalize(path.posix.join(dir, reference.slice(5)));
+                if (matches(dependency, dir)) continue;
+                assert([...(suite.paths || []), ...(suite.shared_paths || [])].some(p => matches(dependency, p)),
+                    `${id}: declare external local dependency ${dependency} as a shared input`);
+            }
+        }
+    }
+    if (!suite.workflow) continue;
+    if (suite.workflow === '.circleci/config.yml') continue;
+    const workflow = workflows.get(suite.workflow);
+    assert(workflow, `Missing workflow for ${id}: ${suite.workflow}`);
+    const calls = Object.values(workflow.jobs).flatMap(job => job.steps || [])
+        .filter(step => step.uses === './.github/actions/compute-matrix' && step.with?.suite === id);
+    assert.strictEqual(calls.length, 1, `Suite ${id} must have exactly one workflow selector`);
+}
+let sampleJobs = 0;
+for (const [filename, workflow] of workflows) {
+    if (!/\/samples-[^/]+\.ya?ml$/.test(filename) && !filename.endsWith('/gradle-test.yaml')) continue;
+    assert(workflow.on.workflow_dispatch !== undefined, `${filename}: missing full-run dispatch`);
+    assert(!workflow.on.pull_request?.paths, `${filename}: PR path filters can hide relevant changes`);
+    assert(!workflow.on.push?.paths, `${filename}: safety pushes must run full scope`);
+    assert.deepStrictEqual(workflow.on.push.branches, ['master', '[5-9]+.[0-9]+.x'],
+        `${filename}: unexpected safety branches`);
+    for (const [jobId, job] of Object.entries(workflow.jobs)) {
+        if (jobId === 'setup') continue;
+        sampleJobs++;
+        const id = `${path.posix.basename(filename).replace(/\.ya?ml$/, '')}.${jobId}`;
+        const suite = manifest.suites[id];
+        assert(suite, `Missing scope for ${filename}/${jobId}`);
+        assert.strictEqual(job.needs, 'setup', `${id}: selection must precede runner allocation`);
+        assert(job.if?.includes("_changed == 'true'"), `${id}: missing explicit empty-selection guard`);
+        if (suite.matrix) {
+            assert.strictEqual(typeof job.strategy.matrix, 'string', `${id}: duplicate static matrix`);
+            assert(job.strategy.matrix.includes('fromJSON(needs.setup.outputs.'), `${id}: disconnected matrix`);
+        }
+    }
+}
+console.log(`CI scope coverage: ${sampleJobs} sample jobs, ${Object.keys(manifest.suites).length} suites, ` +
+    `${candidates.size} sample roots (${unowned.length} documented exclusions).`);

.github/actions/compute-matrix/action.yaml

@@ -0,0 +1,49 @@
+name: Select changed CI scope
+description: Compute a conservative PR matrix using the shared ownership policy.
+inputs:
+  suite:
+    description: Suite ID in CI/change-scopes.json
+    required: true
+  mode:
+    description: enforce filters matrices; shadow preserves historical workflow gating and executes full matrices
+    default: shadow
+  force-full:
+    description: Execute the original complete matrix
+    default: 'false'
+outputs:
+  matrix:
+    description: JSON matrix object
+    value: ${{ steps.select.outputs.matrix }}
+  samples:
+    description: JSON array of selected sample paths
+    value: ${{ steps.select.outputs.samples }}
+  has_changes:
+    description: Whether the suite should execute
+    value: ${{ steps.select.outputs.has_changes }}
+  run_all:
+    description: Whether execution uses the full suite
+    value: ${{ steps.select.outputs.run_all }}
+  reasons:
+    description: JSON array of decision reasons
+    value: ${{ steps.select.outputs.reasons }}
+  proposed_selection:
+    description: Proposed filtered decision in shadow mode
+    value: ${{ steps.select.outputs.proposed_selection }}
+runs:
+  using: composite
+  steps:
+    - id: select
+      shell: bash
+      env:
+        CI_SUITE: ${{ inputs.suite }}
+        CI_SELECTION_MODE: ${{ inputs.mode }}
+        CI_ACTION_FORCE_FULL: ${{ inputs.force-full }}
+      run: |
+        args=()
+        if [[ "$CI_ACTION_FORCE_FULL" == "true" ]]; then
+          args+=(--force-full)
+        elif [[ "$CI_ACTION_FORCE_FULL" != "false" ]]; then
+          echo "force-full must be true or false" >&2
+          exit 2
+        fi
+        python3 CI/select_changes.py --provider github --suite "$CI_SUITE" --mode "$CI_SELECTION_MODE" "${args[@]}"

.github/workflows/docker.yaml

@@ -3,26 +3,33 @@ name: Docker tests
 on:
   workflow_dispatch:
   push:
-    paths:
-      - Dockerfile
-      - run-in-docker.sh
-      - pom.xml
-      - modules/openapi-generator-online/pom.xml
-      - modules/openapi-generator-online/Dockerfile
-      - modules/openapi-generator-cli/pom.xml
-      - modules/openapi-generator-cli/Dockerfile
+    branches:
+      - master
+      - '[5-9]+.[0-9]+.x'
   pull_request:
-    paths:
-      - Dockerfile
-      - run-in-docker.sh
-      - pom.xml
-      - modules/openapi-generator-online/pom.xml
-      - modules/openapi-generator-online/Dockerfile
-      - modules/openapi-generator-cli/pom.xml
-      - modules/openapi-generator-cli/Dockerfile
+
 jobs:
+  setup:
+    name: Select changes
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      build_changed: ${{ steps.build.outputs.has_changes }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+      - uses: ./.github/actions/compute-matrix
+        id: build
+        with:
+          suite: docker.build
+          mode: ${{ vars.CI_SELECTION_MODE || 'shadow' }}
+
   build:
     name: 'Build: Docker'
+    needs: setup
+    if: needs.setup.outputs.build_changed == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Check out code

.github/workflows/gradle-plugin-tests.yaml

@@ -1,16 +1,35 @@
 name: Gradle plugin tests
 
 on:
+  workflow_dispatch:
   push:
-    paths:
-      - modules/openapi-generator-gradle-plugin/**
+    branches:
+      - master
+      - '[5-9]+.[0-9]+.x'
   pull_request:
-    paths:
-      - modules/openapi-generator-gradle-plugin/**
 
 jobs:
+  setup:
+    name: Select changes
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    outputs:
+      test_changed: ${{ steps.test.outputs.has_changes }}
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 1
+      - uses: ./.github/actions/compute-matrix
+        id: test
+        with:
+          suite: gradle-plugin-tests.test
+          mode: ${{ vars.CI_SELECTION_MODE || 'shadow' }}
+
   test:
     name: Gradle plugin tests
+    needs: setup
+    if: needs.setup.outputs.test_changed == 'true'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7