diff --git a/.claude/rules/llm-docs-maintenance.md b/.claude/rules/llm-docs-maintenance.md index 5e5f533a6c8..0dd2ee8473e 100644 --- a/.claude/rules/llm-docs-maintenance.md +++ b/.claude/rules/llm-docs-maintenance.md @@ -63,3 +63,5 @@ So a relevant deep-dive actually gets read when a matching task comes up, each ` Granularity rule: the pointer inherits its host rule's `paths:`, so it must live in a rule scoped **no broader than the doc's topic** — otherwise it loads on unrelated work (a reveal.js pointer does not belong in a rule scoped to all of `src/format/**`). If the best existing rule is too generic, create a small dedicated rule scoped to that doc's source instead. **When adding a new `llm-docs/*.md`:** add its discovery pointer to a matching path-scoped rule, or create a narrowly-scoped rule for it. + +**When renaming an `##`/`###` heading:** search `.claude/rules/` for pointers that quote the old heading and update them. Otherwise, a reader following the pointer will not find the named section. diff --git a/.claude/rules/testing/built-version-ci.md b/.claude/rules/testing/built-version-ci.md new file mode 100644 index 00000000000..d12c832e269 --- /dev/null +++ b/.claude/rules/testing/built-version-ci.md @@ -0,0 +1,26 @@ +--- +paths: + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/test-ff-matrix.yml + - .github/workflows/create-release.yml + - .github/actions/build-dist-tarball/** + - .github/actions/merge-extension-tests/** +--- + +# Built-Version CI Workflows + +Use `llm-docs/built-version-testing-architecture.md` for detailed context. Read only the sections relevant to the edit: + +- OS lists, bucket globs, concurrency-group suffixes, `runners:` inputs → "Built-mode test legs" (scheduler layout). +- Adding/reordering a leg, changing `quarto-install` resolution or artifact plumbing → "Flow diagrams" + "Built-mode test legs". +- Removing, inverting, or simplifying an existing mechanism → "Design decisions" (D1-D11) first. + +For a one-line YAML edit, the invariants below and the document map are enough. + +Invariants when editing: + +- Playwright and ff-matrix legs in `test-smokes-built.yml` must check both the source mode and `github.event.inputs.buckets == ''`. +- `test-ff-matrix.yml` owns the ff-matrix bucket glob. +- Scheduler jobs in `test-smokes-built.yml` set per-leg OS scope through their `runners:` inputs. +- Keep the per-call suffix in `test-ff-matrix.yml`'s concurrency group so sibling calls cannot cancel one another. diff --git a/.claude/rules/testing/overview.md b/.claude/rules/testing/overview.md index bec82f921b2..ffd4a4ec1b7 100644 --- a/.claude/rules/testing/overview.md +++ b/.claude/rules/testing/overview.md @@ -1,11 +1,12 @@ --- paths: - - "tests/**" + - tests/** --- # Test Infrastructure -Quarto's test suite lives in `tests/`. For comprehensive documentation, see `tests/README.md`. +Quarto's test suite lives in `tests/`. +For comprehensive documentation, see `tests/README.md`. ## Running Tests @@ -28,18 +29,23 @@ QUARTO_TESTS_NO_CONFIG="true" ./run-tests.sh test.ts # Linux/macOS $env:QUARTO_TESTS_NO_CONFIG=$true; .\run-tests.ps1 # Windows ``` +**Binary mode:** set `QUARTO_TEST_BIN` to an installed Quarto outside the checkout. +With no test arguments, binary mode runs `smoke/`; pass Playwright or feature-format targets explicitly. +See `tests/README.md` and `llm-docs/built-version-testing-architecture.md`. + ## Test Types -| Type | Location | File Pattern | Details | -|------|----------|--------------|---------| -| Unit | `tests/unit/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | -| Smoke | `tests/smoke/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | -| Smoke-all | `tests/docs/smoke-all/` | `*.qmd` | `.claude/rules/testing/smoke-all-tests.md` | -| Playwright | `tests/integration/playwright/` | `*.spec.ts` | `.claude/rules/testing/playwright-tests.md` | +| Type | Location | File Pattern | Details | +| ---------- | ------------------------------- | ------------ | ------------------------------------------- | +| Unit | `tests/unit/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | +| Smoke | `tests/smoke/` | `*.test.ts` | `.claude/rules/testing/typescript-tests.md` | +| Smoke-all | `tests/docs/smoke-all/` | `*.qmd` | `.claude/rules/testing/smoke-all-tests.md` | +| Playwright | `tests/integration/playwright/` | `*.spec.ts` | `.claude/rules/testing/playwright-tests.md` | ## Dependencies -Tests require R, Python, and Julia. Run configuration script to set up: +Tests require R, Python, and Julia. +Run configuration script to set up: ```bash # Linux/macOS @@ -56,12 +62,13 @@ Managed via: ## Core Files -| File | Purpose | -|------|---------| -| `test.ts` | Test infrastructure (`testQuartoCmd`, `unitTest`) | -| `verify.ts` | Verification functions | -| `utils.ts` | Path utilities (`docs()`, `outputForInput()`) | -| `README.md` | Comprehensive documentation | +| File | Purpose | +| --------------- | ----------------------------------------------------------------------------------------- | +| `test.ts` | Test infrastructure (`testQuartoCmd`, `unitTest`) | +| `quarto-cmd.ts` | Quarto invocation dispatch (`runQuarto`; in-process dev vs `QUARTO_TEST_BIN` binary mode) | +| `verify.ts` | Verification functions | +| `utils.ts` | Path utilities (`docs()`, `outputForInput()`) | +| `README.md` | Comprehensive documentation | ## Debugging diff --git a/.claude/rules/testing/playwright-tests.md b/.claude/rules/testing/playwright-tests.md index 8cc3781a043..d74aab5e0c8 100644 --- a/.claude/rules/testing/playwright-tests.md +++ b/.claude/rules/testing/playwright-tests.md @@ -1,12 +1,13 @@ --- paths: - - "tests/integration/playwright/**/*.spec.ts" - - "tests/integration/playwright/**/*.ts" + - tests/integration/playwright/**/*.spec.ts + - tests/integration/playwright/**/*.ts --- # Playwright Tests -Browser-based tests for interactive features. Tests live in `tests/integration/playwright/tests/`. +Browser-based tests for interactive features. +Tests live in `tests/integration/playwright/tests/`. ## Local Development Workflow @@ -44,6 +45,10 @@ The wrapper (`playwright-tests.test.ts`): 3. Runs `npx playwright test` 4. Cleans up rendered output +The suite runs in dev and binary-mode CI. +Render spawns must use `quartoSpawnEnvOptions()` so a built Quarto cannot inherit dev-tree paths. +Browser assertions are ignored on Windows CI, so built-version CI has no Windows Playwright leg. + ## Test Structure Tests use `@playwright/test` framework: @@ -63,7 +68,8 @@ test("Feature description", async ({ page }) => { ### Parameterized Tests -When testing the same behavior across multiple formats or configurations, use `test.describe` with a test cases array instead of separate spec files. See `html-math-katex.spec.ts` and `axe-accessibility.spec.ts` for examples. +When testing the same behavior across multiple formats or configurations, use `test.describe` with a test cases array instead of separate spec files. +See `html-math-katex.spec.ts` and `axe-accessibility.spec.ts` for examples. ```typescript const testCases = [ @@ -81,11 +87,13 @@ test.describe('Feature across formats', () => { }); ``` -**When to use:** Same assertion logic applied to multiple formats, output modes, or configurations. Reduces file count and centralizes shared helpers. +**When to use:** Same assertion logic applied to multiple formats, output modes, or configurations. +Reduces file count and centralizes shared helpers. ### Expected Failures -Use `test.fail()` to document known failures. Playwright inverts the result: the test passes if it fails, and flags if it unexpectedly passes (signaling the fix landed). +Use `test.fail()` to document known failures. +Playwright inverts the result: the test passes if it fails, and flags if it unexpectedly passes (signaling the fix landed). ```typescript test('Feature that is known broken', async ({ page }) => { diff --git a/.claude/rules/testing/test-anti-patterns.md b/.claude/rules/testing/test-anti-patterns.md index e7523f9028e..152129beb4c 100644 --- a/.claude/rules/testing/test-anti-patterns.md +++ b/.claude/rules/testing/test-anti-patterns.md @@ -1,14 +1,25 @@ --- paths: - - "tests/**/*.ts" - - "tests/**/*.test.ts" + - tests/**/*.ts + - tests/**/*.test.ts --- # Test Anti-Patterns +## Don't: Rely on `execute()` Throwing to Fail a Test + +The harness converts errors from `TestDescriptor.execute` into log records. +Put assertions in `verify`, and include a log-reading verifier such as `noErrors` when using `runQuarto(..., { throwOnFailure: false })`. + +## Don't: Import `src/quarto.ts` in Tests + +Importing `src/quarto.ts` bypasses binary mode. +Use `testQuartoCmd()`/`runQuarto()` and resolve direct subprocess spawns with `quartoDevCmd()`. + ## Don't: Modify Environment Variables -`Deno.env.set()` modifies process-global state. Deno runs test files in parallel by default, so other tests can see modified values. +`Deno.env.set()` modifies process-global state. +Deno runs test files in parallel by default, so other tests can see modified values. **Details:** `llm-docs/testing-patterns.md` → "Environment Variable Testing Pitfalls" @@ -20,6 +31,9 @@ Never create `Project.toml`, `.venv/`, or `renv.lock` in test fixture directorie ## Don't: `Deno.chdir()` inside the test body -`Deno.chdir()` mutates process-global cwd, so a test that changes it can leak into other tests in the same process. The harness already changes and restores the working directory: return the directory from `TestContext.cwd`, create fixtures in `setup`, clean up in `teardown` (examples: `tests/unit/dotenv-config.test.ts`, `tests/smoke/use/template.test.ts`). For a temp directory you don't need to run *from*, use `withTempDir` (`tests/utils.ts`). A test that only needs a *relative* input can pass a path relative to the current cwd without changing it. +`Deno.chdir()` mutates process-global cwd, so a test that changes it can leak into other tests in the same process. +The harness already changes and restores the working directory: return the directory from `TestContext.cwd`, create fixtures in `setup`, clean up in `teardown` (examples: `tests/unit/dotenv-config.test.ts`, `tests/smoke/use/template.test.ts`). +For a temp directory you don't need to run *from*, use `withTempDir` (`tests/utils.ts`). +A test that only needs a *relative* input can pass a path relative to the current cwd without changing it. **Details:** `llm-docs/testing-patterns.md` → "Working-Directory-Sensitive Tests" diff --git a/.claude/rules/testing/typescript-tests.md b/.claude/rules/testing/typescript-tests.md index ea49883ba3d..c98baa9989f 100644 --- a/.claude/rules/testing/typescript-tests.md +++ b/.claude/rules/testing/typescript-tests.md @@ -1,12 +1,13 @@ --- paths: - - "tests/smoke/**/*.test.ts" - - "tests/unit/**/*.test.ts" + - tests/smoke/**/*.test.ts + - tests/unit/**/*.test.ts --- # TypeScript Tests -TypeScript-based tests using Deno. Smoke tests render documents; unit tests verify isolated functionality. +TypeScript-based tests using Deno. +Smoke tests render documents; unit tests verify isolated functionality. ## Running Tests @@ -22,17 +23,22 @@ TypeScript-based tests using Deno. Smoke tests render documents; unit tests veri ## Core Infrastructure -Core test files (`test.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. +Core test files (`test.ts`, `quarto-cmd.ts`, `verify.ts`, `utils.ts`) are described in `.claude/rules/testing/overview.md` § Core Files. + +### Binary mode compatibility + +Smoke tests must work with either the dev sources or `QUARTO_TEST_BIN`: + +- Do not import `quarto` from `src/quarto.ts`. Invoke Quarto through `testQuartoCmd()` or `runQuarto()`. +- For subprocesses, use `quartoDevCmd()` or `quartoDevBinCmd()` and pass `quartoSpawnEnvOptions()`. +- Set `TestContext.requiresDevQuarto: true` only for tests that require in-process Quarto internals. Binary mode ignores these tests. ### Search for an existing verifier before writing one -`verify.ts` already covers many output shapes — including parsed-content -verifiers, not just raw-text regex (e.g. `ensureIpynbCellMatches` JSON-parses a -notebook and matches against joined cell source; `ensureHtmlElements` / -`ensureHtmlSelectorSatisfies` parse the DOM). Before adding a new `Verify`, -grep `verify.ts` for the format or assertion you need. Reuse or extend the -existing helper rather than hand-rolling a near-duplicate. Same applies to -mock-context and fixture helpers in `tests/unit/**` and `tests/utils.ts`. +`verify.ts` already covers many output shapes — including parsed-content verifiers, not just raw-text regex (e.g. `ensureIpynbCellMatches` JSON-parses a notebook and matches against joined cell source; `ensureHtmlElements` / `ensureHtmlSelectorSatisfies` parse the DOM). +Before adding a new `Verify`, grep `verify.ts` for the format or assertion you need. +Reuse or extend the existing helper rather than hand-rolling a near-duplicate. +Same applies to mock-context and fixture helpers in `tests/unit/**` and `tests/utils.ts`. ## Smoke Tests (`tests/smoke/`) @@ -112,11 +118,14 @@ const markdownWithContent = asMappedString("# Title\nSome content"); **Mock Contexts:** -Several subsystems use context interfaces passed to functions. For unit tests, create `createMock*()` helpers with no-op stubs. Key pattern: async callbacks (like `withSpinner`) should just `await op()` so errors propagate normally. Check existing test files for helpers before writing new ones. - -| Context | Interface | Existing helpers | -|---------|-----------|-----------------| -| `ProjectContext` | `src/project/types.ts` | `tests/unit/project/utils.ts` → `createMockProjectContext()` | -| `InstallContext` | `src/tools/types.ts` | `tests/unit/tools/chrome-headless-shell.test.ts` → `createMockContext()` | -| `Format` | `src/config/types.ts` | `tests/unit/format-utils.ts` → `createMockFormat()` (wraps the real `createFormat()` from `src/format/formats-shared.ts` — no hand-rolled cast) | -| `TempContext` | `src/core/temp-types.ts` | No mock needed — use the real `createTempContext()` from `src/core/temp.ts` directly (see `tests/unit/sass-cache.test.ts`, `tests/unit/ral/safe-remove-dir.test.ts`) | +Several subsystems use context interfaces passed to functions. +For unit tests, create `createMock*()` helpers with no-op stubs. +Key pattern: async callbacks (like `withSpinner`) should just `await op()` so errors propagate normally. +Check existing test files for helpers before writing new ones. + +| Context | Interface | Existing helpers | +| ---------------- | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ProjectContext` | `src/project/types.ts` | `tests/unit/project/utils.ts` → `createMockProjectContext()` | +| `InstallContext` | `src/tools/types.ts` | `tests/unit/tools/chrome-headless-shell.test.ts` → `createMockContext()` | +| `Format` | `src/config/types.ts` | `tests/unit/format-utils.ts` → `createMockFormat()` (wraps the real `createFormat()` from `src/format/formats-shared.ts` — no hand-rolled cast) | +| `TempContext` | `src/core/temp-types.ts` | No mock needed — use the real `createTempContext()` from `src/core/temp.ts` directly (see `tests/unit/sass-cache.test.ts`, `tests/unit/ral/safe-remove-dir.test.ts`) | diff --git a/.github/actions/build-dist-tarball/action.yml b/.github/actions/build-dist-tarball/action.yml new file mode 100644 index 00000000000..de90c0b4778 --- /dev/null +++ b/.github/actions/build-dist-tarball/action.yml @@ -0,0 +1,57 @@ +name: "Build Quarto dist tarball" +description: | + Builds and uploads a versioned Quarto distribution tarball from the current + checkout. +inputs: + version: + description: "Version to stamp into the distribution (--set-version)" + required: true + arch: + description: "Target architecture: amd64 or arm64" + required: false + default: "amd64" + tarball-name: + description: "File name of the produced .tar.gz" + required: true + artifact-name: + description: "Workflow artifact name to upload the tarball as" + required: true + retention-days: + description: "Artifact retention in days; empty uses the repository default" + required: false + default: "" +runs: + using: "composite" + steps: + - name: Configure + shell: bash + run: ./configure.sh + + - name: Prepare Distribution + shell: bash + run: | + pushd package/src/ + if [ "${{ inputs.arch }}" = "arm64" ]; then + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --arch aarch64 --log-level info + else + ./quarto-bld prepare-dist --set-version ${{ inputs.version }} --log-level info + fi + popd + + - name: Make Tarball + shell: bash + run: | + pushd package/ + mv pkg-working quarto-${{ inputs.version }} + tar --owner=root --group=root -czf "${{ inputs.tarball-name }}" quarto-${{ inputs.version }} + mv quarto-${{ inputs.version }} pkg-working + popd + + - name: Upload Artifact + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.artifact-name }} + path: ./package/${{ inputs.tarball-name }} + retention-days: ${{ inputs.retention-days }} + # The tarball is already compressed. + compression-level: 0 diff --git a/.github/actions/merge-extension-tests/action.yml b/.github/actions/merge-extension-tests/action.yml index 2091b9c95f8..428b1f74e8f 100644 --- a/.github/actions/merge-extension-tests/action.yml +++ b/.github/actions/merge-extension-tests/action.yml @@ -11,6 +11,11 @@ runs: - name: Merge julia-engine tests shell: bash run: | + # The upstream tests do not yet sanitize Quarto subprocess environments. + if [[ -n "$QUARTO_TEST_BIN" ]]; then + echo "::notice title=julia-engine tests skipped::The subtree tests do not support binary mode yet" + exit 0 + fi SUBTREE=src/resources/extension-subtrees/julia-engine/tests cp -r "$SUBTREE/docs/julia-engine" tests/docs/julia-engine cp -r "$SUBTREE/smoke/julia-engine" tests/smoke/julia-engine diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 2e3e7451422..04ef2071e1b 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -24,13 +24,19 @@ on: - windows - macos default: all + smoke-artifacts-only: + description: "Build only the Linux amd64 tarball and signed Windows zip used by built-version tests. For unpublished branch builds; requires publish-release=false." + required: false + type: boolean + default: false env: NFPM_VERSION: "2.43.1" concurrency: # make publishing release concurrent (but others trigger not) - group: building-releases-${{ inputs.publish-release && 'prerelease' || github.run_id }} + # Partial test builds must not queue with releases. + group: building-releases-${{ inputs.publish-release && !inputs.smoke-artifacts-only && 'prerelease' || github.run_id }} jobs: configure: @@ -45,6 +51,13 @@ jobs: tag_pushed: ${{ steps.version_commit.outputs.tag_pushed }} if: github.event_name != 'schedule' || (github.event_name == 'schedule' && github.repository == 'quarto-dev/quarto-cli') steps: + # Prevent a partial build from reaching the version commit and tag. + - name: Fail on publish-release + smoke-artifacts-only + if: ${{ inputs.publish-release && inputs.smoke-artifacts-only }} + run: | + echo "::error::smoke-artifacts-only builds cannot publish - re-dispatch with publish-release=false" + exit 1 + - uses: actions/checkout@v6 with: fetch-depth: 0 @@ -116,6 +129,7 @@ jobs: default_author: github_actions make-source-tarball: + if: ${{ !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [configure] steps: @@ -150,35 +164,18 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-amd64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-amd64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-amd64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: amd64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-amd64.tar.gz + artifact-name: Deb Zip make-arm64-tarball: runs-on: ubuntu-latest needs: [configure] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux') }} steps: - uses: actions/checkout@v6 with: @@ -188,30 +185,13 @@ jobs: if: ${{ inputs.publish-release }} uses: ./.github/workflows/actions/prevent-rerun - - name: Configure - run: | - ./configure.sh - - - name: Prepare Distribution - run: | - pushd package/src/ - ./quarto-bld prepare-dist --set-version ${{needs.configure.outputs.version}} --arch aarch64 --log-level info - popd - - - name: Make Tarball - run: | - pushd package/ - mv pkg-working quarto-${{needs.configure.outputs.version}} - tar --owner=root --group=root -cvf quarto-${{needs.configure.outputs.version}}-linux-arm64.tar quarto-${{needs.configure.outputs.version}} - gzip quarto-${{needs.configure.outputs.version}}-linux-arm64.tar - mv quarto-${{needs.configure.outputs.version}} pkg-working - popd - - - name: Upload Artifact - uses: actions/upload-artifact@v7 + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball with: - name: Deb Arm64 Zip - path: ./package/quarto-${{needs.configure.outputs.version}}-linux-arm64.tar.gz + version: ${{ needs.configure.outputs.version }} + arch: arm64 + tarball-name: quarto-${{ needs.configure.outputs.version }}-linux-arm64.tar.gz + artifact-name: Deb Arm64 Zip make-tarball-rhel: runs-on: ubuntu-latest @@ -266,7 +246,7 @@ jobs: make-installer-linux: runs-on: ubuntu-latest needs: [configure] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'linux') }} strategy: matrix: arch: [x86_64, aarch64] @@ -396,9 +376,7 @@ jobs: cargo build --release --all-features --manifest-path package/launcher/Cargo.toml cp package/launcher/target/release/quarto.exe package/pkg-working/bin/quarto.exe - # Scheduled runs never publish, so they skip DigiCert signing. Gated on - # `schedule` and not `publish-release`: those dispatches are the pre-merge - # check for bundled-binary bumps and must keep signing. + # Scheduled builds skip signing; dispatched no-publish builds keep it. - name: Sign files before making ZIP and MSI installer if: ${{ github.event_name != 'schedule' }} id: sign-files @@ -505,7 +483,7 @@ jobs: make-installer-mac: runs-on: macos-latest needs: [configure] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos') }} steps: - uses: actions/checkout@v6 with: @@ -575,7 +553,7 @@ jobs: test-zip-mac: runs-on: macos-latest needs: [configure, make-installer-mac] - if: ${{ github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos' }} + if: ${{ !inputs.smoke-artifacts-only && (github.event_name != 'workflow_dispatch' || inputs.os == 'all' || inputs.os == 'macos') }} steps: - uses: actions/checkout@v6 with: @@ -613,7 +591,8 @@ jobs: quarto --version publish-release: - if: ${{ inputs.publish-release }} + # Never publish from a partial build. + if: ${{ inputs.publish-release && !inputs.smoke-artifacts-only }} runs-on: ubuntu-latest needs: [ configure, diff --git a/.github/workflows/test-ff-matrix.yml b/.github/workflows/test-ff-matrix.yml index a1195c4a59f..cfcdf66f422 100644 --- a/.github/workflows/test-ff-matrix.yml +++ b/.github/workflows/test-ff-matrix.yml @@ -1,5 +1,43 @@ name: Feature-Format Tests on: + # Reusable so built-version CI can invoke the same bucket definition. + workflow_call: + inputs: + extra-r-packages: + description: "Comma-separated R packages to pass to renv::install" + required: false + type: string + default: "" + quarto-install: + description: "Quarto source: dev tree, published release, or workflow artifact" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is release" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Artifact containing the Quarto tarball or zip" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Run ID containing the artifact; empty uses the current run" + required: false + type: string + default: "" + ref: + description: "Git ref to check out; empty uses the default" + required: false + type: string + default: "" + runners: + description: "JSON array of runner labels" + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: extra-r-packages: @@ -20,6 +58,7 @@ on: - ".github/workflows/performance-check.yml" - ".github/workflows/stale-needs-repro.yml" - ".github/workflows/test-bundle.yml" + - ".github/workflows/test-smokes-built.yml" - ".github/workflows/test-smokes-parallel.yml" - ".github/workflows/test-install.yml" - ".github/workflows/test-quarto-latexmk.yml" @@ -35,12 +74,15 @@ on: - ".github/workflows/stale-needs-repro.yml" - ".github/workflows/test-bundle.yml" - ".github/workflows/test-install.yml" + - ".github/workflows/test-smokes-built.yml" - ".github/workflows/test-smokes-parallel.yml" - ".github/workflows/test-quarto-latexmk.yml" - ".github/workflows/update-test-timing.yml" concurrency: - group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }} + # Reusable workflows evaluate concurrency in the caller's context, so each + # call needs a distinct suffix. + group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}-${{ inputs.runners && format('call-{0}-{1}', github.run_id, inputs.runners) || 'dev' }} cancel-in-progress: true jobs: @@ -48,5 +90,13 @@ jobs: name: Run feature-format matrix uses: ./.github/workflows/test-smokes.yml with: + # Keep the feature-format glob owned by this workflow. buckets: '[ "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" ]' extra-r-packages: ${{ inputs.extra-r-packages }} + # Non-call triggers have empty inputs and use dev defaults. + quarto-install: ${{ inputs.quarto-install || 'dev' }} + quarto-version: ${{ inputs.quarto-version }} + quarto-artifact-name: ${{ inputs.quarto-artifact-name }} + quarto-artifact-run-id: ${{ inputs.quarto-artifact-run-id }} + ref: ${{ inputs.ref }} + runners: ${{ inputs.runners || '["ubuntu-latest", "windows-latest"]' }} diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml new file mode 100644 index 00000000000..856e4b8a8ba --- /dev/null +++ b/.github/workflows/test-smokes-built.yml @@ -0,0 +1,359 @@ +name: Smoke Tests (Built Version) +on: + # Test scheduled and manually dispatched create-release builds. + workflow_run: + workflows: ["Build Installers"] + types: [completed] + workflow_dispatch: + inputs: + source: + description: "Quarto source: build this ref, use artifacts from a create-release run, or install a published release." + required: false + type: choice + options: + - build + - nightly + - release + default: build + buckets: + description: "JSON array of smoke-test buckets or globs. When set, only those buckets run; empty also runs the full smoke, Playwright, and feature-format legs." + required: false + type: string + default: "" + version: + description: "For source=release: pre-release, release, or a version such as 1.9.10" + required: false + type: string + default: "pre-release" + run-id: + description: "For source=nightly: create-release run ID; empty uses the latest successful run" + required: false + type: string + default: "" + +# Required by test-smokes.yml for Julia cache cleanup. +permissions: + actions: write + contents: read + +jobs: + # Build the selected ref (the dispatch default). + build-artifact: + name: Build quarto dist (linux-amd64) + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.rec.outputs.sha }} + steps: + - name: Checkout Repo + uses: actions/checkout@v6 + + - name: Record commit under test + id: rec + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + - name: Compute version marker + id: version + shell: bash + # Build metadata keeps quarto-required range checks valid. + run: echo "version=$(cat version.txt)+test.$(date +%Y%m%d)" >> "$GITHUB_OUTPUT" + + - name: Build dist tarball + uses: ./.github/actions/build-dist-tarball + with: + version: ${{ steps.version.outputs.version }} + arch: amd64 + tarball-name: built-quarto-linux-amd64.tar.gz + artifact-name: built-quarto-linux-amd64 + # Consumed only by this workflow run. + retention-days: 1 + + run-smokes-artifact: + name: Smoke tests against built artifact + if: (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-playwright-artifact: + name: Playwright tests against built artifact + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + && github.event.inputs.buckets == '' + needs: [build-artifact] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-artifact: + name: Feature-format matrix against built artifact + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'build' + && github.event.inputs.buckets == '' + needs: [build-artifact] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.build-artifact.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: built-quarto-linux-amd64 + runners: '["ubuntu-latest"]' + + # Test an existing published release. + resolve-release: + name: Resolve release version + if: github.event.inputs.source == 'release' + runs-on: ubuntu-latest + outputs: + version: ${{ steps.r.outputs.version }} + steps: + - name: Resolve version + id: r + shell: bash + env: + VERSION_INPUT: ${{ github.event.inputs.version }} + run: | + input="$VERSION_INPUT" + case "$input" in + pre-release | prerelease | "") + version="$(curl -fsSL https://quarto.org/docs/download/_prerelease.json | jq -r '.version')" + ;; + release) + version="$(curl -fsSL https://quarto.org/docs/download/_download.json | jq -r '.version')" + ;; + *) + version="$input" + ;; + esac + if [ -z "$version" ] || [ "$version" = "null" ]; then + echo "::error::Could not resolve a Quarto version from '$input'. Use pre-release, release, or an explicit version." + exit 1 + fi + echo "Resolved version: $version" + echo "version=$version" >> "$GITHUB_OUTPUT" + + - name: Preflight - tag must have binary-mode harness support + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERSION: ${{ steps.r.outputs.version }} + # Older tags do not contain the binary-mode test harness. + run: | + if output="$(gh api "repos/${GITHUB_REPOSITORY}/contents/tests/quarto-cmd.ts?ref=refs/tags/v${VERSION}" --silent 2>&1)"; then + echo "refs/tags/v${VERSION} has binary-mode harness support" + elif echo "$output" | grep -q 'HTTP 404'; then + echo "::error::refs/tags/v${VERSION} does not exist or lacks tests/quarto-cmd.ts. Choose a release with binary-mode harness support." + exit 1 + else + echo "::error::gh api failed while checking refs/tags/v${VERSION}: $output" + exit 1 + fi + + run-smokes-release: + name: Smoke tests against published release + if: github.event.inputs.source == 'release' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest", "windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-playwright-release: + name: Playwright tests against published release + if: github.event.inputs.source == 'release' && github.event.inputs.buckets == '' + needs: [resolve-release] + uses: ./.github/workflows/test-smokes.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-release: + name: Feature-format matrix against published release + if: github.event.inputs.source == 'release' && github.event.inputs.buckets == '' + needs: [resolve-release] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: refs/tags/v${{ needs.resolve-release.outputs.version }} + quarto-install: release + quarto-version: ${{ needs.resolve-release.outputs.version }} + runners: '["ubuntu-latest", "windows-latest"]' + + # Reuse artifacts from a completed create-release run. + resolve-nightly: + name: Resolve create-release run + # Only successful workflow_run builds are testable. + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && (github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success') + runs-on: ubuntu-latest + outputs: + run-id: ${{ steps.r.outputs.run-id }} + sha: ${{ steps.r.outputs.sha }} + has-linux: ${{ steps.r.outputs.has-linux }} + has-windows: ${{ steps.r.outputs.has-windows }} + has-mac: ${{ steps.r.outputs.has-mac }} + steps: + - name: Resolve run id, commit and artifacts + id: r + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID_INPUT: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.id || github.event.inputs.run-id }} + run: | + run_id="$RUN_ID_INPUT" + if [ -z "$run_id" ]; then + run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow create-release.yml \ + --branch main --status success --limit 1 --json databaseId --jq '.[0].databaseId // empty')" + fi + if [ -z "$run_id" ]; then + echo "::error::No successful create-release run was found. Provide a run ID or complete a create-release run first." + exit 1 + fi + sha="$(gh run view "$run_id" --repo "$GITHUB_REPOSITORY" --json headSha --jq '.headSha // empty')" + if [ -z "$sha" ]; then + echo "::error::Could not resolve the head SHA for create-release run ${run_id}" + exit 1 + fi + # Run only legs whose artifacts still exist. + names="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts" \ + --paginate --jq '.artifacts[] | select(.expired | not) | .name')" + has() { grep -Fxq "$1" <<<"$names" && echo true || echo false; } + has_linux="$(has 'Deb Zip')" + has_windows="$(has 'Windows Zip')" + has_mac="$(has 'Mac Zip')" + if [ "$has_linux" = false ] && [ "$has_windows" = false ] && [ "$has_mac" = false ]; then + echo "::error::run ${run_id} produced none of the expected artifacts (Deb Zip / Windows Zip / Mac Zip)" + exit 1 + fi + echo "Using create-release run ${run_id} at commit ${sha} (linux=${has_linux} windows=${has_windows} mac=${has_mac})" + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + echo "has-linux=$has_linux" >> "$GITHUB_OUTPUT" + echo "has-windows=$has_windows" >> "$GITHUB_OUTPUT" + echo "has-mac=$has_mac" >> "$GITHUB_OUTPUT" + + run-smokes-nightly-linux: + name: Smoke tests against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + # Publish runs add only a version/changelog commit after this SHA. + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-windows: + name: Smoke tests against nightly build (windows) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # Scheduled builds leave quarto.exe unsigned; dispatch builds sign it. + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + run-smokes-nightly-mac: + name: Smoke tests against nightly build (macOS) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + # Signed and notarized; also provides the only macOS smoke coverage. + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: ${{ github.event.inputs.buckets }} # empty = full run + + # Each caller job accepts one OS-specific artifact. + run-playwright-nightly-linux: + name: Playwright tests against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-playwright-nightly-mac: + name: Playwright tests against nightly build (macOS) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-mac == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-smokes.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Mac Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["macos-latest"]' + buckets: '["integration/playwright-tests.test.ts"]' + + run-ff-matrix-nightly-linux: + name: Feature-format matrix against nightly build (linux) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-linux == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Deb Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["ubuntu-latest"]' + + run-ff-matrix-nightly-windows: + name: Feature-format matrix against nightly build (windows) + if: >- + (github.event_name == 'workflow_run' && 'nightly' || github.event.inputs.source || 'build') == 'nightly' + && needs.resolve-nightly.outputs.has-windows == 'true' + && github.event.inputs.buckets == '' + needs: [resolve-nightly] + uses: ./.github/workflows/test-ff-matrix.yml + with: + ref: ${{ needs.resolve-nightly.outputs.sha }} + quarto-install: artifact + quarto-artifact-name: Windows Zip + quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} + runners: '["windows-latest"]' diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 5297f5f8f02..d694798da2f 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -5,9 +5,10 @@ on: workflow_call: inputs: buckets: - description: "JSON string for buckets of tests to run in loop. Array of grouped tests." - required: true + description: "JSON array of test buckets. Empty runs the full suite." + required: false type: string + default: "" time-test: description: "Should we run tests to produce test file" required: false @@ -18,6 +19,36 @@ on: required: false type: string default: "" + quarto-install: + description: "Quarto source: dev tree, published release, or workflow artifact" + required: false + type: string + default: "dev" + quarto-version: + description: "Version to install when quarto-install is release" + required: false + type: string + default: "" + quarto-artifact-name: + description: "Artifact containing the Quarto tarball or zip" + required: false + type: string + default: "" + quarto-artifact-run-id: + description: "Run ID containing the artifact; empty uses the current run" + required: false + type: string + default: "" + ref: + description: "Git ref to check out; empty uses the default" + required: false + type: string + default: "" + runners: + description: "JSON array of runner labels; macos-latest is reserved for scheduled built-version runs" + required: false + type: string + default: '["ubuntu-latest", "windows-latest"]' workflow_dispatch: inputs: buckets: @@ -50,7 +81,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + # Non-call triggers use the default runner matrix. + os: ${{ fromJSON(inputs.runners || '["ubuntu-latest", "windows-latest"]') }} time-test: - ${{ inputs.time-test }} exclude: @@ -63,6 +95,9 @@ jobs: steps: - name: Checkout Repo uses: actions/checkout@v6 + with: + # Empty keeps the standard checkout behavior. + ref: ${{ inputs.ref }} - name: Fix temp dir to use runner one (windows) if: runner.os == 'Windows' @@ -85,16 +120,13 @@ jobs: with: node-version: 22 - # Playwright setup is only needed when this job can actually run the - # playwright suite: a bucket containing playwright-tests.test.ts, or a - # full run (empty buckets, which includes integration/). Windows never - # needs it: the suite is ignored on Windows CI (playwright-tests.test.ts, - # `ignore: gha.isGitHubActions() && isWindows`). + # Provision Playwright only for jobs that can run its assertions. - name: Cache multiplex server node_modules if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) uses: actions/cache@v5 with: path: tests/integration/playwright/multiplex-server/node_modules @@ -105,8 +137,9 @@ jobs: - name: Install node dependencies if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) run: npm install --loglevel=error --no-audit working-directory: ./tests/integration/playwright shell: bash @@ -114,8 +147,9 @@ jobs: - name: Get Playwright version if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) run: | VERSION=$(node -p "require('@playwright/test/package.json').version") echo "PLAYWRIGHT_VERSION=$VERSION" >> "$GITHUB_ENV" @@ -125,8 +159,9 @@ jobs: - name: Cache Playwright browsers if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) uses: actions/cache@v5 with: path: ${{ env.PLAYWRIGHT_BROWSERS_PATH }} @@ -137,8 +172,9 @@ jobs: - name: Install Playwright system dependencies if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) timeout-minutes: 15 run: npx playwright install-deps working-directory: ./tests/integration/playwright @@ -146,8 +182,9 @@ jobs: - name: Install Playwright Browsers if: >- runner.os != 'Windows' - && (format('{0}', inputs.buckets) == '' - || contains(inputs.buckets, 'playwright-tests.test.ts')) + && (contains(inputs.buckets, 'playwright-tests.test.ts') + || (format('{0}', inputs.buckets) == '' + && (inputs.quarto-install == 'dev' || inputs.quarto-install == ''))) timeout-minutes: 15 run: npx playwright install working-directory: ./tests/integration/playwright @@ -196,6 +233,14 @@ jobs: sudo apt-get update -y sudo apt-get install -y librsvg2-bin + - name: Install missing system deps (macOS) + if: runner.os == 'macOS' + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + # PDF tests need poppler and librsvg beyond the preinstalled tools. + run: | + brew install poppler librsvg + - name: Restore R packages working-directory: tests run: | @@ -250,8 +295,122 @@ jobs: run: | uv sync --frozen + # The harness needs the dev runtime even when testing another binary. - uses: ./.github/workflows/actions/quarto-dev + - name: Set up release quarto + if: inputs.quarto-install == 'release' + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: ${{ inputs.quarto-version }} + + - name: Download built quarto artifact + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id == '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + + - name: Download built quarto artifact (from another run) + if: inputs.quarto-install == 'artifact' && inputs.quarto-artifact-run-id != '' + uses: actions/download-artifact@v8 + with: + name: ${{ inputs.quarto-artifact-name }} + path: ${{ runner.temp }}/quarto-artifact + run-id: ${{ inputs.quarto-artifact-run-id }} + github-token: ${{ github.token }} + + - name: Install built quarto outside the checkout (Windows) + if: inputs.quarto-install == 'artifact' && runner.os == 'Windows' + shell: pwsh + # Git for Windows' bash ships GNU tar, which cannot read a zip archive, + # and the Windows artifact is always a zip - so this step runs under + # pwsh instead, where Expand-Archive handles it natively. + run: | + $zips = Get-ChildItem -Path "$env:RUNNER_TEMP/quarto-artifact" -Filter *.zip + if ($zips.Count -ne 1) { + Write-Host "::error::expected exactly one .zip in the quarto artifact, found: $($zips.Name -join ', ')" + exit 1 + } + $dest = "$env:RUNNER_TEMP/quarto-under-test" + Expand-Archive -Path $zips[0].FullName -DestinationPath $dest -Force + if (-not (Test-Path "$dest/bin")) { + $tops = Get-ChildItem -Path $dest -Force -Directory + if ($tops.Count -ne 1 -or -not (Test-Path (Join-Path $tops[0].FullName "bin"))) { + Write-Host "::error::could not locate bin/ in the extracted quarto artifact" + exit 1 + } + Get-ChildItem -Path $tops[0].FullName -Force | Move-Item -Destination $dest + } + Add-Content $env:GITHUB_PATH "$dest/bin" + + - name: Install built quarto outside the checkout + if: inputs.quarto-install == 'artifact' && runner.os != 'Windows' + shell: bash + # An in-checkout distribution would launch the dev sources. + run: | + mkdir -p "$RUNNER_TEMP/quarto-under-test" + shopt -s nullglob + artifacts=("$RUNNER_TEMP"/quarto-artifact/*.tar.gz "$RUNNER_TEMP"/quarto-artifact/*.zip) + if [ "${#artifacts[@]}" -ne 1 ]; then + echo "::error::expected exactly one .tar.gz or .zip in the quarto artifact, found: ${artifacts[*]:-none}" + exit 1 + fi + # Linux has a top directory; macOS archives are flat. + tar -xf "${artifacts[0]}" -C "$RUNNER_TEMP/quarto-under-test" + if [ ! -d "$RUNNER_TEMP/quarto-under-test/bin" ]; then + tops=("$RUNNER_TEMP"/quarto-under-test/*/) + if [ "${#tops[@]}" -ne 1 ]; then + echo "::error::expected exactly one directory in the extracted quarto artifact, found: ${#tops[@]}" + exit 1 + fi + top="${tops[0]%/}" + if [ ! -d "$top/bin" ]; then + echo "::error::could not locate bin/ in the extracted quarto artifact" + exit 1 + fi + shopt -s dotglob + mv "$top"/* "$RUNNER_TEMP/quarto-under-test"/ + fi + echo "$RUNNER_TEMP/quarto-under-test/bin" >> "$GITHUB_PATH" + + - name: Pin and verify test target + if: inputs.quarto-install != 'dev' && inputs.quarto-install != '' + shell: bash + env: + QUARTO_VERSION_INPUT: ${{ inputs.quarto-version }} + run: | + v="$(quarto --version)" + echo "quarto under test: $v ($(command -v quarto))" + if [ "$v" = "99.9.9" ]; then + echo "::error::Quarto on PATH reports the dev version 99.9.9. The selected built or released Quarto was not installed." + exit 1 + fi + # Only build metadata preserves quarto-required range semantics. + if ! echo "$v" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+(\+[0-9A-Za-z.-]+)?$'; then + echo "::error::Quarto version '$v' is invalid for built-version tests. Use X.Y.Z or X.Y.Z+build metadata." + exit 1 + fi + # Older tags do not contain the binary-mode harness. + if [ ! -f tests/quarto-cmd.ts ]; then + echo "::error::This checkout lacks tests/quarto-cmd.ts. Use a ref with binary-mode harness support." + exit 1 + fi + # Convert the executable to a native Windows path for Deno.Command. + if [ "$RUNNER_OS" = "Windows" ]; then + bin="$(command -v quarto.exe || command -v quarto.cmd || command -v quarto)" + bin="$(cygpath -w "$bin")" + else + bin="$(command -v quarto)" + fi + echo "QUARTO_TEST_BIN=$bin" >> "$GITHUB_ENV" + # Verify every spawn when the caller requested a concrete version. + if [ -n "$QUARTO_VERSION_INPUT" ] \ + && [ "$QUARTO_VERSION_INPUT" != "pre-release" ] \ + && [ "$QUARTO_VERSION_INPUT" != "release" ]; then + echo "QUARTO_TEST_EXPECTED_VERSION=$QUARTO_VERSION_INPUT" >> "$GITHUB_ENV" + fi + - name: Install Tinytex env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -331,10 +490,21 @@ jobs: env: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUCKETS_JSON: ${{ inputs.buckets }} run: | haserror=0 failed_tests=() - readarray -t my_array < <(echo '${{ inputs.buckets }}' | jq -rc '.[]') + # Reject malformed and empty arrays instead of running zero buckets. + if ! bucket_list=$(printf '%s' "$BUCKETS_JSON" | jq -e -rc '.[]' 2>&1); then + echo "::error::BUCKETS_JSON is not a valid non-empty JSON array: $BUCKETS_JSON" + echo "$bucket_list" + exit 1 + fi + # macOS Bash 3.2 does not provide readarray. + my_array=() + while IFS= read -r bucket; do + my_array+=("$bucket") + done <<< "$bucket_list" for file in "${my_array[@]}"; do echo "::group::Testing ${file}" echo ">>> ./run-tests.sh ${file}" @@ -369,10 +539,16 @@ jobs: env: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUCKETS_JSON: ${{ inputs.buckets }} run: | $haserror=$false $failed_tests=@() - foreach ($file in ('${{ inputs.buckets }}' | ConvertFrom-Json)) { + $buckets=@($env:BUCKETS_JSON | ConvertFrom-Json) + if ($buckets.Count -eq 0) { + Write-Host "::error::BUCKETS_JSON resolved to zero test buckets: $env:BUCKETS_JSON" + Exit 1 + } + foreach ($file in $buckets) { Write-Host "::group::Testing ${file}" Write-Host ">>> ./run-tests.ps1 ${file}" ./run-tests.ps1 $file @@ -423,9 +599,9 @@ jobs: key: ${{ steps.cache-renv-packages-restore.outputs.cache-primary-key }} - uses: actions/upload-artifact@v7 - # Upload pLaywright test report if they exists (playwright is only running on Linux for now) + # Include the OS because sibling jobs share the workflow run. if: ${{ !cancelled() && runner.os != 'Windows' && hashFiles('tests/integration/playwright/playwright-report/**/*') != '' }} with: - name: playwright-report + name: playwright-report-${{ runner.os }} path: ./tests/integration/playwright/playwright-report/ retention-days: 30 diff --git a/dev-docs/checklist-make-a-new-quarto-prerelease.md b/dev-docs/checklist-make-a-new-quarto-prerelease.md index a7c0927cbe9..4845be1d820 100644 --- a/dev-docs/checklist-make-a-new-quarto-prerelease.md +++ b/dev-docs/checklist-make-a-new-quarto-prerelease.md @@ -13,5 +13,6 @@ - New release prerelease should be on Github at - A new tag should be on main for the new prerelease version - `version.txt` on main should have been updated by the workflow to the pre-release version just released: https://github.com/quarto-dev/quarto-cli/blob/main/version.txt +- Optionally validate the published prerelease: in Actions, run "Smoke Tests (Built Version)" with source `release`. The version defaults to `pre-release`. Note: Cloudsmith publishing is skipped for prereleases (only runs for stable releases). diff --git a/dev-docs/debugging-flaky-tests.md b/dev-docs/debugging-flaky-tests.md index 67e30dc33e4..59ed5aa85eb 100644 --- a/dev-docs/debugging-flaky-tests.md +++ b/dev-docs/debugging-flaky-tests.md @@ -184,8 +184,16 @@ quarto install tinytex for test in test1.ts test2.ts test3.ts; do ./run-tests.sh $test || break done + +# Run against a built quarto instead of the in-process dev sources +# (binary mode; see llm-docs/built-version-testing-architecture.md) +QUARTO_TEST_BIN=/path/to/installed/quarto ./run-tests.sh path/to/test.ts ``` +By default, the harness invokes Quarto in-process. With `QUARTO_TEST_BIN` set, +each invocation runs in a subprocess. A flake that disappears in binary mode +may indicate shared state in the dev harness. + ### Package/Dependency Comparison ```bash diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md new file mode 100644 index 00000000000..dea19b74a2b --- /dev/null +++ b/llm-docs/built-version-testing-architecture.md @@ -0,0 +1,290 @@ +--- +main_commit: e5850df75 +analyzed_date: 2026-09-10 +key_files: + - tests/quarto-cmd.ts + - tests/test.ts + - tests/run-tests.sh + - tests/run-tests.ps1 + - tests/integration/playwright-tests.test.ts + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/test-ff-matrix.yml + - .github/workflows/create-release.yml + - .github/actions/build-dist-tarball/action.yml + - .github/actions/merge-extension-tests/action.yml +--- + +# Built-Version Testing Architecture + +This document explains how the test suite runs against a **built** Quarto distribution (binary mode) and records the design decisions behind the test harness and CI wiring. + +Document map: + +- **This doc** — architecture summary, flow diagrams, and design decisions. +- `tests/README.md` → "Binary mode" — local commands and authoring rules. +- `llm-docs/testing-patterns.md` → "Dev Mode vs Binary Mode" — authoring patterns for tests that must work in both modes. + +## Architecture in one paragraph + +Every `testQuartoCmd()` test invokes Quarto through `runQuarto()` in `tests/quarto-cmd.ts`. +Dev mode calls the in-process `quarto()` entry point from `src/quarto.ts`. +Binary mode spawns the executable in `QUARTO_TEST_BIN`, merges its JSON-stream log into the test log, and uses the same verifiers. + +In CI, `test-smokes.yml` accepts dev, release, or artifact install sources. +`test-smokes-built.yml` resolves build, nightly, and release sources, then schedules smoke, Playwright, and feature-format legs. + +## Flow diagrams + +### Test invocation + +Every `testQuartoCmd()`-based test goes through `runQuarto()`. +Verifiers read the same log records and rendered outputs in both modes. + +```mermaid +flowchart TB + subgraph deno ["Deno test process (tests/ harness, always runs from the repo checkout)"] + TQC["testQuartoCmd / testRender / testSite / smoke-all driver"] + RQ{"runQuarto()
tests/quarto-cmd.ts"} + LOG[("json-stream log file
{msg, level, levelName} per line")] + OUT[("rendered output files")] + VER["verifiers (tests/verify.ts)
noErrors, printsMessage, ensureHtmlElements, ..."] + end + DEV["in-process quarto()
imported from src/quarto.ts
dev TS sources, version 99.9.9"] + BIN["spawned subprocess: built quarto
--log file --log-format json-stream
dev env vars stripped (QUARTO_SHARE_PATH, DENO_DIR, ...)"] + + TQC --> RQ + RQ -->|"dev mode (default:
QUARTO_TEST_BIN unset)"| DEV + RQ -->|"binary mode
(QUARTO_TEST_BIN set)"| BIN + DEV --> LOG + DEV --> OUT + BIN --> LOG + BIN --> OUT + LOG --> VER + OUT --> VER +``` + +### Binary-mode test lifecycle + +```mermaid +sequenceDiagram + participant T as test() (tests/test.ts) + participant R as runQuarto() + participant Q as built quarto (subprocess) + participant V as verifiers + + T->>T: create temp json-stream log file + Note over T: harness logger disabled
(the child owns log capture) + T->>R: execute(logFile) + R->>Q: spawn QUARTO_TEST_BIN render ...
--log (per-invocation temp) --log-format json-stream
env = ambient minus dev-tree vars, plus TestContext.env + Q->>Q: render, write log records + output files + Q-->>R: exit (code, stdout/stderr drained) + R->>T: merge child log into the test log file + alt exit != 0 and no ERROR record in child log + R->>T: append synthetic ERROR record
(exit code + stderr tail) + end + alt timeout + R->>Q: kill process tree + R->>T: append timeout ERROR record + end + T->>V: verify(log records) + verify(output files) +``` + +### CI workflow + +```mermaid +flowchart LR + subgraph dev ["Dev mode: quarto = in-process TS sources"] + PR["PR / push"] --> TSP["test-smokes-parallel.yml
sharded buckets"] + DAILY["daily schedule"] --> TSfull["full run"] + end + subgraph built ["Binary mode: quarto = built distribution (QUARTO_TEST_BIN)"] + TSB["test-smokes-built.yml
after every nightly build + manual dispatch
smoke + playwright + ff-matrix"] + BUILDM["source: build
build linux-amd64 dist from this ref"] + NIGHTM["source: nightly
reuse packaged create-release artifacts"] + RELM["source: release
install published (pre-)release"] + TSB -->|"dispatch"| BUILDM + TSB -->|"after each nightly build
or dispatch"| NIGHTM + TSB -->|"dispatch"| RELM + end + CR["create-release.yml
scheduled build without publish
dispatch can publish or build test artifacts"] + ACT[".github/actions/build-dist-tarball
shared build recipe"] + + FFM["test-ff-matrix.yml (reusable)
owns the feature-format bucket"] + TS["test-smokes.yml (reusable)
inputs: install source, ref, runners,
buckets, artifact"] + TSP --> TS + DAILY --> TS + FFM --> TS + BUILDM -->|"smoke + playwright"| TS + NIGHTM -->|"smoke + playwright"| TS + RELM -->|"smoke + playwright"| TS + BUILDM -->|"ff-matrix"| FFM + NIGHTM -->|"ff-matrix"| FFM + RELM -->|"ff-matrix"| FFM + BUILDM -. uses .-> ACT + CR -. "tarball jobs use" .-> ACT + NIGHTM -. "downloads artifacts from" .-> CR +``` + +### `QUARTO_TEST_BIN` propagation in CI + +The "Pin and verify test target" step in `test-smokes.yml` computes `QUARTO_TEST_BIN` and exports it through `$GITHUB_ENV`. +Dev-mode callers skip the install and pin steps. + +```mermaid +flowchart TB + IN["workflow input
quarto-install: artifact | release"] + QD["quarto-dev action
provisions the harness Deno runtime"] + INST["install quarto under test
artifact: extract outside checkout
release: quarto-actions/setup"] + PIN["pin and verify test target
reject 99.9.9; check semver
export QUARTO_TEST_BIN"] + RTS["./run-tests.sh
binary-mode guard
default selection = smoke/"] + QC["tests/quarto-cmd.ts
runQuarto spawns QUARTO_TEST_BIN"] + + IN --> QD --> INST --> PIN --> RTS --> QC +``` + +## When to use which mode + +| Mode | Quarto under test | Trigger | Suites (legs) | Question answered | +| ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------- | +| dev (`test-smokes.yml`) | in-process TS sources (99.9.9) | every PR/push + daily cron | everything (sharded per-commit; ff-matrix via its own cron/push/PR) | did this code change break behavior? | +| nightly | packaged nightly artifacts (Linux tarball, real `quarto.exe`, notarized Mac zip); Windows signing is skipped on the *scheduled* build, see D11 | automatic, after each nightly build | smoke (linux+windows+mac) + playwright (linux+mac) + ff-matrix (linux+windows) | does what we *ship* work? (bundling/packaging/launcher bugs; only macOS smoke coverage in CI) | +| build | fresh linux-amd64 dist from the current ref (unsigned) | manual dispatch | smoke + playwright + ff-matrix (all linux) | does this ref work when packaged? (works on forks/PR branches) | +| release | published (pre-)release via quarto-actions/setup, harness at its `v` tag | manual dispatch | smoke (linux+windows) + playwright (linux) + ff-matrix (linux+windows) | does the published version pass? | + +Dev mode covers unit tests, dev-only paths, and in-process behavior. Built modes cover the packaged product. +The non-Playwright integration tests remain in the dev shards. + +In practice: + +- `nightly` runs automatically after each nightly build and tests the available packaged binaries. + Scheduled builds skip Windows signing (D11). +- Dispatch `build` for immediate Linux feedback on packaging or harness changes. +- For signed Windows artifacts from a branch, dispatch `create-release` with `publish-release=false` and `smoke-artifacts-only=true`. The resulting workflow run starts the available test legs automatically (D7). +- Dispatch `nightly` with a `run-id` to retest an earlier `create-release` run. +- Dispatch `release` for post-publish verification, such as the optional prerelease checklist step. + It works only for releases cut after the harness support merged (D10). + +## Built-mode test legs (scheduler layout) + +`test-smokes-built.yml` = the mode **resolvers** (build-artifact / resolve-nightly / resolve-release, unchanged) + a **scheduler**: per-leg caller jobs fanning out to the reusable workflows. +Each source mode schedules three independent legs: + +| leg | goes through | bucket | OS scope | +| ---------- | ------------------------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------ | +| smoke | `test-smokes.yml` | `inputs.buckets` (empty = binary-mode `smoke/` default) | build: linux; nightly: linux+windows+mac (`has-*` gated); release: linux+windows | +| playwright | `test-smokes.yml` | `["integration/playwright-tests.test.ts"]` | linux (+ mac on nightly) — **never windows**, see below | +| ff-matrix | `test-ff-matrix.yml` (reusable) | owned by `test-ff-matrix.yml` | linux (+ windows on nightly/release) — no macOS (Julia/TeX toolchain unproven there) | + +Key points: + +- **Smoke is also the general bucket runner.** When a manual dispatch sets `buckets`, only smoke jobs run. The Playwright and feature-format jobs require an empty `buckets` input. +- **Windows has no Playwright leg.** `playwright-tests.test.ts` ignores browser assertions on Windows CI. Before adding this leg, update that gate and the report-upload gate in `test-smokes.yml`. +- **Playwright renders need a sanitized environment.** Calls through `execProcess` and `quartoDevCmd()` must pass `quartoSpawnEnvOptions()` so built Quarto does not inherit dev-tree paths. +- **Playwright reports include the OS in their artifact names.** Sibling jobs share a workflow run and cannot upload artifacts with the same name. +- **Scheduler jobs own OS scope.** Set it through their `runners` inputs. + +### `test-ff-matrix.yml` is reusable (`workflow_call`) + +The feature-format bucket glob (`../dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) is defined only in `test-ff-matrix.yml`. +Built-mode callers use its `workflow_call` trigger, while its existing dev triggers remain. +The workflow forwards install, artifact, ref, runner, and R-package inputs to `test-smokes.yml`, with dev defaults for non-call triggers. + +Reusable-workflow concurrency is evaluated in the caller's context. The group therefore includes a suffix based on `inputs.runners` and `github.run_id`, preventing sibling feature-format legs from canceling one another. +Dev triggers use a constant `-dev` suffix. +The workflow does not declare `permissions`, so it inherits the caller's `actions: write` permission for Julia cache cleanup. + +## Design decisions + +Each entry: what was decided, why, and what would justify revisiting. + +### D1. Nightly wiring: `workflow_run`, not `workflow_call`/dispatch/inversion + +**Decision.** `test-smokes-built.yml` listens for completed "Build Installers" (`create-release.yml`) runs via `workflow_run` and reuses their artifacts cross-run (`quarto-artifact-run-id`). +The release pipeline is not modified for testing purposes. + +**Alternatives considered (2026-07, maintainer question):** + +- *Dispatch tests from create-release* — adds permissions and dispatch code to the release workflow without avoiding the default-branch constraint. +- *Call `test-smokes.yml` from create-release* — simplifies artifact access, but test failures would mark release builds as failed and test configuration would move into the release workflow. +- *Let `test-smokes-built` call create-release* — still requires the full daily build, couples nightly builds to test-workflow availability, and moves them out of the "Build Installers" history. + +**Rationale:** `workflow_run` keeps release and test status separate without adding test orchestration to the release workflow. + +**Known weaknesses:** the trigger depends on the workflow display name (`workflows: ["Build Installers"]`), so renaming the workflow stops the trigger. GitHub does not report a missing trigger as a failure. +The trigger fires after every completed create-release run, including manual and partial builds. Each OS leg therefore checks that its artifact exists. + +**Revisit when:** maintainers want a single nightly build-and-test status and are willing to couple the workflows. + +### D2. Version marker: semver *build metadata* (`X.Y.Z+test.YYYYMMDD`) + +Built test distributions use `$(cat version.txt)+test.$(date +%Y%m%d)`. +Do not use a prerelease suffix, which fails plain `>=X.Y` `quarto-required` ranges, or a fourth numeric component, which is invalid semver. +Build metadata preserves range comparisons while distinguishing the build from the `99.9.9` dev version. + +### D3. Dist outside the checkout + `99.9.9` sentinel refusal + +Installed launchers detect dev mode via a sibling `src/quarto.ts`: an in-repo `package/dist/bin/quarto` silently runs the TS sources instead of the built code. +Therefore the dist under test must be extracted *outside* the repo, and both `run-tests.[sh|ps1]` and `assertTestBinary()` refuse a binary reporting `99.9.9` (`kLocalDevelopment`). +CI extracts artifacts to `RUNNER_TEMP`. + +### D4. Child env: inherit ambient + strip dev vars (not clearEnv+allowlist) + +Binary-mode spawns inherit the ambient environment minus a strip list (`QUARTO_SHARE_PATH`, `QUARTO_BIN_PATH`, `DENO_DIR`, `QUARTO_DEBUG`, `QUARTO_FORCE_VERSION`, ...), with `TestContext.env` overlaid last. +A `clearEnv` allowlist was rejected because the required Windows system variables (`SystemRoot`, `PATHEXT`, and others) are difficult to maintain reliably. +The dev-tree exports in `run-tests.[sh|ps1]` are kept in all modes — the *harness* process still needs them; only the *child* is sanitized. + +### D5. Silent-green guard: synthetic ERROR records + +Some failures exit without an error log record, including pre-logger failures and command passthroughs. Many smoke-all documents rely only on log verification. +When a failed child has no error record, `runQuarto()` appends one with the exit code and stderr tail. It also records timeouts after killing the process tree. + +### D6. `QUARTO_TEST_BIN` is set at runtime, never declared statically + +The "Pin and verify test target" step in `test-smokes.yml` resolves the installed binary, verifies it (sentinel refusal, semver shape, optional `QUARTO_TEST_EXPECTED_VERSION` match), and exports it via `$GITHUB_ENV` so every later step — including the unchanged `run-tests.sh` invocation — sees it. +These steps are skipped when `quarto-install` is `dev`. + +### D7. `smoke-artifacts-only` is for partial branch builds + +This `create-release.yml` input omits source and arm64 tarballs, Linux installers, and the Mac build. +For a branch build, dispatch create-release with `publish-release=false` and `smoke-artifacts-only=true`; the `workflow_run` trigger tests the artifacts. +The configure job rejects this mode when publishing is enabled, and partial builds use a per-run concurrency group. +The daily path still uses the full build because the Mac zip provides macOS smoke coverage. + +### D8. macOS runners: scheduled/built runs only, never per-commit + +`test-smokes-parallel.yml` (per-commit) must stay fast, so it never passes `runners` and keeps the `ubuntu-latest`/`windows-latest` default. +The only `macos-latest` smoke job is the nightly Mac leg in `test-smokes-built.yml`. +Encoded in the `runners` input description in `test-smokes.yml`. + +### D9. Built mode runs smoke + playwright + ff-matrix daily; the dev crons stay + +Built mode runs smoke, Playwright (`integration/playwright-tests.test.ts`), and the feature-format matrix. +The test-script default previously excluded the latter two; the harness now supports both. + +What stays dev-only: `unit/` (in-process by definition), the non-playwright `integration/` tests (`guess-chunk-options-format-document.test.ts`, `mermaid/github-issue-1340.test.ts` — dev shards only), `QUARTO_DEBUG` paths, the `quarto check` dev branch, and in-process races. +The julia-engine subtree tests are temporarily dev-only. Their direct Quarto subprocesses inherit dev-tree variables, so `merge-extension-tests` skips them in binary mode until the upstream spawns are sanitized. + +Remaining built-mode gaps are preview and serve paths, publishing, installer behavior, Linux arm64, and Playwright visual snapshots. +Windows browser behavior and macOS feature-format coverage are also excluded as described in "Built-mode test legs". +Dev schedules remain because they test source behavior, while built schedules test packaged behavior. +All built legs currently run after every completed `create-release` run. If CI cost becomes excessive, gate heavy legs on scheduled runs. + +### D10. Release mode only works for post-harness tags + +Release mode checks out the tag, so the harness at that tag must already contain `tests/quarto-cmd.ts` — a preflight fails clearly for older releases. +Testing an older binary with the current harness would require decoupling the harness from the target ref. + +Nightly and release legs use the harness from the target ref. Refs that contain `tests/quarto-cmd.ts` but predate the `quartoSpawnEnvOptions()` fix in `playwright-tests.test.ts` (#14706) run Playwright with dev-tree resources. +The preflight cannot detect this window. Smoke and feature-format legs are unaffected because their subprocesses use `runQuarto()`. + +### D11. The automatic nightly leg tests an *unsigned* `quarto.exe` + +`create-release.yml` gates both DigiCert steps ("Sign files before making ZIP and MSI installer", "Sign MSI installer") on `github.event_name != 'schedule'`, so the daily scheduled build — the one `workflow_run` fires on every night — packages an unsigned `quarto.exe` into `Windows Zip`. +macOS is unaffected: `make-installer-mac` signs and notarizes on every event. + +This is sufficient for the packaging and launcher checks in binary mode. +The daily run exercises the real launcher (`package/launcher` `quarto.exe`) rather than the dev `.cmd` shim. +Signing changes the bytes, not the launcher's argument handling or resource resolution. + +Signed Windows coverage comes from dispatched `create-release` runs, which `workflow_run` also tests (D7). The scheduled path does not validate signing. diff --git a/llm-docs/testing-patterns.md b/llm-docs/testing-patterns.md index 5dc9e9d2aae..31d250ad9bb 100644 --- a/llm-docs/testing-patterns.md +++ b/llm-docs/testing-patterns.md @@ -1,3 +1,13 @@ +--- +main_commit: e5850df75 +analyzed_date: 2026-09-10 +key_files: + - tests/test.ts + - tests/quarto-cmd.ts + - tests/verify.ts + - tests/utils.ts +--- + # Quarto Test Patterns This document describes the standard patterns for writing smoke tests in the Quarto CLI test suite. @@ -7,9 +17,26 @@ This document describes the standard patterns for writing smoke tests in the Qua Quarto uses Deno for testing with custom verification helpers located in: - `tests/test.ts` - Core test runner (`testQuartoCmd`) +- `tests/quarto-cmd.ts` - Quarto invocation dispatch (`runQuarto`: in-process dev quarto vs built binary) - `tests/verify.ts` - Verification helpers (`fileExists`, `pathDoNotExists`, etc.) - `tests/utils.ts` - Utility functions (`docs()`, `outputForInput()`, etc.) +### Dev Mode vs Binary Mode + +`testQuartoCmd` invokes Quarto through `runQuarto()` in `tests/quarto-cmd.ts`: + +- **Dev mode (default):** Quarto runs in-process via the `quarto()` entry point imported from `src/quarto.ts`. +- **Binary mode:** `QUARTO_TEST_BIN` points to an installed Quarto outside the checkout. `runQuarto()` spawns it with JSON-stream logging and removes dev-tree variables from its environment. + +The test scripts reject binaries that report the `99.9.9` dev version and default binary-mode runs to `smoke/`. +See `llm-docs/built-version-testing-architecture.md` for CI behavior and design decisions. + +Consequences for writing smoke tests: + +- Invoke Quarto through `testQuartoCmd()` or `runQuarto()`; do not import it from `src/quarto.ts`. +- For direct subprocesses, resolve the executable with `quartoDevCmd()` and pass `quartoSpawnEnvOptions()`. +- Set `TestContext.requiresDevQuarto: true` only for tests that require in-process internals. Prefer a unit test when possible. + ## Common Test Patterns ### Simple Render Tests @@ -91,7 +118,8 @@ testQuartoCmd( ### Performance Budget (Render Timeout) -`testQuartoCmd` runs the render under a default 10-minute timeout. For a test guarding a *performance* regression — a render that must not hang — set a tight budget via `TestContext.timeout` (milliseconds) so a regression fails fast instead of riding the 10-minute default: +`testQuartoCmd` runs the render under a default 10-minute timeout. +For a test guarding a *performance* regression — a render that must not hang — set a tight budget via `TestContext.timeout` (milliseconds) so a regression fails fast instead of riding the 10-minute default: ```typescript testQuartoCmd("render", [projectDir], [noErrors /*, ... */], { @@ -109,8 +137,9 @@ testQuartoCmd("render", [projectDir], [noErrors /*, ... */], { **Key points:** -- The budget is machine-dependent (post-fix render time must sit well under it, pre-fix hang well over it), so it is defense-in-depth. Pair it with a deterministic unit test on the actual fix mechanism as the primary guard. -- A timed-out render subprocess is not killed by the harness, so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. +- Allow enough margin for slower machines. Pair the timeout with a deterministic unit test of the underlying fix. +- In dev (in-process) mode a timed-out render is not killed by the harness (the timeout only rejects), so on Windows it may still hold the output directory; use `safeRemoveSync` in teardown and treat cleanup as best-effort. + In binary mode (`QUARTO_TEST_BIN`) the spawned process tree *is* killed on timeout, but the kill is best-effort — keep the same defensive teardown. ### Extension Template Tests @@ -171,11 +200,9 @@ testQuartoCmd( ### Working-Directory-Sensitive Tests -Some tests need to run from a specific directory (e.g. reproducing a bug that -depends on the process cwd). **Do not `Deno.chdir()` inside the test body** — -it mutates process-global cwd and can leak into other tests in the same -process. Use the `TestContext` options instead; the harness changes the cwd -before the test and restores it afterward: +Some tests need to run from a specific directory (e.g. reproducing a bug that depends on the process cwd). +**Do not `Deno.chdir()` inside the test body** — it mutates process-global cwd and can leak into other tests in the same process. +Use the `TestContext` options instead; the harness changes the cwd before the test and restores it afterward: ```typescript const workingDir = Deno.makeTempDirSync(); @@ -191,17 +218,11 @@ unitTest("runs from workingDir", async () => { **Key points:** -- The harness calls `cwd()` **before** `setup()`, so the directory must already - exist when `cwd()` runs — create it at module scope, not in `setup`. -- `teardown` runs **before** the harness restores the cwd, so on Windows the - temp dir may still be the cwd and resist removal. Wrap the removal in - try/catch (best-effort) — see `tests/smoke/use/template.test.ts` and - `tests/unit/dotenv-config.test.ts`. -- For a temp directory you don't need to run *from*, prefer `withTempDir` - (`tests/utils.ts`), which creates and recursively removes it in a `finally`. -- A test that only needs a **relative** input (not a specific cwd) can pass a - path relative to the current cwd (`relative(Deno.cwd(), absFile)`) without - changing directories at all. +- The harness calls `cwd()` **before** `setup()`, so the directory must already exist when `cwd()` runs — create it at module scope, not in `setup`. +- `teardown` runs **before** the harness restores the cwd, so on Windows the temp dir may still be the cwd and resist removal. + Wrap the removal in try/catch (best-effort) — see `tests/smoke/use/template.test.ts` and `tests/unit/dotenv-config.test.ts`. +- For a temp directory you don't need to run *from*, prefer `withTempDir` (`tests/utils.ts`), which creates and recursively removes it in a `finally`. +- A test that only needs a **relative** input (not a specific cwd) can pass a path relative to the current cwd (`relative(Deno.cwd(), absFile)`) without changing directories at all. ## Verification Helpers @@ -356,7 +377,8 @@ See `tests/smoke/use/template.test.ts` for extension template patterns. - **Python**: `tests/.venv/` (managed by uv/pyproject.toml) - **R**: `tests/renv/` + `tests/renv.lock` -The `configure-test-env` scripts ONLY manage these main environments. CI builds depend on this structure. +The `configure-test-env` scripts ONLY manage these main environments. +CI builds depend on this structure. **Do NOT create language environment files in test subdirectories:** @@ -395,42 +417,32 @@ cd tests Rscript -e "renv::install(); renv::snapshot()" ``` -**Note:** While Quarto supports local Project.toml files in document directories for production use, the quarto-cli test infrastructure specifically does NOT support this pattern. All test dependencies must be in the main `tests/` environment. +**Note:** While Quarto supports local Project.toml files in document directories for production use, the quarto-cli test infrastructure specifically does NOT support this pattern. +All test dependencies must be in the main `tests/` environment. ### R Tests That Change Working Directory -R resolves `.Rprofile` from the **exact** process cwd (no parent-directory -search). On CI, rmarkdown/knitr live only in `tests/renv`'s project library, -activated when cwd is `tests/` (via `tests/.Rprofile` sourcing -`renv/activate.R`). Most knitr tests never leave `tests/` — they pass paths -relative to the current cwd instead of changing directories — so activation -happens automatically. - -A test that must run with cwd set elsewhere (a scratch temp dir, via -`TestContext.cwd()` — see "Working-Directory-Sensitive Tests" above) loses -that activation: the R subprocess starts outside `tests/`, renv never -activates, and package loads fail with `there is no package called -'rmarkdown'`. This is CI-only — a developer machine with rmarkdown on the -default `.libPaths()` masks it entirely. The render pipeline also tends to -swallow the underlying subprocess error, so the failure can be silent beyond -the bare package-load message. - -**Fix:** in the fixture's cwd, write a `.Rprofile` that re-points renv at the -real project, regardless of where the test's cwd actually is: +R resolves `.Rprofile` from the **exact** process cwd (no parent-directory search). +On CI, rmarkdown/knitr live only in `tests/renv`'s project library, activated when cwd is `tests/` (via `tests/.Rprofile` sourcing `renv/activate.R`). +Most knitr tests never leave `tests/` — they pass paths relative to the current cwd instead of changing directories — so activation happens automatically. + +A test that changes cwd through `TestContext.cwd()` loses that activation. The R subprocess starts outside `tests/`, and package loads may fail with `there is no package called 'rmarkdown'`. +A developer machine with rmarkdown on the default `.libPaths()` may mask this CI failure. + +**Fix:** write a `.Rprofile` in the fixture cwd that points renv to the test project: ```r Sys.setenv(RENV_PROJECT = "") source("/renv/activate.R") ``` -`renv/activate.R` reads `RENV_PROJECT` to determine the project root if set, -falling back to `getwd()` otherwise — setting it explicitly decouples renv -activation from the test's cwd. +`renv/activate.R` uses `RENV_PROJECT` as the project root instead of the current directory. ## Best Practices 1. **Always clean up**: Use teardown to remove generated files -2. **Use helpers**: Leverage `docs()`, `fileExists()`, etc. instead of manual checks +2. **Use helpers**: Leverage `docs()`, `fileExists()`, etc. + instead of manual checks 3. **Absolute paths**: Use `join()` for all path construction to handle platform differences 4. **Test isolation**: Use temp directories for tests that create files 5. **Clear names**: Use descriptive variable names like `projectDir`, `outputDir`, `templateFolder` @@ -439,23 +451,21 @@ activation from the test's cwd. ## Environment Variable Testing Pitfalls -`Deno.env.set()` modifies process-global state. Deno runs test files in parallel by default (same OS process), so concurrent tests can see modified values. Save/restore patterns don't help - other tests see the modified value during the test window. +`Deno.env.set()` modifies process-global state. +Deno runs test files in parallel by default (same OS process), so concurrent tests can see modified values. +Save/restore patterns don't help - other tests see the modified value during the test window. | Execution Mode | Risk | Why | | -------------------------- | ------------------ | --------------------------------------- | | `./run-tests.sh` (default) | **Race condition** | Files run in parallel, share `Deno.env` | | `./run-parallel-tests.sh` | **None** | Separate OS processes | -**Existing bad pattern** - `tests/smoke/website/drafts-env.test.ts`: +**Preferred approach:** pass per-test variables through `TestContext.env`. This works in both modes without mutating process-global state. -```typescript -// BAD: Sets env var, never restores it -// Only "works" because no other test reads QUARTO_PROFILE -Deno.env.set("QUARTO_PROFILE", "drafts"); -testQuartoCmd("render", [renderDir], [...]); -``` +**Exception:** `tests/smoke/website/drafts-env.test.ts` sets `QUARTO_PROFILE` at module load and in `context.env`. +Dev mode reads the module-level value when caching the base profile; binary mode receives the context value. -**Alternatives:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. +**Alternatives for new tests:** Unit test the env var reader, refactor code to accept parameters, or use subprocess isolation. ## Testing File Exclusion @@ -479,7 +489,8 @@ Run test **without fix** first to verify it fails, then verify it passes with fi ## Smoke-All Tests (YAML-Based) -Smoke-all tests embed test specifications directly in `.qmd` files using `_quarto.tests` metadata. See `.claude/rules/testing/smoke-all-tests.md` for full documentation. +Smoke-all tests embed test specifications directly in `.qmd` files using `_quarto.tests` metadata. +See `.claude/rules/testing/smoke-all-tests.md` for full documentation. ### YAML String Escaping for Regex @@ -511,33 +522,25 @@ _quarto: ### Probe enough keys to surface the bug -A precedence test where the template reads only the one key being -overridden can pass under a deep-merge bug. The dropped sibling keys -never resolve, but no assertion notices. +A precedence test where the template reads only the one key being overridden can pass under a deep-merge bug. +The dropped sibling keys never resolve, but no assertion notices. -Example: the merge of user-supplied `variables.quarto.language.crossref-ch-prefix: Bouquin` -onto Quarto's built `format.language` table under -`variables.quarto.language`. Under a shallow spread (`{ ...a, ...b }`), -`b.language` replaces the entire localized map — all other -`$quarto.language.$` resolutions silently return empty. A template -that reads only `$quarto.language.crossref-ch-prefix$` still asserts -"Bouquin", so the regression test passes. +Example: the merge of user-supplied `variables.quarto.language.crossref-ch-prefix: Bouquin` onto Quarto's built `format.language` table under `variables.quarto.language`. +Under a shallow spread (`{ ...a, ...b }`), `b.language` replaces the entire localized map — all other `$quarto.language.$` resolutions silently return empty. +A template that reads only `$quarto.language.crossref-ch-prefix$` still asserts "Bouquin", so the regression test passes. -The fix is to probe at least one non-overridden sibling key in the same -template. Concretely, the regression guard -`tests/docs/smoke-all/markdown/lang-fr-user-override-deep-merge.qmd` -uses the template +The fix is to probe at least one non-overridden sibling key in the same template. +Concretely, the regression guard `tests/docs/smoke-all/markdown/lang-fr-user-override-deep-merge.qmd` uses the template ``` $quarto.language.crossref-ch-prefix$|$quarto.language.toc-title-document$ ``` -and asserts the full string `^Bouquin\|Table des matières\s*$`. Pre-fix -the output was `Bouquin|`; post-fix it is `Bouquin|Table des matières`. +and asserts the full string `^Bouquin\|Table des matières\s*$`. +Pre-fix the output was `Bouquin|`; post-fix it is `Bouquin|Table des matières`. -Heuristic: when writing a precedence smoke test for any merge between -two structured config trees, ensure the assertion exercises at least -one path the user did NOT override. Otherwise the test only proves -"the overridden value wins" — not "the rest survives". +Heuristic: when writing a precedence smoke test for any merge between two structured config trees, ensure the assertion exercises at least one path the user did NOT override. +Otherwise the test only proves "the overridden value wins" — not "the rest survives". -**Recommendation:** Use single-quoted strings. They're simpler - only `'` itself needs escaping (as `''`). +**Recommendation:** Use single-quoted strings. +They're simpler - only `'` itself needs escaping (as `''`). diff --git a/tests/README.md b/tests/README.md index ef401eddaad..4ada5d9bc39 100644 --- a/tests/README.md +++ b/tests/README.md @@ -12,7 +12,8 @@ Tests are run in our CI workflow on GHA at each commit, and for each PR. ## How the tests are created and organized ? -Tests are running through `Deno.test()` framework, adapted for our Quarto project and all written in Typescript. Infrastructure are in `tests.ts`, `tests.deps.ts` `verify.ts` and `utils.ts` which contains the helper functions that can be used. +Tests use the `Deno.test()` framework with Quarto-specific helpers. +The test infrastructure is in `test.ts`, `test-deps.ts`, `quarto-cmd.ts`, `verify.ts`, and `utils.ts`. - `unit/` and `integration/`, `smoke/`contain some `.ts` script representing each tests. - `docs/` is a special folder containing of the necessary files and projects used for the tests. @@ -23,21 +24,25 @@ Tests are running through `Deno.test()` framework, adapted for our Quarto projec Here are what is expected in the environment for the tests : -- R should be installed and in PATH - [**rig**](https://github.com/r-lib/rig) is a good tool to manage R versions. e.g `rig install 4.4.2` and `rig default 4.4.2` to install and set the version to 4.4.2 +- R should be installed and on `PATH`. [**rig**](https://github.com/r-lib/rig) can manage R versions. + For example, run `rig install 4.4.2` and `rig default 4.4.2`. - On Windows, Rtools should be too (for source package installation) - Python should be installed and in PATH - [**pyenv**](https://github.com/pyenv/pyenv) is a good option to manage Python versions. - - On Windows, it will be [`pyenv-win`](https://pyenv-win.github.io/pyenv-win/) to manage versions. Otherwise or install from https://www.python.org/ manually or using `winget`. + - On Windows, use [`pyenv-win`](https://pyenv-win.github.io/pyenv-win/), install Python from , or use `winget`. - Julia should be installed and in PATH - [**juliaup**](https://github.com/JuliaLang/juliaup) is a good option to manage Julia versions. - On Windows, one way is using `winget install julia -s msstore` and then add `%LOCALAPPDATA%/Programs/Julia/bin` to PATH Running tests require to have a local environment setup with Quarto development, TinyTeX, R, Python and Julia. -To help with this configuration, the `tests/` folder contains `configure-test-env.sh` and `configure-test-env.ps1`. It will check for the tools and update the dependencies to what is used by Quarto tests. -Running the script at least one will insure you are correctly setup. Then, it is run as part of running the tests so that dependencies are always updated. Set `QUARTO_TESTS_NO_CONFIG` to skip this step when running tests. +The `tests/` folder contains `configure-test-env.sh` and `configure-test-env.ps1`. +These scripts check the required tools and synchronize test dependencies. +Run the appropriate script at least once. It also runs before tests by default. +Set `QUARTO_TESTS_NO_CONFIG` to skip this step when running tests. #### Optional test dependencies -The configure scripts also check for optional tools that some tests require. Tests will gracefully skip when these tools are not available, but having them installed enables full test coverage: +The configure scripts also check for optional tools that some tests require. +Tests will gracefully skip when these tools are not available, but having them installed enables full test coverage: **Java** (version 8, 11, 17, or 21) @@ -77,11 +82,15 @@ Dependencies are managed using the following tools: #### R -We use [**renv**](https://rstudio.github.io/renv/). `renv.lock` and `renv/` folders are the files used to recreate the environment for R. +We use [**renv**](https://rstudio.github.io/renv/). +The `renv.lock` file and `renv/` folder define the R environment. -Updating `renv.lock` is done using `renv::snapshot()`. File shouldn't be modified manually. +Updating `renv.lock` is done using `renv::snapshot()`. +Do not modify the file manually. -Our project is using [explicit dependencies discovery](https://rstudio.github.io/renv/reference/dependencies.html?q=dependen#explicit-dependencies) through a `DESCRIPTION` file. This is to avoid a costly scanning of all files in `tests/` to guess R dependencies. This means that if you need to add a test with a new R package dependencies: +The test project uses [explicit dependency discovery](https://rstudio.github.io/renv/reference/dependencies.html?q=dependen#explicit-dependencies) through a `DESCRIPTION` file. +This avoids scanning every file under `tests/` for R dependencies. +To add an R package dependency: - Add package(s) to `DESCRIPTION` in `tests/` - `renv::install()` the package into the project library @@ -91,26 +100,34 @@ Our project is using [explicit dependencies discovery](https://rstudio.github.io See [documentation](https://rstudio.github.io/renv/) if you need to tweak the R environment. -After a dependency update, you can run `configure-test-env.sh` or `configure-test-env.ps1` to update the environment, or manually run `renv::restore()` to recreate the environment with new versions. Be sure to update your R version if needed. +After a dependency update, you can run `configure-test-env.sh` or `configure-test-env.ps1` to update the environment, or manually run `renv::restore()` to recreate the environment with new versions. +Be sure to update your R version if needed. #### Python -We now use [**uv**](https://docs.astral.sh/uv) (previously, it was [**pipenv**](https://pipenv.pypa.io/en/latest/)) to manage dependencies and recreate easily on all OS. `uv` will **not** be installed as part of the configuration - so it needs to be installed manually - see various way at: https://docs.astral.sh/uv/getting-started/installation/ +We use [**uv**](https://docs.astral.sh/uv) to manage Python and its test dependencies on all platforms. +Install `uv` separately by following its [installation instructions](https://docs.astral.sh/uv/getting-started/installation/). -**uv** will handle the python versions, including its installation, based on the `.python-version` we have in `tests/` folder. It will also manage the virtual environment in `.venv` folder. +`uv` installs the Python version specified in `tests/.python-version` and manages the `.venv` virtual environment. -A virtual environment will be created locally in `.venv` folder (ignored on git) and activated when running tests. `uv run` can help activating the environment outside of running tests to run a command in the environment. +The test scripts activate the local `.venv`, which Git ignores. +Use `uv run` to run other commands in that environment. -`pyproject.toml` contains our dependencies requirement for the tests project. It can be manually updated but it is best to just use `uv` commands. For instance, adding a new dependency can be done with `uv add plotly` and it will update the file, update the `uv.lock` and install in the virtual environment. `uv.lock` should never be updated manually, and it is tracked by git, as it allows to recreate the exact environment on different environment (Linux, Mac, Windows, locally and on CI). +`pyproject.toml` defines the test project's Python dependencies. +Use commands such as `uv add plotly` to update `pyproject.toml`, update `uv.lock`, and install the package. +Do not edit `uv.lock` manually. Git tracks it so local and CI environments use the same dependency versions. See other [`uv` command](https://docs.astral.sh/uv/getting-started/features/) if you need to do more. -For a change of python versionn, `.python-version` needs to be updated, and then `uv` will take care of the rest. `configure-test-env` script will check for `uv` and if installed, it will called `uv sync` to make sure the project virtual environment is up to date with the lockfile. +To change the Python version, update `.python-version`. +The `configure-test-env` script runs `uv sync` when `uv` is installed. Note that `./run-test.ps1` and `.run-tests.sh` : - run `configure-test-env` script by default, unless `QUARTO_TESTS_NO_CONFIG` environment variable is set to a non-empty value. -- Activate the local virtualenv espected in `.venv`. Set `QUARTO_TESTS_FORCE_NO_VENV` to a non-empty value to prevent this behavior. (It replaces `QUARTO_TESTS_FORCE_NO_PIPENV` which still is considered for backward compatibility but deprecated) +- Activate the local virtual environment in `.venv`. + Set `QUARTO_TESTS_FORCE_NO_VENV` to a non-empty value to prevent this behavior. + The deprecated `QUARTO_TESTS_FORCE_NO_PIPENV` variable remains supported for compatibility. #### Julia @@ -118,7 +135,8 @@ Julia uses built-in package manager [**Pkg.jl**](https://pkgdocs.julialang.org/v `Project.toml` contains our direct dependency and `Manifest.toml` is the lock file that will be created (`Pkg.resolve()`). -**Important:** All test dependencies must be in the main `tests/` environment. Julia searches UP the directory tree for `Project.toml` starting from the document being rendered. +**Important:** All test dependencies must be in the main `tests/` environment. +Julia searches UP the directory tree for `Project.toml` starting from the document being rendered. **Adding a new package dependency:** @@ -128,9 +146,13 @@ julia --project=. -e 'using Pkg; Pkg.add("PackageName")' ./configure-test-env.sh # or .ps1 on Windows ``` -**Do NOT create** local `Project.toml` files in test subdirectories (e.g., `tests/docs/*/Project.toml`). Julia will use that environment instead of the main `tests/` environment. The `configure-test-env` scripts only manage the main environment, so tests with local environments will fail in CI even if they work locally. +**Do NOT create** local `Project.toml` files in test subdirectories (e.g., `tests/docs/*/Project.toml`). +Julia will use that environment instead of the main `tests/` environment. +The `configure-test-env` scripts only manage the main environment, so tests with local environments will fail in CI even if they work locally. -**Note:** This applies to ALL engines (Julia, Python, R). Python and R will also use local `.venv/` or `renv.lock` if present. The quarto-cli test infrastructure uses a single managed environment per language at `tests/`, and CI only configures these main environments. +**Note:** This applies to ALL engines (Julia, Python, R). +Python and R will also use local `.venv/` or `renv.lock` if present. +The quarto-cli test infrastructure uses a single managed environment per language at `tests/`, and CI only configures these main environments. See [documentation](https://pkgdocs.julialang.org/v1/managing-packages/) on how to add, remove, update if you need to tweak the Julia environment. @@ -198,7 +220,8 @@ For convenience, `run-fast-tests.sh` and `run-fast-tests.ps1` are provided to sk ./run-fast-tests.ps1 ``` -These scripts set `QUARTO_TESTS_NO_CONFIG` automatically. Use after running `configure-test-env` at least once. +These scripts set `QUARTO_TESTS_NO_CONFIG` automatically. +Use after running `configure-test-env` at least once. **QUARTO_TEST_KEEP_OUTPUTS** (or use `--keep-outputs`/`-k` flag) - Keep test output artifacts instead of cleaning them up @@ -230,7 +253,9 @@ $env:QUARTO_TEST_KEEP_OUTPUTS="true" #### About smoke-all tests -`docs/smoke-all/` is a specific folder to run some tests written directly within `.qmd`, `.md` or `.ipynb` files (but files starting with `_` will be ignored). They are run through the `smoke/smoke-all.tests.ts` script. To ease running smoke-all tests, `run-tests.sh` has a special behavior where it will run `./smoke/smoke-all.tests.ts` when passed a `.qmd`, `.md` or `.ipynb` file, not starting with `_`. +`docs/smoke-all/` contains tests defined in `.qmd`, `.md`, or `.ipynb` files. Files whose names start with `_` are ignored. +The `smoke/smoke-all.test.ts` script runs these tests. +When `run-tests.sh` receives a supported document path, it invokes that script automatically. ```bash # run tests for all documents in docs/smoke-all/ @@ -406,9 +431,10 @@ When a snapshot test fails: ### Limitations -- `smoke-all.test.ts` accept only one argument. You need to use glob pattern to run several smoke-all test documents. +- `smoke-all.test.ts` accepts only one argument. Use a glob to run several smoke-all documents. -- Individual `smoke-all` tests and other test can't be run at the same time with `run-test.[sh|ps1]`. This is because `smoke-all.test.ts` requires arguments. If a smoke-all document and another smoke-test are passed as argument, the smoke-all test will be prioritize and other will be ignored (with a warning). +- A `smoke-all` document and another test cannot run in the same `run-tests.[sh|ps1]` invocation. + The smoke-all document takes precedence, and the script warns that it ignored the other test. Example with Linux: @@ -430,9 +456,55 @@ Don't do ./run-tests.sh smoke/extensions/extension-render-doc.test.ts smoke/smoke-all.test.ts -- ./docs/smoke-all/2023/01/04/issue-3847.qmd ``` +### Binary mode (`QUARTO_TEST_BIN`) + +Tests normally run Quarto in-process from the dev sources. +Set `QUARTO_TEST_BIN` to an installed Quarto to run commands against that binary instead. +See [Built-Version Testing Architecture](../llm-docs/built-version-testing-architecture.md) for the harness and CI design. + +To run in binary mode locally: + +```bash +# 1. Build a distribution (after ./configure.sh) +cd package/src +./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" +cd ../.. + +# 2. Copy the distribution outside the checkout. An in-repo launcher uses +# the dev sources when it finds a sibling src/quarto.ts. The test scripts +# reject that launcher because it reports the 99.9.9 dev version. +cp -r package/pkg-working ~/quarto-under-test + +# 3. Run the tests against it +cd tests +QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh +``` + +In binary mode: + +- With no arguments, `run-tests.[sh|ps1]` runs `smoke/`. + Unit tests remain dev-only. + Playwright and feature-format tests support binary mode but must be passed explicitly: + + ```bash + # playwright suite against a built quarto + QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh integration/playwright-tests.test.ts + # feature-format matrix against a built quarto + QUARTO_TEST_BIN=~/quarto-under-test/bin/quarto ./run-tests.sh "../dev-docs/feature-format-matrix/qmd-files/**/*.qmd" + ``` + +- The test environment is configured as usual; set `QUARTO_TESTS_NO_CONFIG` to skip that step as in dev mode. +- Tests with `requiresDevQuarto: true` in their `TestContext` are ignored. + +Authoring rules that keep tests working in both modes: + +- Invoke Quarto through `testQuartoCmd()` or `runQuarto()`; do not import `quarto` from `src/quarto.ts`. +- For direct subprocesses, resolve the executable with `quartoDevCmd()` or `quartoDevBinCmd()` and pass `quartoSpawnEnvOptions()`. + ## Debugging within tests -`.vscode/launch.json` has a `Run Quarto test` configuration that can be used to debug when running tests. One need to modify the `program` and `args` fields to match the test to run. +`.vscode/launch.json` has a `Run Quarto test` configuration. +Set its `program` and `args` fields for the test you want to debug. Example: @@ -449,9 +521,9 @@ _Short version can't be use here as we are calling `deno test` directly and not This lives in `run-parallel-tests.ts` and called through `run-parallel-tests.sh`. -### How does is works ? +### How it works -- It requires a text file with tested timed and following a specific format. (Default is `timing.txt` and here is an example [in our repo](./timing.txt)) +- It requires a timing file in the format shown in [`timing.txt`](./timing.txt). - Based on this file, the tests will be split in buckets to minimize the tests time (buckets are filled by their minimum overall time). - Then `./run-tests.sh` will be run for each bucket from deno using `Promise.all()` and `run-tests.sh` on the whole bucket's test files, so that the buckets are ran in parallel. @@ -468,9 +540,11 @@ QUARTO_TEST_TIMING='timing.txt' ./run-tests.sh When this is done, any other argument will be ignored, and the following happens - All the `*.test.ts` file are found and run individually using `/usr/bin/time` to store timing in the file -- When `smoke-all.test.ts` is found, all the `*.qmd`, `*.md` and `*.ipynb` in `docs/smoke-all/` not starting with `_` are found and run individually using same logic. This means each `smoke-all` test is timed. +- When `smoke-all.test.ts` is found, all the `*.qmd`, `*.md` and `*.ipynb` in `docs/smoke-all/` not starting with `_` are found and run individually using same logic. + This means each `smoke-all` test is timed. -The results is written in the `$QUARTO_TEST_TIMING` file. Here is an example: +The results are written to the `$QUARTO_TEST_TIMING` file. +Here is an example: ``` ./smoke/directives/include-fixups.test.ts @@ -491,7 +565,8 @@ This will be read by `run-parallel-tests.ts` to get the `real` value and fill th #### Specific behavior for `smoke-all.test.ts` -`smoke-all` tests are special because they are in the form of individual `.qmd` or `.ipynb` document that needs to be run using `smoke-all.test.ts` script, with arguments. Unfortunately, this prevent running individual `smoke-all` documents in same buclets as other individual smoke test (which are their own `.test.ts` file). +Each `smoke-all` test is a `.qmd` or `.ipynb` document passed as an argument to `smoke-all.test.ts`. +The local parallel runner cannot place these documents in the same buckets as standalone `.test.ts` files. So, if the timed file contains some individual timing for `smoke-all` documents like this @@ -499,16 +574,20 @@ So, if the timed file contains some individual timing for `smoke-all` documents ./smoke/smoke-all.test.ts -- docs/smoke-all/2022/12/12/code-annotation.qmd ``` -then they are ignored and `.smoke-all.test.ts` will be run in its own bucket. It will usually be the longest test run. +then they are ignored and `.smoke-all.test.ts` will be run in its own bucket. +It will usually be the longest test run. -Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tests on GHA CI as the buckets are split into their own runners and each test in a bucket if run using `run-test.sh`. This allows a bucket to contains some `*.test.ts` but also some document `*.qmd` or `*.ipynb`. More details in [test-smoke.yml](.github/workflows/test-smokes.yml) and [test-smokes-parallel.yml](.github/workflows/test-smokes-parallel.yml) +CI uses individual `smoke-all` timings when assigning tests to runner buckets. A CI bucket can contain both `*.test.ts` files and test documents. +See `test-smokes.yml` and `test-smokes-parallel.yml`. ### Arguments that control behavior -- `-n=`: Number of buckets to create to run in parallel. `run-parallel-tests.sh -n=5` split tests in 5 buckets and run them at the same time. For local run, `n` should be a number of core. For CI run, `n` will be the number of runners to use at the same time (mulplied by 2 because Linux and Windows are ran on CI). -- `--verbose`: show some verbosity. Otherwise, no specific logging in console in done. -- `--dry-run`: show the buckets of tests, but do not run. Otherwise, they are run. -- `--timing-file=`: Which file to use as timed tests information to creates the buckets. (default to `timing.txt` ). `run-parallel-tests.sh --timing-file='timing2.txt'` will use `timing2.txt` to run the file. +- `-n=`: Number of buckets to create to run in parallel. + `run-parallel-tests.sh -n=5` creates five concurrent buckets. + For local runs, use the number of available cores. In CI, this is the number of runners per operating system. +- `--verbose`: Show detailed console output. +- `--dry-run`: Show the buckets without running them. +- `--timing-file=`: Select the timing file. The default is `timing.txt`. - `--json-for-ci`: Special flag to trigger splitting tests in buckets for the parallel run on CI and that makes `run-parallel-tests.sh` outputs JSON string specifically formatted for GHA processing. ### About tests in CI with GHA @@ -516,6 +595,18 @@ Individual `smoke-all` tests timing are useful for Quarto parallelized smoke tes - `test-smokes-parallel.yml` will be triggered to load `timing-for-ci.txt` and split tests in buckets. It will create a matrix to trigger `test-smokes.yml` on `workflow_call` event for each bucket. - PR against main and commits to main will trigger this workflow, and tests will be ran in parallel jobs. - A `workflow_dispatch` event can be used to trigger it through API call, `gh` CLI tool or GHA GUI online. -- `test-smokes.yml` is the main CI workflow which configure the environment, and run the tests on Ubuntu and Windows. - - If it was triggerred by `workflow_call`, then it will run each test in using `run-tests.[sh|ps1]` in a for-loop. +- `test-smokes.yml` is the main CI workflow for configuring the environment and running tests on Ubuntu and Windows. + - When called through `workflow_call`, it runs each bucket with `run-tests.[sh|ps1]`. - Scheduled tests are still run daily in their sequential version. + - Callers can use its install, version, artifact, ref, and runner inputs to test a built Quarto. +- `test-smokes-built.yml` runs smoke, Playwright, and feature-format legs + against a built Quarto. It runs after nightly builds and supports manual + dispatches: + + | Mode | Trigger | Use it to answer | + |---|---|---| + | `nightly` | automatic after create-release; dispatch with `run-id` to retest an older run | Does the packaged nightly build pass on each available OS? | + | `build` | dispatch (default) | Does this ref work when packaged as a Linux amd64 distribution? | + | `release` | dispatch | Does the published release pass? | + + Full rationale and design decisions: `llm-docs/built-version-testing-architecture.md`. diff --git a/tests/docs/convert/issue-12318.qmd b/tests/docs/convert/issue-12318.qmd new file mode 100644 index 00000000000..8859466ef00 --- /dev/null +++ b/tests/docs/convert/issue-12318.qmd @@ -0,0 +1,28 @@ +--- +title: Quarto Crossrefs +format: html +jupyter: python3 +categories: [plot, python, equation] +--- + +## Overview + +See @fig-simple in @sec-plot for a demonstration of a simple plot. + +See @eq-stddev to better understand standard deviation. + +## Plot {#sec-plot} + +```{python} +#| label: fig-simple +#| fig-cap: "Simple Plot" +import matplotlib.pyplot as plt +plt.plot([1,23,2,4]) +plt.show() +``` + +## Equation {#sec-equation} + +$$ +s = \sqrt{\frac{1}{N-1} \sum_{i=1}^N (x_i - \overline{x})^2} +$$ {#eq-stddev} diff --git a/tests/docs/crossrefs/editor-support/all.qmd b/tests/docs/crossrefs/editor-support/all.qmd new file mode 100644 index 00000000000..b027c14e316 --- /dev/null +++ b/tests/docs/crossrefs/editor-support/all.qmd @@ -0,0 +1,26 @@ +--- +title: All crossref types +--- + +## Introduction {#sec-intro} + +See @sec-intro for the overview, @fig-plot for the figure, @tbl-data for the +table, @eq-var for the equation, and @thm-line for the theorem. + +![A simple plot.](plot.png){#fig-plot} + +| Column A | Column B | +|----------|----------| +| 1 | 2 | + +: A small table. {#tbl-data} + +$$ +\sigma^2 = \frac{1}{N} \sum_{i=1}^N (x_i - \mu)^2 +$$ {#eq-var} + +::: {#thm-line} +## Line + +The equation of a straight line is $y = mx + b$. +::: diff --git a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml index eb86e8b6bf2..1a6f1d67ea7 100644 --- a/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml +++ b/tests/docs/smoke-all/2023/01/06/input-relative/test/_extensions/test/_extension.yml @@ -1,7 +1,7 @@ title: Test author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: project: project: diff --git a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/2023/04/24/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore b/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore index 92d902f2733..43c566bbdf3 100644 --- a/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore +++ b/tests/docs/smoke-all/2024/10/23/issue-10829/.gitignore @@ -1,2 +1,3 @@ /.quarto/ -_site/ \ No newline at end of file +_site/ +**/*.quarto_ipynb diff --git a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml index 2d5882388ab..66571285084 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension-github/_extensions/my-org/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml index 2d5882388ab..66571285084 100644 --- a/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/logo/logo-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My-brand author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml index 859925eab79..c95827710de 100644 --- a/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml +++ b/tests/docs/smoke-all/brand/typography/remote-font-extension/_extensions/my-brand/_extension.yml @@ -1,7 +1,7 @@ title: My Brand author: Quarto version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/dashboard/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/format/html/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml index b9d9fd11d27..c4db023f92a 100644 --- a/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml +++ b/tests/docs/smoke-all/lightbox/_extensions/dragonstyle/lipsum/_extension.yml @@ -1,7 +1,7 @@ title: Lipsum author: Charles Teague version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: shortcodes: - lipsum.lua diff --git a/tests/docs/smoke-all/listings/image-lazy-loading/.gitignore b/tests/docs/smoke-all/listings/image-lazy-loading/.gitignore index 075b2542afb..13c72f9a057 100644 --- a/tests/docs/smoke-all/listings/image-lazy-loading/.gitignore +++ b/tests/docs/smoke-all/listings/image-lazy-loading/.gitignore @@ -1 +1,4 @@ /.quarto/ +blog/index-*.html +/search.json +/site_libs/ diff --git a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml index 106a96726ca..4bc546fbf56 100644 --- a/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml +++ b/tests/docs/smoke-all/typst/brand-yaml/typography/brand-extension/_extensions/typst-brand-typography-example/_extension.yml @@ -1,7 +1,7 @@ title: Typst-brand-typography-example author: Gordon Woodhull version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: metadata: project: diff --git a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml index f11d93da35b..bbc62e89a3c 100644 --- a/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml +++ b/tests/docs/smoke-all/typst/font-paths/extension-font-paths/_extensions/font-provider/_extension.yml @@ -1,7 +1,7 @@ title: Font Provider Extension author: Test version: 1.0.0 -quarto-required: ">=99.9.0" +quarto-required: ">=1.9" contributes: formats: typst: diff --git a/tests/integration/playwright-tests.test.ts b/tests/integration/playwright-tests.test.ts index 4318d9836c0..dbf191095b3 100644 --- a/tests/integration/playwright-tests.test.ts +++ b/tests/integration/playwright-tests.test.ts @@ -1,5 +1,5 @@ /* - * smoke-all.test.ts + * playwright-tests.test.ts * * Copyright (C) 2022 Posit Software, PBC * @@ -13,6 +13,7 @@ import { } from "../../src/core/lib/yaml-validation/state.ts"; import { cleanoutput } from "../smoke/render/render.ts"; import { execProcess } from "../../src/core/process.ts"; +import { quartoSpawnEnvOptions } from "../quarto-cmd.ts"; import { quartoDevCmd } from "../utils.ts"; import { fail } from "testing/asserts"; import { isWindows } from "../../src/deno_ral/platform.ts"; @@ -72,11 +73,13 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { // mediabag inspection if we don't wait all renders // individually. This is very slow.. console.log(`Rendering ${input}...`); + // Prevent a built Quarto from inheriting dev-tree paths. const result = await execProcess({ cmd: quartoDevCmd(), args: ["render", input, ...options], stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); if (!result.success) { @@ -91,8 +94,8 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { } Deno.test({ - name: "Playwright tests are passing", - // currently we run playwright tests only on Linux + name: "Playwright tests are passing", + // Windows CI renders the inputs but does not run browser assertions. ignore: gha.isGitHubActions() && isWindows, fn: async () => { try { diff --git a/tests/quarto-cmd.ts b/tests/quarto-cmd.ts new file mode 100644 index 00000000000..62e7d784344 --- /dev/null +++ b/tests/quarto-cmd.ts @@ -0,0 +1,426 @@ +/* + * quarto-cmd.ts + * + * Single dispatch point for invoking the quarto under test. + * + * Runs the dev sources in-process or QUARTO_TEST_BIN as a subprocess. + * See llm-docs/built-version-testing-architecture.md. + * + * Copyright (C) 2020-2026 Posit Software, PBC + */ +import { quarto } from "../src/quarto.ts"; +import { kLocalDevelopment } from "../src/core/quarto.ts"; +import { isWindows } from "../src/deno_ral/platform.ts"; +import { join } from "../src/deno_ral/path.ts"; + +// Strip dev-tree and logging state from built-binary spawns. Other ambient +// variables are inherited, then the per-test environment is applied. +const kStripEnvVars = [ + "QUARTO_SHARE_PATH", + "QUARTO_BIN_PATH", + "QUARTO_DEBUG", + "DENO_DIR", + "QUARTO_DENO", + "QUARTO_DENO_DOM", + "QUARTO_ROOT", + "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION", + "QUARTO_VERSION_REQUIREMENT", + "QUARTO_PROJECT_DIR", + "QUARTO_PROFILE", + "QUARTO_LOG", + "QUARTO_LOG_LEVEL", + "QUARTO_LOG_FORMAT", + "RSTUDIO", +]; + +// std/log LogLevels.ERROR, as expected by readExecuteOutput(). +const kErrorLevel = 40; + +// Default per-invocation render timeout (dev and binary mode alike). +const kDefaultRenderTimeoutMs = 600000; + +export function quartoTestBin(): string | undefined { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + return bin && bin.length > 0 ? bin : undefined; +} + +export function isBinaryMode(): boolean { + return quartoTestBin() !== undefined; +} + +// Use QUARTO_TEST_BIN in binary mode; otherwise pin the local dev CLI. +export function quartoDevBinCmd(): string { + const bin = quartoTestBin(); + if (bin) { + return bin; + } + const binPath = Deno.env.get("QUARTO_BIN_PATH") ?? + join("..", "package", "dist", "bin"); + return join(binPath, isWindows ? "quarto.cmd" : "quarto"); +} + +export function buildBinaryEnv( + overlay?: Record, +): Record { + const env = Deno.env.toObject(); + for (const name of kStripEnvVars) { + delete env[name]; + } + return { ...env, ...(overlay ?? {}) }; +} + +// Sanitize direct subprocess spawns in binary mode. Dev-mode spawns inherit +// the ambient environment and apply only the requested overlay. +export function quartoSpawnEnvOptions( + overlay?: Record, +): { env?: Record; clearEnv?: boolean } { + if (isBinaryMode()) { + return { env: buildBinaryEnv(overlay), clearEnv: true }; + } + return overlay !== undefined ? { env: overlay } : {}; +} + +// Appends a synthetic ERROR record to a json-stream log file. Only call +// after the child process has exited (single-writer at that point). +export function appendLogError(logFile: string, msg: string) { + const record = JSON.stringify({ + msg, + level: kErrorLevel, + levelName: "ERROR", + }); + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // file may not exist yet + } + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + record + "\n"); +} + +// A timeout can interrupt a log write. Remove only the torn trailing record +// so readExecuteOutput() can remain strict. +function stripTornTrailingLine(content: string): string { + const lines = content.split("\n"); + let i = lines.length - 1; + while (i >= 0 && lines[i] === "") { + i--; + } + if (i < 0) { + return content; + } + try { + JSON.parse(lines[i]); + return content; + } catch { + lines.splice(i, 1); + return lines.join("\n"); + } +} + +function hasErrorRecordText(content: string): boolean { + for (const line of content.split("\n")) { + if (!line) continue; + try { + const record = JSON.parse(line); + if ( + typeof record?.levelName === "string" && + record.levelName.toLowerCase() === "error" + ) { + return true; + } + } catch { + // tolerate partial/corrupt lines + } + } + return false; +} + +// Reject an in-checkout launcher, which reports the 99.9.9 dev sentinel. +let checkedBinary: string | undefined; +export function assertTestBinary(bin: string) { + if (checkedBinary === bin) { + return; + } + // Probe with the same sanitized environment used by test spawns. + const result = new Deno.Command(bin, { + args: ["--version"], + stdout: "piped", + stderr: "piped", + env: buildBinaryEnv(), + clearEnv: true, + }).outputSync(); + const version = new TextDecoder().decode(result.stdout).trim(); + if (result.code !== 0) { + const stderr = new TextDecoder().decode(result.stderr).trim(); + throw new Error( + `QUARTO_TEST_BIN (${bin}) failed to report a version (exit ${result.code}):\n${stderr}`, + ); + } + if (version.length === 0) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reported an empty version. ` + + `The distribution is likely incomplete (missing share/version).`, + ); + } + if (version === kLocalDevelopment) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports the dev version sentinel ${kLocalDevelopment}. ` + + `It is resolving to a dev-mode quarto (the launcher runs the TS sources when a ` + + `sibling src/quarto.ts exists). Point QUARTO_TEST_BIN at a built distribution ` + + `extracted outside the git checkout.`, + ); + } + const expected = Deno.env.get("QUARTO_TEST_EXPECTED_VERSION"); + if (expected && version !== expected) { + throw new Error( + `QUARTO_TEST_BIN (${bin}) reports version ${version}, expected ${expected} ` + + `(QUARTO_TEST_EXPECTED_VERSION).`, + ); + } + console.log(`[binary mode] testing quarto ${version} at ${bin}`); + checkedBinary = bin; +} + +// The launcher waits on Deno, so kill the process tree deepest first. +async function killProcessTree(pid: number) { + if (isWindows) { + let killed = false; + try { + // taskkill reports failure through its exit code. + const result = await new Deno.Command("taskkill", { + args: ["/PID", String(pid), "/T", "/F"], + stdout: "null", + stderr: "null", + }).output(); + killed = result.code === 0; + } catch { + // Fall through to a direct kill. + } + if (!killed) { + // Ensure child.output() can resolve even if the tree kill failed. + try { + Deno.kill(pid, "SIGKILL"); + } catch { + // already exited + } + } + return; + } + const pids: number[] = []; + const stack = [pid]; + while (stack.length > 0) { + const current = stack.pop()!; + pids.push(current); + try { + // pgrep -P works on Linux and macOS/BSD. + const result = new Deno.Command("pgrep", { + args: ["-P", String(current)], + stdout: "piped", + stderr: "null", + }).outputSync(); + const children = new TextDecoder() + .decode(result.stdout) + .split("\n") + .map((line) => parseInt(line.trim(), 10)) + .filter((child) => !isNaN(child)); + stack.push(...children); + } catch { + // pgrep unavailable; fall back to killing what we have + } + } + for (const target of pids.reverse()) { + try { + Deno.kill(target, "SIGKILL"); + } catch { + // already exited + } + } +} + +export interface RunQuartoOptions { + // Per-test environment overlay. + env?: Record; + // Binary-mode working directory. + cwd?: string; + // Binary-mode log target and options. + logFile?: string; + logLevel?: string; + logFormat?: string; + timeoutMs?: number; + // Binary mode only. Defaults to true; testQuartoCmd disables it so + // verifiers receive failures through log records. + throwOnFailure?: boolean; +} + +export interface RunQuartoResult { + // Present only for binary-mode spawns. Dev mode has no exit code - it + // reports failure by rejecting instead (see runDevQuarto). + code?: number; + timedOut: boolean; + stderrTail?: string; +} + +// Dispatch to the in-process dev sources or the configured built binary. +export async function runQuarto( + args: string[], + options: RunQuartoOptions = {}, +): Promise { + const bin = quartoTestBin(); + return bin + ? runBinaryQuarto(bin, args, options) + : runDevQuarto(args, options); +} + +// A dev-mode timeout rejects but cannot stop the in-process render. +async function runDevQuarto( + args: string[], + options: RunQuartoOptions, +): Promise { + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + let timer: ReturnType | undefined; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(reject, timeoutMs, `timed out after ${timeoutMs}ms`); + }); + try { + await Promise.race([quarto(args, undefined, options.env), timeout]); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + } + // quarto() either resolves or rejects: on CommandError or commandFailed() + // it calls exitWithCleanup(1), which Deno.exits the whole test process + // before this function could return a failure code anyway. + return { timedOut: false }; +} + +// Spawn the built binary and enforce timeout, logging, and failure policy. +async function runBinaryQuarto( + bin: string, + args: string[], + options: RunQuartoOptions, +): Promise { + assertTestBinary(bin); + const timeoutMs = options.timeoutMs ?? kDefaultRenderTimeoutMs; + const throwOnFailure = options.throwOnFailure ?? true; + + // LogFileHandler truncates its target, so each child writes a temporary + // log that is merged into the test log after exit. + const spawnArgs = [...args]; + let childLog: string | undefined; + if (options.logFile) { + childLog = Deno.makeTempFileSync({ suffix: ".json" }); + spawnArgs.push( + "--log", + childLog, + "--log-format", + options.logFormat ?? "json-stream", + // per-test log intent must land in the flags: explicit flags beat + // QUARTO_LOG_LEVEL env in quarto's logOptions, so passing the env + // var through would be silently ignored + "--log-level", + options.logLevel ?? options.env?.QUARTO_LOG_LEVEL ?? "info", + ); + } + + const child = new Deno.Command(bin, { + args: spawnArgs, + cwd: options.cwd ?? Deno.cwd(), + env: buildBinaryEnv(options.env), + clearEnv: true, + stdout: "piped", + stderr: "piped", + }).spawn(); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + // child.output() resolves after the kill; avoid an unhandled rejection. + killProcessTree(child.pid).catch(() => {}); + }, timeoutMs); + + // Drain both streams to avoid pipe-buffer deadlocks. + const output = await child.output(); + clearTimeout(timer); + + const stderrText = new TextDecoder().decode(output.stderr); + const stderrTail = stderrText.split("\n").slice(-25).join("\n").trim(); + const commandLine = `quarto ${args.join(" ")}`; + + if (options.logFile && childLog) { + mergeChildLog(options.logFile, childLog, { + timedOut, + code: output.code, + timeoutMs, + commandLine, + stderrTail, + }); + } + + if ((output.code !== 0 || timedOut) && throwOnFailure) { + throw new Error( + timedOut + ? `${commandLine} timed out after ${timeoutMs}ms` + : `${commandLine} exited with code ${output.code}\nstderr (tail):\n${stderrTail}`, + ); + } + + return { code: output.code, timedOut, stderrTail }; +} + +// Merge the child log and synthesize an ERROR when a failed child logged none. +function mergeChildLog( + logFile: string, + childLog: string, + outcome: { + timedOut: boolean; + code: number; + timeoutMs: number; + commandLine: string; + stderrTail: string; + }, +) { + let childContent = ""; + try { + childContent = Deno.readTextFileSync(childLog); + } catch { + // child never wrote the log (e.g. failed before logger init) + } + try { + Deno.removeSync(childLog); + } catch { + // best effort + } + // Only a timeout kill can tear a line; a clean exit flushes whole records. + if (outcome.timedOut) { + childContent = stripTornTrailingLine(childContent); + } + // A quiet successful command still needs an empty log for its verifiers. + let existing = ""; + try { + existing = Deno.readTextFileSync(logFile); + } catch { + // log file may not exist yet + } + if (childContent.length > 0) { + const sep = existing.length === 0 || existing.endsWith("\n") ? "" : "\n"; + Deno.writeTextFileSync(logFile, existing + sep + childContent); + } else if (existing.length === 0) { + Deno.writeTextFileSync(logFile, ""); + } + if (outcome.timedOut) { + appendLogError( + logFile, + `${outcome.commandLine} timed out after ${outcome.timeoutMs}ms and was killed`, + ); + } else if (outcome.code !== 0 && !hasErrorRecordText(childContent)) { + // Startup and commandFailed paths can exit without logging an error. + appendLogError( + logFile, + `${outcome.commandLine} exited with code ${outcome.code} without logging an error\n` + + `stderr (tail):\n${outcome.stderrTail}`, + ); + } +} diff --git a/tests/run-tests.ps1 b/tests/run-tests.ps1 index f0f3ef77f7c..8effaf97948 100644 --- a/tests/run-tests.ps1 +++ b/tests/run-tests.ps1 @@ -66,6 +66,52 @@ If ($null -eq $Env:QUARTO_DENO_DIR) { $Env:DENO_DIR = $Env:QUARTO_DENO_DIR } +# QUARTO_TEST_BIN selects an installed Quarto outside this checkout. +# The harness still uses the dev runtime configured above. +If (-not [string]::IsNullOrEmpty($Env:QUARTO_TEST_BIN)) { + If (-not (Test-Path $Env:QUARTO_TEST_BIN)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) does not exist" + Exit 1 + } + # Strip dev paths while probing the installed binary. + $probeStrip = @( + "QUARTO_SHARE_PATH", "QUARTO_BIN_PATH", "QUARTO_DEBUG", "DENO_DIR", + "QUARTO_DENO", "QUARTO_DENO_DOM", "QUARTO_ROOT", "QUARTO_SRC_PATH", + "QUARTO_FORCE_VERSION" + ) + $probeSaved = @{} + ForEach ($name in $probeStrip) { + $probeSaved[$name] = [Environment]::GetEnvironmentVariable($name) + Remove-Item "Env:$name" -ErrorAction SilentlyContinue + } + Try { + $QUARTO_TEST_BIN_VERSION = & $Env:QUARTO_TEST_BIN --version + $QUARTO_TEST_BIN_PROBE_EXIT = $LASTEXITCODE + } Finally { + ForEach ($name in $probeStrip) { + If ($null -ne $probeSaved[$name]) { + [Environment]::SetEnvironmentVariable($name, $probeSaved[$name]) + } + } + } + If ($QUARTO_TEST_BIN_PROBE_EXIT -ne 0) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) exited with code $QUARTO_TEST_BIN_PROBE_EXIT while reporting its version." + Exit 1 + } + If ([string]::IsNullOrWhiteSpace($QUARTO_TEST_BIN_VERSION)) { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN ($($Env:QUARTO_TEST_BIN)) did not report a version." + Write-Host -ForegroundColor red "The distribution is likely incomplete (missing share/version)." + Exit 1 + } + If ($QUARTO_TEST_BIN_VERSION -eq "99.9.9") { + Write-Host -ForegroundColor red "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + Write-Host -ForegroundColor red "The selected launcher runs the dev sources because it has a sibling src/quarto.ts." + Write-Host -ForegroundColor red "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + Exit 1 + } + Write-Host "> BINARY MODE: testing built quarto $QUARTO_TEST_BIN_VERSION at $($Env:QUARTO_TEST_BIN)" +} + # Preparing running Deno with default arguments $QUARTO_IMPORT_MAP_ARG="--importmap=$(Join-Path $QUARTO_SRC_DIR "import_map.json")" @@ -161,6 +207,12 @@ If ($customArgs[0] -notlike "*smoke-all.test.ts") { $TESTS_TO_RUN=$customArgs } +# Binary mode defaults to smoke tests; other compatible suites are explicit. +If (-not [string]::IsNullOrEmpty($Env:QUARTO_TEST_BIN) -and $TESTS_TO_RUN.count -eq 0 -and $customArgs.count -eq 0) { + $TESTS_TO_RUN = @("smoke/") + Write-Host "> BINARY MODE: defaulting to smoke/ tests (pass a path explicitly to run others, e.g. integration/playwright-tests.test.ts)" +} + # ---- Running tests with Deno ------- $DENO_ARGS = @() diff --git a/tests/run-tests.sh b/tests/run-tests.sh index a2c134fd392..70db95098cb 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -62,6 +62,36 @@ export QUARTO_DEBUG=true QUARTO_DENO_OPTIONS="--config test-conf.json --v8-flags=--enable-experimental-regexp-engine,--max-old-space-size=8192,--max-heap-size=8192 --unstable-kv --unstable-ffi --no-lock --allow-all" +# QUARTO_TEST_BIN selects an installed Quarto outside this checkout. +# The harness still uses the dev runtime configured above. +if [[ -n "$QUARTO_TEST_BIN" ]]; then + if [[ ! -x "$QUARTO_TEST_BIN" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) does not exist or is not executable" + exit 1 + fi + # Strip dev paths while probing the installed binary. + QUARTO_TEST_BIN_VERSION="$(env -u QUARTO_SHARE_PATH -u QUARTO_BIN_PATH \ + -u QUARTO_DEBUG -u DENO_DIR -u QUARTO_DENO -u QUARTO_DENO_DOM \ + -u QUARTO_ROOT -u QUARTO_SRC_PATH -u QUARTO_FORCE_VERSION \ + "$QUARTO_TEST_BIN" --version 2>/dev/null)" + QUARTO_TEST_BIN_PROBE_EXIT=$? + if [[ $QUARTO_TEST_BIN_PROBE_EXIT -ne 0 ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) exited with code $QUARTO_TEST_BIN_PROBE_EXIT while reporting its version." + exit 1 + fi + if [[ -z "$QUARTO_TEST_BIN_VERSION" ]]; then + echo "ERROR: QUARTO_TEST_BIN ($QUARTO_TEST_BIN) did not report a version." + echo "The distribution is likely incomplete (missing share/version)." + exit 1 + fi + if [[ "$QUARTO_TEST_BIN_VERSION" == "99.9.9" ]]; then + echo "ERROR: QUARTO_TEST_BIN reports the dev version sentinel 99.9.9." + echo "The selected launcher runs the dev sources because it has a sibling src/quarto.ts." + echo "Point QUARTO_TEST_BIN at a built distribution extracted outside the git checkout." + exit 1 + fi + echo "> BINARY MODE: testing built quarto ${QUARTO_TEST_BIN_VERSION} at ${QUARTO_TEST_BIN}" +fi if [[ -z $GITHUB_ACTION ]] && [[ -z $QUARTO_TESTS_NO_CONFIG ]] then @@ -161,6 +191,11 @@ else TESTS_TO_RUN=("${SMOKE_ALL_TEST_FILE}" "--" "${SMOKE_ALL_FILES[@]}") fi fi + # Binary mode defaults to smoke tests; other compatible suites are explicit. + if [[ -n "$QUARTO_TEST_BIN" && "${#TESTS_TO_RUN[@]}" -eq 0 && -z "$*" ]]; then + TESTS_TO_RUN=("smoke/") + echo "> BINARY MODE: defaulting to smoke/ tests (pass a path explicitly to run others, e.g. integration/playwright-tests.test.ts)" + fi # TESTS_TO_RUN is an array and quoted here on purpose: a bucket can be a # literal, unexpanded ** glob pattern (e.g. from the ff-matrix CI bucket), # and smoke-all.test.ts expands it itself via expandGlobSync. Expanding it diff --git a/tests/smoke/axe/axe-exit-codes.test.ts b/tests/smoke/axe/axe-exit-codes.test.ts index 5f98b4b05c4..ebadd870c0e 100644 --- a/tests/smoke/axe/axe-exit-codes.test.ts +++ b/tests/smoke/axe/axe-exit-codes.test.ts @@ -19,22 +19,11 @@ import { assert, assertEquals } from "testing/asserts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; -import { isWindows } from "../../../src/deno_ral/platform.ts"; import { execProcess } from "../../../src/core/process.ts"; import { unitTest } from "../../test.ts"; -import { quartoDevCmd } from "../../utils.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { AxeFindings } from "../../../src/command/call/axe/schemas.ts"; -// Under run-tests.sh/.ps1 the dev binary's dir is exported; resolve it -// explicitly so the subprocess never falls through to a release quarto that -// happens to be on PATH. -function quartoBin(): string { - const binPath = Deno.env.get("QUARTO_BIN_PATH"); - return binPath - ? join(binPath, isWindows ? "quarto.cmd" : "quarto") - : quartoDevCmd(); -} - const workingDir = Deno.makeTempDirSync({ prefix: "quarto-axe-exit" }); Deno.mkdirSync(join(workingDir, "site")); Deno.writeTextFileSync( @@ -61,10 +50,10 @@ function axeExitTest( ) { unitTest(name, async () => { const result = await execProcess({ - cmd: quartoBin(), + cmd: quartoDevBinCmd(), args: ["call", "axe", "site", ...args], cwd: workingDir, - env, + ...quartoSpawnEnvOptions(env), stdout: "piped", stderr: "piped", }); @@ -175,10 +164,10 @@ unitTest( Deno.writeTextFileSync(staleFindings, `{"stale": true}`); const result = await execProcess({ - cmd: quartoBin(), + cmd: quartoDevBinCmd(), args: ["call", "axe", "site"], cwd: workingDir, - env: { QUARTO_CHROMIUM: notABrowser }, + ...quartoSpawnEnvOptions({ QUARTO_CHROMIUM: notABrowser }), stdout: "piped", stderr: "piped", }); diff --git a/tests/smoke/axe/shared.ts b/tests/smoke/axe/shared.ts index 80d8ece0c51..0a82f159d14 100644 --- a/tests/smoke/axe/shared.ts +++ b/tests/smoke/axe/shared.ts @@ -16,7 +16,7 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { ExecuteOutput, testQuartoCmd, Verify } from "../../test.ts"; import { fileExists, validJsonWithFields } from "../../verify.ts"; import { docs } from "../../utils.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { AxeCell } from "../../../src/command/call/axe/scan.ts"; import { AxeFinding, @@ -158,7 +158,7 @@ export function axeSmokeTest( { cwd: () => siteDir, setup: async () => { - await quarto(["render"]); + await runQuarto(["render"]); }, teardown: () => { for (const dir of ["_site", "_axe-checks"]) { diff --git a/tests/smoke/convert/issue-12318.test.ts b/tests/smoke/convert/issue-12318.test.ts index 13d95b9e06c..426845b5bd9 100644 --- a/tests/smoke/convert/issue-12318.test.ts +++ b/tests/smoke/convert/issue-12318.test.ts @@ -11,32 +11,51 @@ import { test, } from "../../test.ts"; import { assert } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; (() => { const input = "docs/convert/issue-12318"; + const roundtrip = "issue-12318-2.qmd"; test({ // The name of the test name: "issue-12318", - + // Sets up the test context: { teardown: async () => { if (existsSync(input + '.ipynb')) { Deno.removeSync(input + '.ipynb'); } + if (existsSync(roundtrip)) { + Deno.removeSync(roundtrip); + } } }, - + // Executes the test - execute: async () => { - await quarto(["convert", "docs/convert/issue-12318.qmd"]); - await quarto(["convert", "docs/convert/issue-12318.ipynb", "--output", "issue-12318-2.qmd"]); - const txt = Deno.readTextFileSync("issue-12318-2.qmd"); - assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + execute: async (logFile?: string) => { + await runQuarto(["convert", input + ".qmd"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["convert", input + ".ipynb", "--output", roundtrip], { + logFile, + throwOnFailure: false, + }); }, - - verify: [], + + verify: [ + noErrors, + { + name: "no triple backticks mid-line after roundtrip", + verify: (_outputs: ExecuteOutput[]) => { + const txt = Deno.readTextFileSync(roundtrip); + assert(!txt.includes('}```'), "Triple backticks found not at beginning of line"); + return Promise.resolve(); + }, + }, + ], type: "unit" }); })(); diff --git a/tests/smoke/create/create.test.ts b/tests/smoke/create/create.test.ts index dc685b9d492..1c49bd1ecfc 100644 --- a/tests/smoke/create/create.test.ts +++ b/tests/smoke/create/create.test.ts @@ -11,6 +11,7 @@ import { walkSync } from "../../../src/deno_ral/fs.ts"; import { CreateResult } from "../../../src/command/create/cmd-types.ts"; import { assert } from "testing/asserts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; const kCreateTypes: Record = { "project": ["website", "default", "book", "website:blog"], @@ -54,6 +55,7 @@ for (const type of Object.keys(kCreateTypes)) { args: cmd.slice(1), stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }, stdIn); assert(process.success, process.stderr); if (process.stdout) { @@ -62,11 +64,7 @@ for (const type of Object.keys(kCreateTypes)) { assert(process.success, process.stderr); }); - // Verify all created files are user-writable. - // NOTE: In dev environments, resource files are already writable (0o644), - // so this test passes even without ensureUserWritable. It guards against - // regressions; the unit test in file-permissions.test.ts covers the - // read-only → writable transition directly. + // Verify that files in the generated project are user-writable. await t.step({ name: `> check writable ${type} ${template}`, ignore: Deno.build.os === "windows", @@ -101,6 +99,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(buildProcess.success, buildProcess.stderr); } @@ -115,6 +114,7 @@ for (const type of Object.keys(kCreateTypes)) { cwd: path, stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); assert(process.success, process.stderr); } diff --git a/tests/smoke/crossref/syntax.test.ts b/tests/smoke/crossref/syntax.test.ts index 8d4f913d718..5e1fc1f1983 100644 --- a/tests/smoke/crossref/syntax.test.ts +++ b/tests/smoke/crossref/syntax.test.ts @@ -4,7 +4,7 @@ * Copyright (C) 2020-2022 Posit Software, PBC */ -import { ensureFileRegexMatches } from "../../verify.ts"; +import { ensureFileRegexMatches, noErrors } from "../../verify.ts"; import { testRender } from "../render/render.ts"; import { crossref } from "./utils.ts"; import { @@ -15,7 +15,8 @@ import { Verify, } from "../../test.ts"; import { assert, fail } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; +import { safeRemoveSync } from "../../../src/deno_ral/fs.ts"; const syntaxQmd = crossref("syntax.qmd", "html"); testRender(syntaxQmd.input, "html", false, [ @@ -58,22 +59,30 @@ const verify: Verify = { }; const context: TestContext = { teardown: () => { - Deno.removeSync(imgQmd.output.outputPath); - Deno.removeSync(imgQmd.output.supportPath, { recursive: true }); + // Missing outputs must not let teardown mask the render failure. + safeRemoveSync(imgQmd.output.outputPath); + safeRemoveSync(imgQmd.output.supportPath, { recursive: true }); - Deno.removeSync(divQmd.output.outputPath); - Deno.removeSync(divQmd.output.supportPath, { recursive: true }); + safeRemoveSync(divQmd.output.outputPath); + safeRemoveSync(divQmd.output.supportPath, { recursive: true }); return Promise.resolve(); }, }; const testDesc: TestDescriptor = { // FIXME: why is this test flaky now? Ask @dragonstyle name: "test html produced by different figure syntax", context, - execute: async () => { - await quarto(["render", imgQmd.input]); - await quarto(["render", divQmd.input]); + execute: async (logFile?: string) => { + // Report render errors before comparing output files. + await runQuarto(["render", imgQmd.input], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", divQmd.input], { + logFile, + throwOnFailure: false, + }); }, - verify: [verify], + verify: [noErrors, verify], type: "smoke", }; test(testDesc); diff --git a/tests/smoke/engine/invalid-engine-in-project.test.ts b/tests/smoke/engine/invalid-engine-in-project.test.ts index 4b7081e94f8..782dbd764f5 100644 --- a/tests/smoke/engine/invalid-engine-in-project.test.ts +++ b/tests/smoke/engine/invalid-engine-in-project.test.ts @@ -1,19 +1,16 @@ -import { assertRejects } from "testing/asserts"; -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; +import { testQuartoCmd } from "../../test.ts"; +import { printsMessage } from "../../verify.ts"; -test( - { - name: "invalid engines option errors", - execute: async () => { - assertRejects( - async () => {await quarto(["render", "docs/engine/invalid-project/notebook.qmd"])}, - Error, - "'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine", - ) - }, - type: "smoke", - context: {}, - verify: [], - } -) \ No newline at end of file +testQuartoCmd( + "render", + ["docs/engine/invalid-project/notebook.qmd"], + [ + printsMessage({ + level: "ERROR", + regex: + /'invalid-engine' was specified in the list of engines in the project settings but it is not a valid engine/, + }), + ], + {}, + "invalid engines option errors", +); diff --git a/tests/smoke/env/check.test.ts b/tests/smoke/env/check.test.ts index 9125b71d9c9..d0b41d09899 100644 --- a/tests/smoke/env/check.test.ts +++ b/tests/smoke/env/check.test.ts @@ -5,13 +5,20 @@ * */ import { testQuartoCmd } from "../../test.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { noErrorsOrWarnings, printsMessage } from "../../verify.ts"; +// Dev mode reports the 99.9.9 sentinel version; a built binary reports its +// real version, so only require a semver-shaped version line there. +const versionRegex = isBinaryMode() + ? /Version: \d+\.\d+\.\d+/ + : /Version: 99\.9\.9/; + testQuartoCmd( "check", [], [ noErrorsOrWarnings, - printsMessage({level: "INFO", regex: /Version: 99\.9\.9/}), + printsMessage({level: "INFO", regex: versionRegex}), ], ); diff --git a/tests/smoke/extensions/extension-render-journals.test.ts b/tests/smoke/extensions/extension-render-journals.test.ts index 0327048fb64..9137c9f9819 100644 --- a/tests/smoke/extensions/extension-render-journals.test.ts +++ b/tests/smoke/extensions/extension-render-journals.test.ts @@ -5,10 +5,11 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const journalRepos = [ // { repo: "acm", noSupporting: true }, TODO this format needs changes after this merge. @@ -42,15 +43,14 @@ for (const journalRepo of journalRepos) { // Sets up the test setup: async () => { console.log(`using quarto-journals/${journalRepo.repo}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await quarto([ - "use", - "template", - `quarto-journals/${journalRepo.repo}`, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + `quarto-journals/${journalRepo.repo}`, + "--no-prompt", + ]); + }); }, // Cleans up the test diff --git a/tests/smoke/extensions/extension-render-typst-templates.test.ts b/tests/smoke/extensions/extension-render-typst-templates.test.ts index 5acdbe91487..1d62194c5b7 100644 --- a/tests/smoke/extensions/extension-render-typst-templates.test.ts +++ b/tests/smoke/extensions/extension-render-typst-templates.test.ts @@ -5,10 +5,11 @@ */ import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ensureDirSync, existsSync } from "../../../src/deno_ral/fs.ts"; import { testRender } from "../render/render.ts"; import { removeIfEmptyDir } from "../../../src/core/path.ts"; +import { withCwd } from "../../utils.ts"; const GITHUB_REPO = "quarto-ext/typst-templates"; @@ -40,15 +41,14 @@ for (const name of typstTemplates) { setup: async () => { const source = `${GITHUB_REPO}/${name}`; console.log(`using template: ${source}`); - const wd = Deno.cwd(); - Deno.chdir(workingDir); - await quarto([ - "use", - "template", - source, - "--no-prompt", - ]); - Deno.chdir(wd); + await withCwd(workingDir, async () => { + await runQuarto([ + "use", + "template", + source, + "--no-prompt", + ]); + }); }, teardown: async () => { diff --git a/tests/smoke/filters/editor-support.test.ts b/tests/smoke/filters/editor-support.test.ts index 77729c882e1..51e76f2691b 100644 --- a/tests/smoke/filters/editor-support.test.ts +++ b/tests/smoke/filters/editor-support.test.ts @@ -5,20 +5,17 @@ */ import { docs } from "../../utils.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { assertEquals } from "testing/asserts"; -import { isWindows } from "../../../src/deno_ral/platform.ts"; async function runEditorSupportCrossref(doc: string) { - const cmdLine: string = isWindows ? - "../package/dist/bin/quarto.cmd" : - "../package/dist/bin/quarto"; - - const cmd = new Deno.Command(cmdLine, { + const cmd = new Deno.Command(quartoDevBinCmd(), { args: ["editor-support", "crossref"], stdin: "piped", stdout: "piped", stderr: "piped", + ...quartoSpawnEnvOptions(), }); const child = cmd.spawn(); const writer = child.stdin.getWriter(); @@ -26,7 +23,7 @@ async function runEditorSupportCrossref(doc: string) { Deno.readTextFileSync(doc), ); await writer.write(buf); - writer.releaseLock(); + // close() sends EOF; releaseLock() would detach the writer before close(). await writer.close(); const outputBuf = await child.output(); const status = await child.status; @@ -39,12 +36,17 @@ async function runEditorSupportCrossref(doc: string) { test({ name: "editor-support:crossref:smoke-1", context: {}, - execute: async () => { - const json = await runEditorSupportCrossref(docs("crossrefs/sections.qmd")); - assertEquals(json.entries[0].key, "sec-introduction"); - assertEquals(json.entries[0].caption, "Introduction"); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref output", + verify: async (_outputs) => { + const json = await runEditorSupportCrossref( + docs("crossrefs/sections.qmd"), + ); + assertEquals(json.entries[0].key, "sec-introduction"); + assertEquals(json.entries[0].caption, "Introduction"); + }, + }], type: "smoke", }); @@ -52,10 +54,13 @@ function smokeTestCrossref(name: string, doc: string) { test({ name, context: {}, - execute: async () => { - await runEditorSupportCrossref(doc); - }, - verify: [], + execute: async () => {}, + verify: [{ + name: "editor-support crossref runs cleanly", + verify: async (_outputs) => { + await runEditorSupportCrossref(doc); + }, + }], type: "smoke", }); } diff --git a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts index 2f26608179b..b47758665ee 100644 --- a/tests/smoke/inspect/inspect-standalone-rstudio.test.ts +++ b/tests/smoke/inspect/inspect-standalone-rstudio.test.ts @@ -7,15 +7,15 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { _setIsRStudioForTest } from "../../../src/core/platform.ts"; +import { isBinaryMode } from "../../quarto-cmd.ts"; import { ExecuteOutput, testQuartoCmd, } from "../../test.ts"; import { assert, assertEquals } from "testing/asserts"; -// Test: standalone file inspect with RStudio override should NOT emit project. -// Uses _setIsRStudioForTest to avoid Deno.env.set() race conditions in -// parallel tests (see #14218, PR #12621). +// Dev mode uses the test hook to avoid process-global environment races. +// Binary mode passes RSTUDIO=1 to the spawned Quarto. (() => { const input = "docs/inspect/standalone-hello.qmd"; const output = "docs/inspect/standalone-hello.json"; @@ -34,11 +34,16 @@ import { assert, assertEquals } from "testing/asserts"; } ], { + env: isBinaryMode() ? { RSTUDIO: "1" } : undefined, setup: async () => { - _setIsRStudioForTest(true); + if (!isBinaryMode()) { + _setIsRStudioForTest(true); + } }, teardown: async () => { - _setIsRStudioForTest(undefined); + if (!isBinaryMode()) { + _setIsRStudioForTest(undefined); + } if (existsSync(output)) { Deno.removeSync(output); } diff --git a/tests/smoke/issues/9133/9133.test.ts b/tests/smoke/issues/9133/9133.test.ts index d0b21db04a0..c1df612c02b 100644 --- a/tests/smoke/issues/9133/9133.test.ts +++ b/tests/smoke/issues/9133/9133.test.ts @@ -1,23 +1,19 @@ -import { quarto } from "../../../../src/quarto.ts"; +import { runQuarto } from "../../../quarto-cmd.ts"; import { test } from "../../../test.ts"; if (Deno.build.os !== "windows") { test({ - name: "https://github.com/quarto-dev/quarto-cli/issues/9133", + name: "https://github.com/quarto-dev/quarto-cli/issues/9133", context: { setup: async () => { Deno.mkdirSync("smoke/issues/9133/oh'\"no", { recursive: true }); Deno.copyFileSync("smoke/issues/9133/jl", "smoke/issues/9133/oh'\"no/jl.qmd"); Deno.copyFileSync("smoke/issues/9133/py", "smoke/issues/9133/oh'\"no/py.qmd"); - const timeout = new Promise((_resolve, reject) => { - setTimeout(reject, 600000, "timed out after 10 minutes"); - }); - await Promise.race([ - Promise.all([ - quarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), - quarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), - ]), - timeout, + // Binary mode uses separate processes, so the in-process race may not + // reproduce there. Avoid a shared log for concurrent children. + await Promise.all([ + runQuarto(["render", "smoke/issues/9133/oh'\"no/jl.qmd"]), + runQuarto(["render", "smoke/issues/9133/oh'\"no/py.qmd"]), ]); } }, diff --git a/tests/smoke/jupyter/cache.test.ts b/tests/smoke/jupyter/cache.test.ts index 6a687778aab..c2f2b006247 100644 --- a/tests/smoke/jupyter/cache.test.ts +++ b/tests/smoke/jupyter/cache.test.ts @@ -4,10 +4,10 @@ * Copyright (C) 2023 Posit Software, PBC */ import { dirname, join } from "path"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; import { docs } from "../../utils.ts"; -import { folderExists, printsMessage } from "../../verify.ts"; +import { folderExists, noErrors, printsMessage } from "../../verify.ts"; import { fileLoader } from "../../utils.ts"; import { safeExistsSync, safeRemoveSync } from "../../../src/core/path.ts"; @@ -16,14 +16,17 @@ const cacheFolder = join(dirname(testInput.input), ".jupyter_cache") test({ name: "Jupyter cache is working", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"]); + // Render twice to exercise the cache. + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -39,6 +42,7 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder), // this will check only for the second render that should be read from cache printsMessage({ level: "INFO", regex: /Notebook read from cache/}) @@ -54,14 +58,17 @@ const cacheFolder2 = join(dirname(testInput2.input), ".cache/jupyter-cache") test({ name: "Jupyter cache folder can be change", - execute: async () => { - // return await new Promise((_resolve, reject) => { - // setTimeout(reject, 10000, "timed out after 10 seconds"); - // }) + execute: async (logFile?: string) => { // https://github.com/quarto-dev/quarto-cli/issues/9618 - // repeated executions to trigger jupyter cache - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); - await quarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"]); + // Render twice to exercise the cache. + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); + await runQuarto(["render", testInput2.input, "--to", "html", "--no-execute-daemon"], { + logFile, + throwOnFailure: false, + }); }, context: { teardown: async () => { @@ -80,9 +87,10 @@ test({ } }, verify: [ + noErrors, folderExists(cacheFolder2), // this will check only for the second render that should be read from cache printsMessage({level: "INFO", regex: /Notebook read from cache/}) ], type: "smoke", -}); \ No newline at end of file +}); diff --git a/tests/smoke/jupyter/issue-10097.test.ts b/tests/smoke/jupyter/issue-10097.test.ts index 19586200e28..743e237a76f 100644 --- a/tests/smoke/jupyter/issue-10097.test.ts +++ b/tests/smoke/jupyter/issue-10097.test.ts @@ -1,26 +1,25 @@ /* * parameter-label-duplication.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/10097 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:parameter:label-duplication", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/10097 - await quarto(["render", - "docs/jupyter/parameters/issue-10097.qmd", - "--execute-param", 'datapath:"weird"', - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/10097 +testQuartoCmd( + "render", + [ + "docs/jupyter/parameters/issue-10097.qmd", + "--execute-param", + 'datapath:"weird"', + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:parameter:label-duplication", +); diff --git a/tests/smoke/jupyter/issue-12374.test.ts b/tests/smoke/jupyter/issue-12374.test.ts index 56c5ce67dc0..3007adb3499 100644 --- a/tests/smoke/jupyter/issue-12374.test.ts +++ b/tests/smoke/jupyter/issue-12374.test.ts @@ -1,25 +1,23 @@ /* * issue-12374.test.ts - * + * * https://github.com/quarto-dev/quarto-cli/issues/12374 * * Copyright (C) 2023 Posit Software, PBC */ -import { quarto } from "../../../src/quarto.ts"; -import { test } from "../../test.ts"; -import { assertEquals } from "testing/asserts"; +import { testQuartoCmd } from "../../test.ts"; import { noErrors } from "../../verify.ts"; -test({ - name: "jupyter:issue-12374.test.ts", - context: {}, - execute: async () => { - // https://github.com/quarto-dev/quarto-cli/issues/12374 - await quarto(["render", - "docs/jupyter/issue-12374.ipynb", - "--no-execute-daemon", "--execute"]); - }, - verify: [noErrors], - type: "smoke", -}); +// https://github.com/quarto-dev/quarto-cli/issues/12374 +testQuartoCmd( + "render", + [ + "docs/jupyter/issue-12374.ipynb", + "--no-execute-daemon", + "--execute", + ], + [noErrors], + {}, + "jupyter:issue-12374.test.ts", +); diff --git a/tests/smoke/logging/log-level-and-formats.test.ts b/tests/smoke/logging/log-level-and-formats.test.ts index 9c2bbf703d0..68957613228 100644 --- a/tests/smoke/logging/log-level-and-formats.test.ts +++ b/tests/smoke/logging/log-level-and-formats.test.ts @@ -10,6 +10,7 @@ import { execProcess } from "../../../src/core/process.ts"; import { md5HashSync } from "../../../src/core/hash.ts"; import { safeRemoveIfExists } from "../../../src/core/path.ts"; import { quartoDevCmd, outputForInput } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { assert } from "testing/asserts"; import { LogFormat } from "../../../src/core/log.ts"; import { existsSync } from "../../../src/deno_ral/fs.ts"; @@ -90,7 +91,8 @@ function testLogDirectly(options: { cmd: quartoDevCmd(), args: args, stdout: "piped", - stderr: "piped" + stderr: "piped", + ...quartoSpawnEnvOptions(), }); // Get stdout/stderr with fallback to empty string @@ -137,35 +139,31 @@ function testLogDirectly(options: { } - // If JSON format is specified, verify the output is valid JSON + // Catch parsing errors only; assertion failures must propagate. if (logFile && options.format === "json-stream") { assert(existsSync(logFile), "Log file should exist"); - let foundValidJson = false; + let outputs; try { - const outputs = readExecuteOutput(logFile); - foundValidJson = true; - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldNotContainLevel?.includes(out.levelName)).forEach( - (out) => { - assert(false, `JSON output should not contain level ${out.levelName}, but found: ${out.msg}`); - } + outputs = readExecuteOutput(logFile); + } catch { + outputs = undefined; + } + assert(outputs !== undefined, "JSON format should produce valid JSON output"); + const records = outputs!.filter((out) => out.msg !== ""); + const levels = new Set(records.map((out) => out.levelName)); + for (const lvl of options.expectedOutputs?.shouldNotContainLevel ?? []) { + const offending = records.find((out) => out.levelName === lvl); + assert( + offending === undefined, + `JSON log should not contain level ${lvl}, but found: ${offending?.msg}` ); - outputs.filter((out) => out.msg !== "" && options.expectedOutputs?.shouldContainLevel?.includes(out.levelName)).forEach( - (out) => { - let json = undefined; - try { - json = JSON.parse(out.msg); - } catch { - assert(false, "Error parsing JSON returned by quarto meta"); - } - assert( - Object.keys(json).length > 0, - "JSON returned by quarto meta seems invalid", - ); - } + } + for (const lvl of options.expectedOutputs?.shouldContainLevel ?? []) { + assert( + levels.has(lvl), + `JSON log should contain at least one ${lvl} record; found levels: ${[...levels].join(", ") || "(none)"}` ); - - } catch (e) {} - assert(foundValidJson, "JSON format should produce valid JSON output"); + } } } finally { // Clean up log file if it exists diff --git a/tests/smoke/lua-unit/lua-unit.test.ts b/tests/smoke/lua-unit/lua-unit.test.ts index 5a4f61341b9..7715146d5ae 100644 --- a/tests/smoke/lua-unit/lua-unit.test.ts +++ b/tests/smoke/lua-unit/lua-unit.test.ts @@ -21,6 +21,7 @@ import { fromFileUrl, join } from "../../../src/deno_ral/path.ts"; import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; // Explicit list, relative to tests/unit-lua/. Keep alphabetized. @@ -52,7 +53,7 @@ for (const relPath of LUA_TESTS) { { cmd: quartoDevCmd(), args: ["run", luaScript], - env: { LUA_PATH }, + ...quartoSpawnEnvOptions({ LUA_PATH }), }, undefined, undefined, diff --git a/tests/smoke/project/project-prepost.test.ts b/tests/smoke/project/project-prepost.test.ts index b7895f4a5f1..334e174b84a 100644 --- a/tests/smoke/project/project-prepost.test.ts +++ b/tests/smoke/project/project-prepost.test.ts @@ -63,18 +63,21 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/extension")], - [{ - name: "i-exist.txt exists", + [noErrors, { + name: "prepost extension file effects", verify: async () => { - const path = join(docs("project/prepost/extension"), "i-exist.txt"); - verifyNoPath(path); + verifyNoPath(join(docs("project/prepost/extension"), "i-exist.txt")); + verifyPath(join(docs("project/prepost/extension"), "i-was-created.txt")); } }], { + // Ensure verification covers files created by this render. + setup: async () => { + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-exist.txt")); + }, teardown: async () => { - const path = join(docs("project/prepost/extension"), "i-was-created.txt"); - verifyPath(path); - safeRemoveIfExists(path); + safeRemoveIfExists(join(docs("project/prepost/extension"), "i-was-created.txt")); const siteDir = join(docs("project/prepost/extension"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); @@ -85,19 +88,26 @@ testQuartoCmd( testQuartoCmd( "render", [docs("project/prepost/issue-10828")], - [], + [noErrors, { + name: "project input/output files written", + verify: async () => { + verifyPath(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + verifyPath(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + } + }], { env: { "QUARTO_USE_FILE_FOR_PROJECT_INPUT_FILES": normalizePath(docs("project/prepost/issue-10828/input-files.txt")), "QUARTO_USE_FILE_FOR_PROJECT_OUTPUT_FILES": normalizePath(docs("project/prepost/issue-10828/output-files.txt")) }, + // Ensure verification covers files created by this render. + setup: async () => { + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); + }, teardown: async () => { - const inputPath = normalizePath(docs("project/prepost/issue-10828/input-files.txt")); - const outputPath = normalizePath(docs("project/prepost/issue-10828/output-files.txt")); - verifyPath(inputPath); - safeRemoveIfExists(inputPath); - verifyPath(outputPath); - safeRemoveIfExists(outputPath); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/input-files.txt"))); + safeRemoveIfExists(normalizePath(docs("project/prepost/issue-10828/output-files.txt"))); const siteDir = join(docs("project/prepost/issue-10828"), "_site"); if (existsSync(siteDir)) { await Deno.remove(siteDir, { recursive: true }); diff --git a/tests/smoke/project/project-stdout.test.ts b/tests/smoke/project/project-stdout.test.ts index dfa38d86299..b0539af5bb0 100644 --- a/tests/smoke/project/project-stdout.test.ts +++ b/tests/smoke/project/project-stdout.test.ts @@ -14,6 +14,7 @@ import { docs } from "../../utils.ts"; import { directoryEmptyButFor, fileExists, + noErrors, verifyYamlFile, } from "../../verify.ts"; @@ -30,7 +31,7 @@ const siteOutDir = join(siteProjDir, outDir); testQuartoCmd( "render", [siteProjDir, "-o", "-"], - [], + [noErrors], { teardown: async () => { if (existsSync(siteOutDir)) { diff --git a/tests/smoke/render/render-format-extension.test.ts b/tests/smoke/render/render-format-extension.test.ts index f82374cf32f..09eaa5ab758 100644 --- a/tests/smoke/render/render-format-extension.test.ts +++ b/tests/smoke/render/render-format-extension.test.ts @@ -12,8 +12,8 @@ // Both files serve different purposes and should remain separate. import { safeRemoveSync } from "../../../src/core/path.ts"; -import { docs } from "../../utils.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { docs, withCwd } from "../../utils.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { testRender } from "./render.ts"; @@ -28,19 +28,16 @@ import { testRender } from "./render.ts"; const updateExtensions = async () => { try { console.log("Updating quarto-journals extensions to latest versions..."); - const wd = Deno.cwd(); - Deno.chdir(docs("extensions/format/academic")); - - for (const repo of ["acs", "elsevier"]) { - await quarto([ - "update", - "extension", - `quarto-journals/${repo}`, - "--no-prompt", - ]); - } - - Deno.chdir(wd); + await withCwd(docs("extensions/format/academic"), async () => { + for (const repo of ["acs", "elsevier"]) { + await runQuarto([ + "update", + "extension", + `quarto-journals/${repo}`, + "--no-prompt", + ]); + } + }); console.log("Extensions updated successfully"); } catch (error) { const message = error instanceof Error ? error.message : String(error); diff --git a/tests/smoke/render/render-freeze.test.ts b/tests/smoke/render/render-freeze.test.ts index 7291e24d798..82979a58444 100644 --- a/tests/smoke/render/render-freeze.test.ts +++ b/tests/smoke/render/render-freeze.test.ts @@ -9,10 +9,10 @@ import { dirname, join } from "../../../src/deno_ral/path.ts"; import { assert } from "testing/asserts"; import { Metadata } from "../../../src/config/types.ts"; -import { removeIfEmptyDir } from "../../../src/core/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { removeIfEmptyDir, safeRemoveIfExists } from "../../../src/core/path.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { ExecuteOutput, Verify } from "../../test.ts"; -import { outputCreated } from "../../verify.ts"; +import { noErrors, outputCreated } from "../../verify.ts"; import { testRender } from "./render.ts"; const regex = /output file: .*\.knit\.md/m; @@ -67,9 +67,8 @@ const ignoreFrozen = { const projectOutputExists: Verify = { name: "Make sure project output exists", - verify: (_output: ExecuteOutput[]) => { - outputCreated(path, "html"); - return Promise.resolve(); + verify: (output: ExecuteOutput[]) => { + return outputCreated(path, "html").verify(output); }, }; @@ -107,16 +106,16 @@ function testFileContext( markdown, ); - await quarto(["render", path]); + await runQuarto(["render", path]); }, teardown: async () => { // Clean up the test file await Deno.remove(path); await Deno.remove(quartoProj); - // Get rid of the freezer + // Remove freeze output if the render created it. const freezerDir = join(dirname(path), "_freeze"); - Deno.removeSync(join(freezerDir, testFileName), { recursive: true }); + safeRemoveIfExists(join(freezerDir, testFileName)); // Maybe clean up empty freeze dir removeIfEmptyDir(freezerDir); @@ -130,7 +129,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "clean fzr - auto", ...testContext, @@ -142,7 +141,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, ignoreFrozen], + [noErrors, projectOutputExists, ignoreFrozen], { name: "dirty fzr - auto", setup: async () => { @@ -167,7 +166,7 @@ testRender( dirname(path) + "/", "html", false, - [projectOutputExists, useFrozen], + [noErrors, projectOutputExists, useFrozen], { name: "dirty fzr - freeze", setup: async () => { diff --git a/tests/smoke/render/render-output-file-collision.test.ts b/tests/smoke/render/render-output-file-collision.test.ts index f360ed0d54e..0b802f536f8 100644 --- a/tests/smoke/render/render-output-file-collision.test.ts +++ b/tests/smoke/render/render-output-file-collision.test.ts @@ -14,7 +14,7 @@ */ import { existsSync, safeRemoveSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { docs } from "../../utils.ts"; import { testQuartoCmd } from "../../test.ts"; import { @@ -103,7 +103,7 @@ testQuartoCmd( { setup: async () => { await cleanup(defaultDir, [...defaultOutputs, ".quarto"])(); - await quarto(["render", defaultDir]); + await runQuarto(["render", defaultDir]); }, teardown: cleanup(defaultDir, [...defaultOutputs, ".quarto"]), }, diff --git a/tests/smoke/run/command-passthrough.test.ts b/tests/smoke/run/command-passthrough.test.ts index 6db94e984e9..462580d896c 100644 --- a/tests/smoke/run/command-passthrough.test.ts +++ b/tests/smoke/run/command-passthrough.test.ts @@ -1,16 +1,18 @@ import { assert } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; const testPassthroughCmd = (name: string, command: string, args: string[]) => { unitTest(name, async () => { const result = await execProcess({ - cmd: quartoDevCmd(), + cmd: quartoDevCmd(), args: [ command, ...args, - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }); diff --git a/tests/smoke/run/run-script.test.ts b/tests/smoke/run/run-script.test.ts index ee6fb0f16fe..308a465ffce 100644 --- a/tests/smoke/run/run-script.test.ts +++ b/tests/smoke/run/run-script.test.ts @@ -3,6 +3,7 @@ import { ensureDirSync } from "../../../src/deno_ral/fs.ts"; import { assert, assertEquals } from "testing/asserts"; import { execProcess } from "../../../src/core/process.ts"; import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { unitTest } from "../../test.ts"; import { EOL } from "fs/eol"; import { lines } from "../../../src/core/text.ts"; @@ -20,9 +21,9 @@ const ensureStreams = (name: string, script: string, stdout: string, stderr: str basename(script), ], // disable logging here to allow for checking the output - env: { + ...quartoSpawnEnvOptions({ "QUARTO_LOG_LEVEL": "CRITICAL", - } + }), }, undefined, undefined, @@ -55,7 +56,8 @@ const testRunCmd = (name: string, script: string) => { args: [ "run", basename(script), - ] + ], + ...quartoSpawnEnvOptions(), }); assert(result.success); }, diff --git a/tests/smoke/run/stdlib-run-version.test.ts b/tests/smoke/run/stdlib-run-version.test.ts index bbe7dadf726..eef54b05648 100644 --- a/tests/smoke/run/stdlib-run-version.test.ts +++ b/tests/smoke/run/stdlib-run-version.test.ts @@ -8,15 +8,18 @@ import { execProcess } from "../../../src/core/process.ts"; import { assert } from "testing/asserts"; import { unitTest } from "../../test.ts"; +import { quartoDevCmd } from "../../utils.ts"; +import { quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; import { isWindows } from "../../../src/deno_ral/platform.ts"; unitTest("stdlib-run-version", async () => { const result = await execProcess({ - cmd: "quarto", + cmd: quartoDevCmd(), args: [ "run", "docs/run/test-stdlib.ts", ], + ...quartoSpawnEnvOptions(), }); console.log({result}) assert(result.success); diff --git a/tests/smoke/self-contained/stdout.test.ts b/tests/smoke/self-contained/stdout.test.ts index f03a8e6c730..b58533a49c9 100644 --- a/tests/smoke/self-contained/stdout.test.ts +++ b/tests/smoke/self-contained/stdout.test.ts @@ -1,11 +1,11 @@ -import { quarto } from "../../../src/quarto.ts"; +import { runQuarto } from "../../quarto-cmd.ts"; import { test } from "../../test.ts"; test({ name: "https://github.com/quarto-dev/quarto-cli/issues/11068", context: { setup: async() => { - await quarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); + await runQuarto(["render", "docs/self-contained/simple.qmd", "-o", "-"]); } }, execute: async () => {}, diff --git a/tests/smoke/site/render-site-themes.test.ts b/tests/smoke/site/render-site-themes.test.ts index e12fa17c25a..e8cd0cd0830 100644 --- a/tests/smoke/site/render-site-themes.test.ts +++ b/tests/smoke/site/render-site-themes.test.ts @@ -5,7 +5,7 @@ * */ -import { copySync } from "../../../src/deno_ral/fs.ts"; +import { copySync, safeRemoveSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { testQuartoCmd, Verify } from "../../test.ts"; import { docs } from "../../utils.ts"; @@ -75,8 +75,8 @@ try { return Promise.resolve(); }, teardown: () => { - // clean output - Deno.removeSync(path, { recursive: true }); + // Remove output if setup copied the fixture. + safeRemoveSync(path, { recursive: true }); return Promise.resolve(); }, }, diff --git a/tests/smoke/smoke-all.test.ts b/tests/smoke/smoke-all.test.ts index 8e1a973a840..2082afa844c 100644 --- a/tests/smoke/smoke-all.test.ts +++ b/tests/smoke/smoke-all.test.ts @@ -57,7 +57,7 @@ import { findProjectDir, findProjectOutputDir, outputForInput } from "../utils.t import { jupyterNotebookToMarkdown } from "../../src/command/convert/jupyter.ts"; import { basename, dirname, join, relative } from "../../src/deno_ral/path.ts"; import { WalkEntry } from "../../src/deno_ral/fs.ts"; -import { quarto } from "../../src/quarto.ts"; +import { runQuarto } from "../quarto-cmd.ts"; import { safeExistsSync, safeRemoveSync } from "../../src/core/path.ts"; import { runningInCI } from "../../src/core/ci-info.ts"; @@ -450,7 +450,9 @@ for (const { path: fileName } of files) { projectPath && !renderedProjects.has(projectPath) ) { - await quarto(["render", projectPath]); + // fail-loudly pre-render (throwOnFailure defaults to true); + // dispatches to the built binary when QUARTO_TEST_BIN is set + await runQuarto(["render", projectPath]); renderedProjects.add(projectPath); } diff --git a/tests/smoke/typst-gather/typst-gather.test.ts b/tests/smoke/typst-gather/typst-gather.test.ts index 7920a63158c..d8ed89e651a 100644 --- a/tests/smoke/typst-gather/typst-gather.test.ts +++ b/tests/smoke/typst-gather/typst-gather.test.ts @@ -4,6 +4,16 @@ import { existsSync } from "../../../src/deno_ral/fs.ts"; import { join } from "../../../src/deno_ral/path.ts"; import { execProcess } from "../../../src/core/process.ts"; +import { quartoDevBinCmd, quartoSpawnEnvOptions } from "../../quarto-cmd.ts"; +import { noErrors } from "../../verify.ts"; + +// Remove persistent, gitignored caches before verifying generated content. +const freshCache = (cacheDir: string) => async () => { + if (existsSync(cacheDir)) { + Deno.removeSync(cacheDir, { recursive: true }); + } +}; + // Test 1: Auto-detection from _extension.yml const verifyPackagesCreated: Verify = { name: "Verify typst/packages directory was created", @@ -38,9 +48,10 @@ const verifyExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyPackagesCreated, verifyExamplePackageCached], + [noErrors, verifyPackagesCreated, verifyExamplePackageCached], { cwd: () => "smoke/typst-gather", + setup: freshCache("_extensions/test-format/typst"), }, "typst-gather caches preview packages from extension templates", ); @@ -78,9 +89,10 @@ const verifyConfigExamplePackageCached: Verify = { testQuartoCmd( "call", ["typst-gather"], - [verifyConfigPackagesCreated, verifyConfigExamplePackageCached], + [noErrors, verifyConfigPackagesCreated, verifyConfigExamplePackageCached], { cwd: () => "smoke/typst-gather/with-config", + setup: freshCache("_extensions/config-format/typst"), }, "typst-gather uses rootdir from config file", ); @@ -246,7 +258,7 @@ const verifyNoPackagesStaged: Verify = { testQuartoCmd( "render", [join(noPackagesProjectDir, "index.qmd"), "--to", "typst"], - [verifyNoPackagesStaged], + [noErrors, verifyNoPackagesStaged], { teardown: async () => { try { @@ -263,25 +275,18 @@ testQuartoCmd( ); // Helper to run quarto as an external process and capture exit code -async function runQuarto( +async function execTypstGather( args: string[], cwd: string, env?: Record, ): Promise<{ success: boolean; stdout: string; stderr: string }> { - const quartoCmd = Deno.build.os === "windows" ? "quarto.cmd" : "quarto"; - const quartoPath = join( - Deno.cwd(), - "..", - "package/dist/bin", - quartoCmd, - ); const result = await execProcess({ - cmd: quartoPath, + cmd: quartoDevBinCmd(), args, cwd, stdout: "piped", stderr: "piped", - env: env ? { ...Deno.env.toObject(), ...env } : undefined, + ...quartoSpawnEnvOptions(env), }); return { success: result.success, @@ -299,7 +304,7 @@ unitTest( const configPath = join(cwd, "typst-gather.toml"); try { Deno.writeTextFileSync(configPath, "# existing config\n"); - const result = await runQuarto( + const result = await execTypstGather( ["call", "typst-gather", "--init-config"], cwd, ); @@ -319,7 +324,7 @@ unitTest( "typst-gather --init-config errors with no extension directory", async () => { const cwd = join(Deno.cwd(), "smoke/typst-gather/no-extension"); - const result = await runQuarto( + const result = await execTypstGather( ["call", "typst-gather", "--init-config"], cwd, ); @@ -338,7 +343,7 @@ unitTest( "typst-gather --init-config warns with empty extension (no typst entries)", async () => { const cwd = join(Deno.cwd(), "smoke/typst-gather/empty-extension"); - const result = await runQuarto( + const result = await execTypstGather( ["call", "typst-gather", "--init-config"], cwd, ); @@ -436,7 +441,7 @@ unitTest( Deno.cwd(), "docs/smoke-all/typst/marginalia-only-project", ); - const result = await runQuarto( + const result = await execTypstGather( ["render", "index.qmd", "--to", "typst"], projectDir, { QUARTO_TYPST_GATHER: "/nonexistent/typst-gather-binary" }, @@ -468,7 +473,7 @@ unitTest( Deno.cwd(), "docs/smoke-all/typst/marginalia-only-project", ); - const result = await runQuarto( + const result = await execTypstGather( ["render", "index.qmd", "--to", "typst"], projectDir, { QUARTO_TYPST_GATHER: falseCmd }, diff --git a/tests/smoke/verify/pdf-metadata.test.ts b/tests/smoke/verify/pdf-metadata.test.ts index 31ad2c775cb..7b60ee015d9 100644 --- a/tests/smoke/verify/pdf-metadata.test.ts +++ b/tests/smoke/verify/pdf-metadata.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfMetadata } from "../../verify-pdf-metadata.ts"; import { assert } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,14 +42,22 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf metadata assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/verify/pdf-text-position.test.ts b/tests/smoke/verify/pdf-text-position.test.ts index 2f40fa0e26d..e2cbbfd4743 100644 --- a/tests/smoke/verify/pdf-text-position.test.ts +++ b/tests/smoke/verify/pdf-text-position.test.ts @@ -8,6 +8,7 @@ */ import { testQuartoCmd } from "../../test.ts"; +import { noErrors } from "../../verify.ts"; import { ensurePdfTextPositions, PdfTextPositionAssertion } from "../../verify-pdf-text-position.ts"; import { assert, AssertionError } from "testing/asserts"; import { join } from "../../../src/deno_ral/path.ts"; @@ -41,21 +42,29 @@ async function assertThrowsWithPattern( ); } -// Test: Render fixture and run assertions -testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [], { +testQuartoCmd("render", [fixtureQmd, "--to", "typst"], [ + noErrors, + { + name: "pdf text position assertions (positive + expected failures)", + verify: async () => { + await runPositiveTests(); + await runExpectedFailureTests(); + await runSemanticTagTests(); + await runPageRoleTests(); + await runEdgeOverrideTests(); + await runDistanceConstraintTests(); + await runDistanceConstraintErrorTests(); + await runPageRoleWithEdgeTests(); + await runCenterEdgeTests(); + }, + }, +], { + setup: async () => { + if (safeExistsSync(fixturePdf)) { + safeRemoveSync(fixturePdf); + } + }, teardown: async () => { - // Run the test assertions after render completes - await runPositiveTests(); - await runExpectedFailureTests(); - await runSemanticTagTests(); - await runPageRoleTests(); - await runEdgeOverrideTests(); - await runDistanceConstraintTests(); - await runDistanceConstraintErrorTests(); - await runPageRoleWithEdgeTests(); - await runCenterEdgeTests(); - - // Cleanup if (safeExistsSync(fixturePdf)) { safeRemoveSync(fixturePdf); } diff --git a/tests/smoke/website/drafts-env.test.ts b/tests/smoke/website/drafts-env.test.ts index 7e7584d4d8c..7a8cef50ac5 100644 --- a/tests/smoke/website/drafts-env.test.ts +++ b/tests/smoke/website/drafts-env.test.ts @@ -16,6 +16,8 @@ const renderDir = docs("websites/drafts/drafts-env"); const dir = join(Deno.cwd(), renderDir); const outDir = join(dir, "_site"); +// Dev mode caches the base profile on first render, so set it before tests. +// Binary mode receives the same value through context.env. Deno.env.set("QUARTO_PROFILE", "drafts"); testQuartoCmd( @@ -23,6 +25,7 @@ testQuartoCmd( [renderDir], [noErrorsOrWarnings, ...[doesntHaveContentLinksToDrafts, doesntHaveEnvelopeLinksToDrafts, draftPostIsEmpty, searchDoesntHaveDraft, siteMapDoesntHaveDraft].map((ver) => { return ver(outDir)})], { + env: { QUARTO_PROFILE: "drafts" }, teardown: async () => { if (existsSync(outDir)) { await Deno.remove(outDir, { recursive: true }); diff --git a/tests/test.ts b/tests/test.ts index 80b1d6044b6..8f3fcae774f 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -10,7 +10,7 @@ import { warning } from "../src/deno_ral/log.ts"; import { initDenoDom } from "../src/core/deno-dom.ts"; import { cleanupLogger, initializeLogger, flushLoggers, logError, LogLevel, LogFormat } from "../src/core/log.ts"; -import { quarto } from "../src/quarto.ts"; +import { appendLogError, isBinaryMode, runQuarto } from "./quarto-cmd.ts"; import { join } from "../src/deno_ral/path.ts"; import * as colors from "fmt/colors"; import { runningInCI } from "../src/core/ci-info.ts"; @@ -36,8 +36,8 @@ export interface TestDescriptor { // Sets up the test context: TestContext; - // Executes the test - execute: () => Promise; + // Binary mode passes the child log target. + execute: (logFile?: string) => Promise; // Used to verify the outcome of the test verify: Verify[]; @@ -90,6 +90,9 @@ export interface TestContext { // Defaults to 600000 (10 minutes). Lower it to assert a performance budget // (e.g. a render that must not regress into a hang). timeout?: number; + + // Ignore this test in binary mode because it requires in-process internals. + requiresDevQuarto?: boolean; } // Allow to merge test contexts in Tests helpers @@ -127,6 +130,8 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?: }, // override ignore if provided ignore: additionalContext.ignore ?? baseContext.ignore, + requiresDevQuarto: additionalContext.requiresDevQuarto ?? + baseContext.requiresDevQuarto, // merge env with additional context taking precedence env: { ...baseContext.env, ...additionalContext.env }, // override timeout if provided @@ -147,19 +152,16 @@ export function testQuartoCmd( } test({ name, - execute: async () => { - const timeoutMs = context?.timeout ?? 600000; - const timeout = new Promise((_resolve, reject) => { - setTimeout( - reject, - timeoutMs, - `timed out after ${timeoutMs}ms`, - ); + execute: async (logFile?: string) => { + await runQuarto([cmd, ...args], { + env: context?.env, + logFile, + logLevel: logConfig?.level, + logFormat: logConfig?.format, + timeoutMs: context?.timeout, + // Let verifiers report failures from the log. + throwOnFailure: false, }); - await Promise.race([ - quarto([cmd, ...args], undefined, context?.env), - timeout, - ]); }, verify, context: context || {}, @@ -213,7 +215,8 @@ export function test(test: TestDescriptor) { const sanitizeResources = test.context.sanitize?.resources; const sanitizeOps = test.context.sanitize?.ops; const sanitizeExit = test.context.sanitize?.exit; - const ignore = test.context.ignore; + const ignore = test.context.ignore || + (isBinaryMode() && test.context.requiresDevQuarto); const userSession = !runningInCI(); const args: Deno.TestDefinition = { @@ -223,33 +226,24 @@ export function test(test: TestDescriptor) { const runTest = !test.context.prereq || await test.context.prereq(); if (runTest) { const wd = Deno.cwd(); - if (test.context?.cwd) { - Deno.chdir(test.context.cwd()); - } - if (test.context.setup) { - await test.context.setup(); - } + // The child owns log capture in binary mode. + const binMode = isBinaryMode(); let cleanedup = false; const cleanupLogOnce = async () => { - if (!cleanedup) { + if (!cleanedup && !binMode) { await cleanupLogger(); cleanedup = true; } }; - // Capture the output - const log = Deno.makeTempFileSync({ suffix: ".json" }); - const handlers = await initializeLogger({ - log: test.logConfig?.log || log, - level: test.logConfig?.level || "INFO", - format: test.logConfig?.format || "json-stream", - quiet: true, - }); - - const logOutput = (path: string) => { - if (existsSync(path)) { + let log: string | undefined; + let logTarget: string | undefined; + let handlers: Awaited> | undefined; + + const logOutput = (path?: string) => { + if (path && existsSync(path)) { return readExecuteOutput(path); } else { return undefined; @@ -258,21 +252,53 @@ export function test(test: TestDescriptor) { let lastVerify; try { + // Keep setup and cwd changes inside the cleanup scope. + if (test.context?.cwd) { + Deno.chdir(test.context.cwd()); + } + + if (test.context.setup) { + await test.context.setup(); + } + + // Capture the output. Started only after setup, so a setup that + // renders its own baseline (e.g. building a freeze cache) doesn't + // attribute its output to the execute() run that verify() inspects. + log = Deno.makeTempFileSync({ suffix: ".json" }); + logTarget = test.logConfig?.log || log; + handlers = binMode ? undefined : await initializeLogger({ + log: logTarget, + level: test.logConfig?.level || "INFO", + format: test.logConfig?.format || "json-stream", + quiet: true, + }); try { - await test.execute(); + await test.execute(logTarget); } catch (e) { - logError(e); + if (binMode) { + // Append directly because binary mode has no harness logger. + const message = e instanceof Error + ? `${e.message}\n${e.stack ?? ""}` + : String(e); + appendLogError(logTarget, message); + } else { + logError(e); + } } // Cleanup the output logging await cleanupLogOnce(); - flushLoggers(handlers); + if (handlers) { + flushLoggers(handlers); + } - // Read the output - const testOutput = logOutput(log); - if (testOutput) { + // Both logging modes write to logTarget; a missing log is a failure. + const testOutput = logOutput(logTarget); + if (testOutput === undefined) { + fail(`test log file is missing: ${logTarget}`); + } else { for (const ver of test.verify) { lastVerify = ver; if (userSession) { @@ -320,7 +346,13 @@ export function test(test: TestDescriptor) { ? colors.brightGreen(verifyFailed) : verifyFailed; - const logMessages = logOutput(log); + // Preserve the primary failure if the log is malformed. + let logMessages: ExecuteOutput[] | undefined; + try { + logMessages = logOutput(logTarget); + } catch { + logMessages = undefined; + } // Create distinctive failure marker for easy log navigation // This helps users find the failure when clicking GitHub Actions annotations @@ -356,14 +388,19 @@ export function test(test: TestDescriptor) { fail(output.join("\n")); } finally { - safeRemoveSync(log); - await cleanupLogOnce(); - if (test.context.teardown) { - await test.context.teardown(); + if (log) { + safeRemoveSync(log); } - - if (test.context?.cwd) { - Deno.chdir(wd); + await cleanupLogOnce(); + // Restore the cwd even when teardown fails. + try { + if (test.context.teardown) { + await test.context.teardown(); + } + } finally { + if (test.context?.cwd) { + Deno.chdir(wd); + } } } } else { @@ -383,6 +420,7 @@ export function test(test: TestDescriptor) { Deno.test(args); } +// Keep parsing strict; mergeChildLog() removes timeout-torn trailing records. export function readExecuteOutput(log: string) { const jsonStream = Deno.readTextFileSync(log); const lines = jsonStream.split("\n").filter((line) => !!line); diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence-folded-block-strings.test.ts diff --git a/tests/smoke/yaml-intelligence/yaml-intelligence.test.ts b/tests/unit/yaml-intelligence/yaml-intelligence.test.ts similarity index 100% rename from tests/smoke/yaml-intelligence/yaml-intelligence.test.ts rename to tests/unit/yaml-intelligence/yaml-intelligence.test.ts diff --git a/tests/utils.ts b/tests/utils.ts index eabd8344be7..aedc53a0850 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -32,6 +32,20 @@ export async function withTempDir( } } +// Run fn from dir and restore the process cwd afterward. +export async function withCwd( + dir: string, + fn: () => T | Promise, +): Promise { + const wd = Deno.cwd(); + Deno.chdir(dir); + try { + return await fn(); + } finally { + Deno.chdir(wd); + } +} + // Find a _quarto.yaml file in the directory hierarchy of the input file export function findProjectDir(input: string, until?: RegExp | undefined): string | undefined { let dir = dirname(input); @@ -240,8 +254,13 @@ export function fileLoader(...path: string[]) { }; } +// Resolve the configured test binary or the dev Quarto on PATH. // On Windows, `quarto.cmd` needs to be explicit in `execProcess()` export function quartoDevCmd(): string { + const bin = Deno.env.get("QUARTO_TEST_BIN"); + if (bin && bin.length > 0) { + return bin; + } return isWindows ? "quarto.cmd" : "quarto"; }