From cd0faed3e05a5d007c53aefa6fb1805124663d2b Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 24 Jul 2026 12:16:15 +0100 Subject: [PATCH 1/3] ci: Split code coverage into Package Test and Project Test jobs Convert the existing coverage job to [Code Coverage] Package Test - NGO [ubuntu, trunk] and add [Code Coverage] Project Test - NGO testproject [ubuntu, trunk]. Wire Package Test into PR/nightly triggers and Project Test into the weekly run. --- .yamato/_triggers.yml | 6 +++-- .yamato/code-coverage.yml | 47 ++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index b88547780c..6e30356fe3 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -90,7 +90,7 @@ pr_code_changes_checks: - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_ubuntu_il2cpp_{{ pinnedTrunk }} - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_{{ pinnedTrunk }} # Run code coverage test - - .yamato/code-coverage.yml#code_coverage_ubuntu_{{ validation_editors.default }} + - .yamato/code-coverage.yml#code_coverage_package_test_ubuntu_trunk triggers: expression: |- (pull_request.comment eq "ngo" OR @@ -160,7 +160,7 @@ develop_nightly: # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version - .yamato/vetting-test.yml#vetting_test # Run code coverage test - - .yamato/code-coverage.yml#code_coverage_ubuntu_{{ validation_editors.default }} + - .yamato/code-coverage.yml#code_coverage_package_test_ubuntu_trunk # Run all tests on weekly bases @@ -191,3 +191,5 @@ develop_weekly_trunk: - .yamato/_run-all.yml#run_all_webgl_builds # Run Runtime tests against CMB service - .yamato/_run-all.yml#run_all_project_tests_cmb_service + # Run code coverage on the testproject project tests + - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 64a2692225..3e5310d45e 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -2,33 +2,37 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This job is responsible for executing package tests with code coverage analysis enabled. + # This file defines package/project tests with code coverage analysis enabled. # Coverage analysis provides insights into: # Test coverage metrics for NGO assemblies # Line and branch coverage statistics # Generated HTML reports for coverage visualization # Additional metrics for coverage analysis + # Two jobs are defined: + # [Code Coverage] Package Test - runs the package's own tests (from the packed .tgz) with coverage + # [Code Coverage] Project Test - runs the in-repo testproject tests with coverage # CONFIGURATION STRUCTURE-------------------------------------------------------------- - # Jobs are generated using nested loops through: - # 1. For default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) - # 2. For default editor version (trunk) since coverage would not vary between editors (no need for checks on more editors) + # Jobs are generated for: + # 1. Default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) + # 2. Trunk editor only since coverage would not vary between editors (no need for checks on more editors) #TECHNICAL CONSIDERATIONS--------------------------------------------------------------- - # In theory this job also runs package tests, but we don't want to use it as default since is heavier (because of added coverage analysis) and coverage is not changing that often + # In theory these jobs also run tests, but we don't use them as default since they are heavier (because of added coverage analysis) and coverage is not changing that often # Requires Unity Editor installation # Burst compilation is disabled to ensure accurate coverage measurement # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible # QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # To see where this job is included (in trigger job definitions) look into _triggers.yml file - + # To see where these jobs are included (in trigger job definitions) look into _triggers.yml file. Currently: + # [Code Coverage] Package Test runs on PR changes (pr_code_changes_checks) and nightly (develop_nightly) + # [Code Coverage] Project Test runs weekly (develop_weekly_trunk) +{% assign editor = "trunk" -%} {% for platform in test_platforms.default -%} -{% for editor in validation_editors.default -%} -code_coverage_{{ platform.name }}_{{ editor }}: - name: Code Coverage - NGO [{{ platform.name }}, {{ editor }}] +code_coverage_package_test_{{ platform.name }}_{{ editor }}: + name: '[Code Coverage] Package Test - NGO [{{ platform.name }}, {{ editor }}]' agent: type: {{ platform.type }} image: {{ platform.image }} @@ -47,4 +51,27 @@ code_coverage_{{ platform.name }}_{{ editor }}: dependencies: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} + +{% for platform in test_platforms.default -%} +{% for project in projects.default -%} +code_coverage_project_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: + name: '[Code Coverage] Project Test - NGO {{ project.name }} [{{ platform.name }}, {{ editor }}]' + agent: + type: {{ platform.type }} + image: {{ platform.image }} + flavor: {{ platform.flavor }} +{% if platform.model %} + model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) +{% endif %} + commands: + - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --editor-location=.Editor --enable-code-coverage --coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_project_{{ project.name }}_{{ platform.name }}_{{ editor }};flags:NGOv2_project_{{ project.name }}_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout={{ test_timeout }} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --artifacts-path=test-results + artifacts: + logs: + paths: + - "test-results/**/*" + dependencies: + - .yamato/_run-all.yml#run_quick_checks # initial checks to perform fast validation of common errors + - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} +{% endfor -%} {% endfor -%} From 77c56ebb333f4c482de35140e1b911ab3d04bb66 Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 24 Jul 2026 15:59:28 +0100 Subject: [PATCH 2/3] ci: Run both coverage jobs on PR and nightly --- .yamato/_triggers.yml | 4 ++-- .yamato/code-coverage.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index 6e30356fe3..e63eca9791 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -91,6 +91,7 @@ pr_code_changes_checks: - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_{{ pinnedTrunk }} # Run code coverage test - .yamato/code-coverage.yml#code_coverage_package_test_ubuntu_trunk + - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk triggers: expression: |- (pull_request.comment eq "ngo" OR @@ -161,6 +162,7 @@ develop_nightly: - .yamato/vetting-test.yml#vetting_test # Run code coverage test - .yamato/code-coverage.yml#code_coverage_package_test_ubuntu_trunk + - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk # Run all tests on weekly bases @@ -191,5 +193,3 @@ develop_weekly_trunk: - .yamato/_run-all.yml#run_all_webgl_builds # Run Runtime tests against CMB service - .yamato/_run-all.yml#run_all_project_tests_cmb_service - # Run code coverage on the testproject project tests - - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 3e5310d45e..1f5b63ef0a 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -26,7 +26,7 @@ # QUALITY CONSIDERATIONS-------------------------------------------------------------------- # To see where these jobs are included (in trigger job definitions) look into _triggers.yml file. Currently: # [Code Coverage] Package Test runs on PR changes (pr_code_changes_checks) and nightly (develop_nightly) - # [Code Coverage] Project Test runs weekly (develop_weekly_trunk) + # [Code Coverage] Project Test runs on PR changes (pr_code_changes_checks) and nightly (develop_nightly) {% assign editor = "trunk" -%} From ce72070c2e57b058133035c9cd4153e0d4c6041d Mon Sep 17 00:00:00 2001 From: Nikos Chagialas Date: Fri, 24 Jul 2026 17:25:39 +0100 Subject: [PATCH 3/3] ci: Use pinnedTrunk for PR coverage and drop package-test job --- .yamato/_triggers.yml | 8 +++---- .yamato/code-coverage.yml | 44 ++++++++++----------------------------- 2 files changed, 14 insertions(+), 38 deletions(-) diff --git a/.yamato/_triggers.yml b/.yamato/_triggers.yml index e63eca9791..d70ba68fe9 100644 --- a/.yamato/_triggers.yml +++ b/.yamato/_triggers.yml @@ -89,9 +89,8 @@ pr_code_changes_checks: # Note that our daily tests will anyway run both test configurations in "minimal supported" and "trunk" configurations - .yamato/desktop-standalone-tests.yml#desktop_standalone_test_testproject_ubuntu_il2cpp_{{ pinnedTrunk }} - .yamato/cmb-service-standalone-tests.yml#cmb_service_standalone_test_testproject_ubuntu_il2cpp_{{ pinnedTrunk }} - # Run code coverage test - - .yamato/code-coverage.yml#code_coverage_package_test_ubuntu_trunk - - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk + # Run code coverage test (PRs use the pinned "safe" trunk) + - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_{{ pinnedTrunk }} triggers: expression: |- (pull_request.comment eq "ngo" OR @@ -160,8 +159,7 @@ develop_nightly: - .yamato/project-updated-dependencies-test.yml#updated-dependencies_testproject_NGO_win_6000.0 # Run API validation to early-detect all new APIs that would force us to release new minor version of the package. Note that for this to work the package version in package.json must correspond to "actual package state" which means that it should be higher than last released version - .yamato/vetting-test.yml#vetting_test - # Run code coverage test - - .yamato/code-coverage.yml#code_coverage_package_test_ubuntu_trunk + # Run code coverage test (nightly uses actual trunk) - .yamato/code-coverage.yml#code_coverage_project_test_testproject_ubuntu_trunk diff --git a/.yamato/code-coverage.yml b/.yamato/code-coverage.yml index 1f5b63ef0a..18adbef416 100644 --- a/.yamato/code-coverage.yml +++ b/.yamato/code-coverage.yml @@ -2,58 +2,35 @@ --- # DESCRIPTION-------------------------------------------------------------------------- - # This file defines package/project tests with code coverage analysis enabled. + # This job runs the in-repo testproject tests with code coverage analysis enabled. + # The project tests also exercise the NGO package tests, so a separate package-test coverage job is not needed. # Coverage analysis provides insights into: # Test coverage metrics for NGO assemblies # Line and branch coverage statistics # Generated HTML reports for coverage visualization # Additional metrics for coverage analysis - # Two jobs are defined: - # [Code Coverage] Package Test - runs the package's own tests (from the packed .tgz) with coverage - # [Code Coverage] Project Test - runs the in-repo testproject tests with coverage # CONFIGURATION STRUCTURE-------------------------------------------------------------- # Jobs are generated for: # 1. Default platform only (Ubuntu) since coverage would not vary between platforms (no need for checks on more platforms) - # 2. Trunk editor only since coverage would not vary between editors (no need for checks on more editors) + # 2. Two editors: actual "trunk" (used by nightly) and the "pinnedTrunk" safe version (used by PRs). + # Occasionally trunk breaks our tests, so PRs run against a pinned "safe" trunk while nightly runs actual trunk. #TECHNICAL CONSIDERATIONS--------------------------------------------------------------- - # In theory these jobs also run tests, but we don't use them as default since they are heavier (because of added coverage analysis) and coverage is not changing that often # Requires Unity Editor installation # Burst compilation is disabled to ensure accurate coverage measurement # In order to properly use -coverage-results-path parameter we need to start it with $PWD (which means the absolute path). Otherwise coverage results will not be visible # QUALITY CONSIDERATIONS-------------------------------------------------------------------- - # To see where these jobs are included (in trigger job definitions) look into _triggers.yml file. Currently: - # [Code Coverage] Package Test runs on PR changes (pr_code_changes_checks) and nightly (develop_nightly) - # [Code Coverage] Project Test runs on PR changes (pr_code_changes_checks) and nightly (develop_nightly) + # To see where this job is included (in trigger job definitions) look into _triggers.yml file. Currently: + # [Code Coverage] Project Test runs on PR changes (pr_code_changes_checks, pinnedTrunk) and nightly (develop_nightly, trunk) -{% assign editor = "trunk" -%} - -{% for platform in test_platforms.default -%} -code_coverage_package_test_{{ platform.name }}_{{ editor }}: - name: '[Code Coverage] Package Test - NGO [{{ platform.name }}, {{ editor }}]' - agent: - type: {{ platform.type }} - image: {{ platform.image }} - flavor: {{ platform.flavor }} -{% if platform.model %} - model: {{ platform.model }} # This is set only in platforms where we want non-default model to use (more information in project.metafile) -{% endif %} - commands: - - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - upm-pvp create-test-project test-project --packages "upm-ci~/packages/*.tgz" --unity .Editor - - UnifiedTestRunner --suite=editor --suite=playmode --editor-location=.Editor --testproject=test-project --enable-code-coverage --coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_{{ platform.name }}_{{ editor }};flags:NGOv2_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout={{ test_timeout }} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --artifacts-path=test-results - artifacts: - logs: - paths: - - "test-results/**/*" - dependencies: - - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} -{% endfor -%} +{% assign coverage_editors = "trunk," | append: pinnedTrunk | split: "," -%} {% for platform in test_platforms.default -%} {% for project in projects.default -%} +{% for editor in coverage_editors -%} +{% if editor == "trunk" -%}{% assign editor_label = "trunk" -%}{% else -%}{% assign editor_label = "pinnedTrunk" -%}{% endif -%} code_coverage_project_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: name: '[Code Coverage] Project Test - NGO {{ project.name }} [{{ platform.name }}, {{ editor }}]' agent: @@ -65,7 +42,7 @@ code_coverage_project_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: {% endif %} commands: - unity-downloader-cli --fast --wait -u {{ editor }} -c Editor {% if platform.name == "mac" %} --arch arm64 {% endif %} # For macOS we use ARM64 models - - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --editor-location=.Editor --enable-code-coverage --coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_project_{{ project.name }}_{{ platform.name }}_{{ editor }};flags:NGOv2_project_{{ project.name }}_{{ platform.name }}_{{ editor }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout={{ test_timeout }} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --artifacts-path=test-results + - UnifiedTestRunner --testproject={{ project.path }} --suite=editor --suite=playmode --editor-location=.Editor --enable-code-coverage --coverage-upload-options="reportsDir:$PWD/test-results/CoverageResults;name:NGOv2_project_{{ project.name }}_{{ platform.name }}_{{ editor_label }};flags:NGOv2_project_{{ project.name }}_{{ platform.name }}_{{ editor_label }};verbose" --coverage-results-path=$PWD/test-results/CoverageResults --coverage-options="generateHtmlReport;generateAdditionalMetrics;assemblyFilters:+Unity.Netcode.Editor,+Unity.Netcode.Runtime" --extra-editor-arg=--burst-disable-compilation --timeout={{ test_timeout }} --rerun-strategy=Test --retry={{ num_test_retries }} --clean-library-on-rerun --artifacts-path=test-results artifacts: logs: paths: @@ -75,3 +52,4 @@ code_coverage_project_test_{{ project.name }}_{{ platform.name }}_{{ editor }}: - .yamato/package-pack.yml#package_pack_-_ngo_{{ platform.name }} {% endfor -%} {% endfor -%} +{% endfor -%}