diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 119445a358..fc87d6b3e6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,13 +3,13 @@ - + https://github.com/dotnet/arcade - caa49f7726ab75f513f2fb814030657cf1afc0e4 + e7d5d251b6f84c1c400deefa11febf2900917783 - + https://github.com/dotnet/arcade - caa49f7726ab75f513f2fb814030657cf1afc0e4 + e7d5d251b6f84c1c400deefa11febf2900917783 diff --git a/eng/common/Get-GitHubAppToken.ps1 b/eng/common/Get-GitHubAppToken.ps1 index 9c7e3dcd6a..ea776bd6bc 100644 --- a/eng/common/Get-GitHubAppToken.ps1 +++ b/eng/common/Get-GitHubAppToken.ps1 @@ -110,19 +110,20 @@ $headers = @{ Write-Host "Looking up installation for '$InstallationOwner'..." try { - $installations = @() + $installations = [System.Collections.Generic.List[object]]::new() $page = 1 do { - # Assign the response before wrapping it in @(). PowerShell otherwise - # preserves a top-level JSON array as one nested pipeline object. $pageResponse = Invoke-RestMethod ` -Uri "https://api.github.com/app/installations?per_page=100&page=$page" ` -Headers $headers ` -Method Get - $pageInstallations = @($pageResponse) - $installations += $pageInstallations + $pageInstallationCount = 0 + foreach ($installation in $pageResponse) { + $installations.Add($installation) + $pageInstallationCount++ + } $page++ - } while ($pageInstallations.Count -eq 100) + } while ($pageInstallationCount -eq 100) } catch { Write-PipelineTelemetryError -Category 'Build' -Message "Failed to list GitHub App installations: $_. The signed JWT may be invalid or the App's Client ID ('$AppClientId') may be incorrect." diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml new file mode 100644 index 0000000000..53bbf74927 --- /dev/null +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -0,0 +1,278 @@ +parameters: +# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes. +- name: timeoutInMinutes + type: number + default: 360 + +# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization. +# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty. +- name: organization + type: string + default: '' + +# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository. +# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty. +- name: repository + type: string + default: '' + +# Optional dependency list for the generated job. +- name: dependsOn + type: object + default: [] + +# Optional condition for the generated job. +- name: condition + type: string + default: '' + +# Whether failures in the monitor job should allow the pipeline to continue. +- name: continueOnError + type: boolean + default: false + +# NuGet package id of the Helix job monitor tool. +- name: toolPackageId + type: string + default: Microsoft.DotNet.Helix.JobMonitor + +# Console command exposed by the installed tool package. +- name: toolCommand + type: string + default: dotnet-helix-job-monitor + +# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the +# default code path the version is taken from the consuming repo's .config/dotnet-tools.json. +- name: toolVersion + type: string + default: '' + +# Base URI for the Helix service (--helix-base-uri). +- name: helixBaseUri + type: string + default: https://helix.dot.net/ + +# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable. +- name: helixAccessToken + type: string + default: '' + +# Polling interval in seconds (--polling-interval-seconds). +- name: pollingIntervalSeconds + type: number + default: 30 + +# Maximum number of work items whose results may be downloaded, parsed, and +# uploaded concurrently. +- name: testResultUploadParallelism + type: number + default: 48 + +# When 'true' (the default), Helix work items that exit 0 but have failed AzDO test results +# are treated as failed: they count toward the monitor's exit code and are resubmitted by a +# later invocation's retry pass. Set to 'false' to fall back to exit-code-only outcomes. +# Forwarded as --fail-on-failed-tests. +- name: failWorkItemsWithFailedTests + type: boolean + default: true + +# When true, allow the monitor to succeed when this stage produces no Helix jobs in any attempt. +# Forwarded as --allow-no-helix-jobs. +- name: allowNoHelixJobs + type: boolean + default: false + +# When true, test results are reported to Azure DevOps using the fully qualified test name +# (Namespace.Type.Method) as the stable automatedTestName and the visible title is qualified as +# well (--use-fully-qualified-test-name). Opt-in because it changes AzDO test identity and display; +# primarily useful for frameworks like MSTest whose display name is only the method name. +- name: useFullyQualifiedTestName + type: boolean + default: false + +# Controls per-test output attachments. Defaults to Failed. +- name: testResultAttachmentMode + type: string + default: Failed + values: + - Failed + - All + - None + +# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool +# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into +# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is +# primarily intended for the Arcade repository itself, where the Helix job monitor tool is +# built in the same pipeline that runs this template. +# +# When this parameter is empty (the default), the consuming repository must declare the tool +# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template +# will check out the repo and run 'dotnet tool restore' to install the version pinned there. +- name: toolNupkgArtifactName + type: string + default: '' + +# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults +# to the standard Arcade non-shipping packages location for a Release build (relative to the +# pipeline artifact root, which is itself the build's 'artifacts' directory). +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + +jobs: +- job: HelixJobMonitor + displayName: Monitor Helix Jobs + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + continueOnError: ${{ parameters.continueOnError }} + ${{ if ne(length(parameters.dependsOn), 0) }}: + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + os: linux + demands: ImageOverride -equals build.azurelinux.3.amd64.open + ${{ else }}: + name: $(DncEngInternalBuildPool) + os: linux + demands: ImageOverride -equals build.azurelinux.3.amd64 + steps: + - checkout: self + fetchDepth: 1 + + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Helix Job Monitor artifact + inputs: + buildType: current + artifactName: ${{ parameters.toolNupkgArtifactName }} + itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg' + targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg + + - bash: | + set -euo pipefail + + toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool" + mkdir -p "$toolPath" + + packageId='${{ parameters.toolPackageId }}' + toolVersion='${{ parameters.toolVersion }}' + nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}' + nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath" + + if [ ! -d "$nupkgDir" ]; then + echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2 + exit 1 + fi + + nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1) + if [ -z "$nupkg" ]; then + echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2 + exit 1 + fi + + # Derive the version from the nupkg filename so the local package is selected + # deterministically instead of resolving against any other configured feed. + nupkgBase=$(basename "$nupkg" .nupkg) + derivedVersion="${nupkgBase#${packageId}.}" + if [ -z "$toolVersion" ]; then + toolVersion="$derivedVersion" + fi + + echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + + # Create a minimal NuGet.config that only references the local nupkg directory. + # This avoids conflicts with the repo's package source mapping which blocks --add-source. + toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config" + printf '\n\n \n \n \n \n\n' "$nupkgDir" > "$toolNugetConfig" + + pushd "$(Build.SourcesDirectory)" > /dev/null + ./eng/common/dotnet.sh tool install \ + --tool-path "$toolPath" "$packageId" \ + --version "$toolVersion" \ + --configfile "$toolNugetConfig" + + # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec. + toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1) + toolDll="${toolDll%.deps.json}.dll" + if [ ! -f "$toolDll" ]; then + echo "Could not find tool DLL in '$toolPath/.store'." >&2 + exit 1 + fi + + echo "Tool DLL: $toolDll" + echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll" + displayName: Install Helix Job Monitor + + - ${{ else }}: + - bash: ./eng/common/dotnet.sh tool restore + displayName: Restore Helix Job Monitor + + - bash: | + set -euo pipefail + + toolArgs=( + --helix-base-uri '${{ parameters.helixBaseUri }}' + --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' + --fail-on-failed-tests '${{ parameters.failWorkItemsWithFailedTests }}' + --allow-no-helix-jobs '${{ parameters.allowNoHelixJobs }}' + --use-fully-qualified-test-name '${{ parameters.useFullyQualifiedTestName }}' + --max-wait-minutes "$((${{ parameters.timeoutInMinutes }} - 5))" # Set the tool's timeout slightly lower than the Azure DevOps job timeout to allow it to exit gracefully. + --stage-name '$(System.StageName)' + --stage-attempt '$(System.StageAttempt)' + --job-attempt '$(System.JobAttempt)' + --test-result-upload-parallelism '${{ parameters.testResultUploadParallelism }}' + ) + + organization='${{ parameters.organization }}' + repository='${{ parameters.repository }}' + testResultAttachmentMode='${{ parameters.testResultAttachmentMode }}' + + # Fall back to Azure DevOps-provided environment variables when the caller did not + # supply organization / repository explicitly. BUILD_REPOSITORY_NAME is typically + # 'owner/repo' for GitHub-backed builds and 'owner-repo' for internal builds. + if [ -z "$organization" ] || [ -z "$repository" ]; then + buildRepoName="${BUILD_REPOSITORY_NAME:-}" + if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then + repoOwner="${buildRepoName%%/*}" + repoName="${buildRepoName#*/}" + elif [ -n "$buildRepoName" ] && [[ "$buildRepoName" == *-* ]]; then + repoOwner="${buildRepoName%%-*}" + repoName="${buildRepoName#*-}" + fi + + if [ -n "${repoOwner:-}" ] && [ -n "${repoName:-}" ]; then + if [ -z "$organization" ]; then organization="$repoOwner"; fi + if [ -z "$repository" ]; then repository="$repoName"; fi + fi + fi + + if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi + if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + if [ -n "$testResultAttachmentMode" ]; then + toolArgs+=( --test-result-attachment-mode "$testResultAttachmentMode" ) + fi + + # Build.Reason and Build.SourceBranch are required to derive the Helix source filter + # the same way the Helix SDK submitter does (PR -> 'pr', internal -> 'official', + # otherwise -> 'ci'). Without these, manually-queued / scheduled / CI builds would + # be looked up under the wrong source prefix and find zero jobs. + toolArgs+=( --build-reason "$(Build.Reason)" ) + toolArgs+=( --source-branch "$(Build.SourceBranch)" ) + + if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then + # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet. + export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet" + ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}" + else + # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it + # through the manifest from the repo root. + pushd "$BUILD_SOURCESDIRECTORY" > /dev/null + trap 'popd > /dev/null' EXIT + ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}" + fi + displayName: Monitor Helix Jobs + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index b28af6613c..15d8c1571e 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -9,15 +9,15 @@ parameters: GithubPat: $(BotAccount-dotnet-bot-repo-PAT) # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). - # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT. - # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not - # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter. + # The `internal` and `DevDiv` System.TeamProject values have same-named, project-scoped + # connections. Other values, and any pipeline that sets this to '', use the CeapexPat parameter. CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' - # GitHub App authentication for the OneLoc check-in PR (dnceng/internal only). - # The infrastructure identifiers are centralized here and the App path is enabled by default. - # DevDiv requires its own project-scoped service connection before this path can be enabled there. + # GitHub App authentication for the OneLoc check-in PR. + # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service + # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure. UseGitHubAppAuthentication: true + UseGitHubAppAuthenticationInOtherProjects: false GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' GitHubAppKeyVaultName: 'EngKeyVault' @@ -88,22 +88,24 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} - # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only). - # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal. - - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + # Acquire an Entra token when System.TeamProject is `internal` or `DevDiv`. + - ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: - template: /eng/common/templates/steps/get-federated-access-token.yml parameters: federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} outputVariableName: 'CeapexEntraToken' condition: ${{ parameters.condition }} - # Mint a short-lived GitHub App installation token for the loc check-in PR (dnceng/internal only). - # All other projects fall back to PAT-based auth, since the app service connection is scoped to dnceng/internal. - - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), eq(variables['System.TeamProject'], 'internal')) }}: + # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection + # provisioned in each supported project; other projects must explicitly opt in and override it. + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: - template: /eng/common/core-templates/steps/get-github-app-token.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - azureSubscription: ${{ parameters.GitHubAppServiceConnection }} + ${{ if and(eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.GitHubAppServiceConnection, 'dnceng-oneloc-githubapp')) }}: + azureSubscription: 'devdiv-oneloc-githubapp' + ${{ else }}: + azureSubscription: ${{ parameters.GitHubAppServiceConnection }} keyVaultName: ${{ parameters.GitHubAppKeyVaultName }} keyName: ${{ parameters.GitHubAppKeyName }} appClientId: ${{ parameters.GitHubAppClientId }} @@ -126,15 +128,15 @@ jobs: isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: + ${{ if and(ne(parameters.CeapexServiceConnection, ''), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'))) }}: patVariable: $(CeapexEntraToken) - ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}: + ${{ if or(eq(parameters.CeapexServiceConnection, ''), and(ne(variables['System.TeamProject'], 'internal'), ne(variables['System.TeamProject'], 'DevDiv'))) }}: patVariable: ${{ parameters.CeapexPat }} ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), eq(variables['System.TeamProject'], 'internal')) }}: + ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: gitHubPatVariable: "$(GitHubAppInstallationToken)" - ${{ if or(eq(parameters.UseGitHubAppAuthentication, false), ne(variables['System.TeamProject'], 'internal')) }}: + ${{ else }}: gitHubPatVariable: "${{ parameters.GithubPat }}" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml index 68fa739c4a..37678b0389 100644 --- a/eng/common/core-templates/steps/send-to-helix.yml +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -10,6 +10,7 @@ parameters: HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution + UseHelixMonitor: false # optional -- true will submit Helix jobs configured for the standalone Helix Job Monitor (results are reported/waited on out-of-band; this step will not wait, and WaitForWorkItemCompletion will be overridden) WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects @@ -31,7 +32,15 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: > + $(Build.SourcesDirectory)\eng\common\msbuild.ps1 + $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} + /restore + /p:TreatWarningsAsErrors=false + /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }} + ${{ parameters.HelixProjectArguments }} + /t:Test + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -61,7 +70,15 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false ${{ parameters.HelixProjectArguments }} /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: > + $(Build.SourcesDirectory)/eng/common/msbuild.sh + $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} + /restore + /p:TreatWarningsAsErrors=false + /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }} + ${{ parameters.HelixProjectArguments }} + /t:Test + /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/global.json b/global.json index cfa65a37dc..caea53d551 100644 --- a/global.json +++ b/global.json @@ -3,7 +3,7 @@ "dotnet": "10.0.111" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26414.3", - "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.26414.3" + "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.26451.2", + "Microsoft.DotNet.Helix.Sdk": "10.0.0-beta.26451.2" } }