diff --git a/lambdas/functions/control-plane/package.json b/lambdas/functions/control-plane/package.json index dc74e770a9..ee3aedd210 100644 --- a/lambdas/functions/control-plane/package.json +++ b/lambdas/functions/control-plane/package.json @@ -31,8 +31,8 @@ }, "dependencies": { "@aws-github-runner/aws-powertools-util": "*", - "@aws-github-runner/aws-ssm-util": "*", "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/storage-providers": "*", "@aws-lambda-powertools/parameters": "^2.31.0", "@aws-sdk/client-ec2": "^3.1009.0", "@aws-sdk/client-sqs": "^3.1009.0", diff --git a/lambdas/functions/control-plane/src/github/auth.test.ts b/lambdas/functions/control-plane/src/github/auth.test.ts index dd2cf3b8c2..f87053819e 100644 --- a/lambdas/functions/control-plane/src/github/auth.test.ts +++ b/lambdas/functions/control-plane/src/github/auth.test.ts @@ -2,13 +2,19 @@ import { createAppAuth } from '@octokit/auth-app'; import { StrategyOptions } from '@octokit/auth-app/dist-types/types'; import { request } from '@octokit/request'; import { RequestInterface, RequestParameters } from '@octokit/types'; -import { getParameters } from '@aws-github-runner/aws-ssm-util'; +import { + getGitHubAppCredentialsStore, + type GitHubAppCredential, + type GitHubAppCredentialsStore, +} from '@aws-github-runner/storage-providers'; import { generateKeyPairSync } from 'node:crypto'; import * as nock from 'nock'; import { createGithubAppAuth, createOctokitClient, + getAppCount, + getAppId, getStoredInstallationId, onRateLimit, onSecondaryRateLimit, @@ -25,24 +31,27 @@ type MockProxy = T & { // eslint-disable-next-line @typescript-eslint/no-explicit-any const mock = (implementation?: any): MockProxy => vi.fn(implementation) as any; -vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getGitHubAppCredentialsStore: vi.fn(), +})); vi.mock('@octokit/auth-app'); const cleanEnv = process.env; -const ENVIRONMENT = 'dev'; -const GITHUB_APP_ID = '1'; -const PARAMETER_GITHUB_APP_ID_NAME = `/actions-runner/${ENVIRONMENT}/github_app_id`; -const PARAMETER_GITHUB_APP_KEY_BASE64_NAME = `/actions-runner/${ENVIRONMENT}/github_app_key_base64`; +const GITHUB_APP_ID = 1; -const mockedGetParameters = vi.mocked(getParameters); +const mockedGetGitHubAppCredentialsStore = vi.mocked(getGitHubAppCredentialsStore); +const mockCredentialsGet = vi.fn(); +const credentialsStore = { + get: mockCredentialsGet, +} satisfies GitHubAppCredentialsStore; beforeEach(() => { vi.resetModules(); vi.clearAllMocks(); + mockCredentialsGet.mockReset(); resetAppCredentialsCache(); process.env = { ...cleanEnv }; - process.env.PARAMETER_GITHUB_APP_ID_NAME = PARAMETER_GITHUB_APP_ID_NAME; - process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = PARAMETER_GITHUB_APP_KEY_BASE64_NAME; + mockedGetGitHubAppCredentialsStore.mockReturnValue(credentialsStore); nock.disableNetConnect(); }); @@ -80,38 +89,18 @@ describe('Test createGithubAppAuth', () => { const authType = 'app'; const token = '123456'; const decryptedValue = 'decryptedValue'; - const b64 = Buffer.from(decryptedValue, 'binary').toString('base64'); - - beforeEach(() => { - process.env.ENVIRONMENT = ENVIRONMENT; - }); - it('Throws early when PARAMETER_GITHUB_APP_ID_NAME is not set', async () => { - delete process.env.PARAMETER_GITHUB_APP_ID_NAME; + it('Propagates errors from the credential store', async () => { + const error = new Error('Unable to load GitHub App credentials'); + mockCredentialsGet.mockRejectedValueOnce(error); - await expect(createGithubAppAuth(installationId)).rejects.toThrow( - 'Environment variable PARAMETER_GITHUB_APP_ID_NAME is not set', - ); - expect(mockedGetParameters).not.toHaveBeenCalled(); - }); - - it('Throws early when PARAMETER_GITHUB_APP_KEY_BASE64_NAME is not set', async () => { - delete process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME; - - await expect(createGithubAppAuth(installationId)).rejects.toThrow( - 'Environment variable PARAMETER_GITHUB_APP_KEY_BASE64_NAME is not set', - ); - expect(mockedGetParameters).not.toHaveBeenCalled(); + await expect(createGithubAppAuth(installationId)).rejects.toBe(error); + expect(mockCredentialsGet).toHaveBeenCalledOnce(); }); it('Creates auth object with createJwt callback including jti claim', async () => { // Arrange - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: decryptedValue }]); const mockedAuth = vi.fn(); mockedAuth.mockResolvedValue({ token }); @@ -124,7 +113,7 @@ describe('Test createGithubAppAuth', () => { // Assert expect(mockedCreatAppAuth).toBeCalledTimes(1); const callArgs = mockedCreatAppAuth.mock.calls[0][0] as Record; - expect(callArgs.appId).toBe(parseInt(GITHUB_APP_ID)); + expect(callArgs.appId).toBe(GITHUB_APP_ID); expect(callArgs.createJwt).toBeTypeOf('function'); expect(callArgs).not.toHaveProperty('privateKey'); expect(callArgs.installationId).toBe(installationId); @@ -137,14 +126,7 @@ describe('Test createGithubAppAuth', () => { privateKeyEncoding: { type: 'pkcs8', format: 'pem' }, publicKeyEncoding: { type: 'spki', format: 'pem' }, }); - const b64Key = Buffer.from(privateKey as string).toString('base64'); - - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64Key], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: privateKey as string }]); let capturedCreateJwt: (appId: string | number, timeDifference?: number) => Promise<{ jwt: string }>; mockedCreatAppAuth.mockImplementation((opts: StrategyOptions) => { @@ -173,41 +155,9 @@ describe('Test createGithubAppAuth', () => { expect(payload).toHaveProperty('iss'); }); - it('Creates auth object with line breaks in SSH key.', async () => { - // Arrange - const b64PrivateKeyWithLineBreaks = Buffer.from(decryptedValue + '\n' + decryptedValue, 'binary').toString( - 'base64', - ); - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64PrivateKeyWithLineBreaks], - ]), - ); - - const mockedAuth = vi.fn(); - mockedAuth.mockResolvedValue({ token }); - const mockWithHook = Object.assign(mockedAuth, { hook: vi.fn() }); - mockedCreatAppAuth.mockReturnValue(mockWithHook); - - // Act - const result = await createGithubAppAuth(installationId); - - // Assert - expect(getParameters).toBeCalledWith([PARAMETER_GITHUB_APP_ID_NAME, PARAMETER_GITHUB_APP_KEY_BASE64_NAME]); - expect(mockedCreatAppAuth).toBeCalledTimes(1); - expect(mockedAuth).toBeCalledWith({ type: authType }); - expect(result.token).toBe(token); - }); - it('Creates auth object for public GitHub', async () => { // Arrange - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: decryptedValue }]); const mockedAuth = vi.fn(); mockedAuth.mockResolvedValue({ token }); @@ -218,11 +168,9 @@ describe('Test createGithubAppAuth', () => { const result = await createGithubAppAuth(installationId); // Assert - expect(getParameters).toBeCalledWith([PARAMETER_GITHUB_APP_ID_NAME, PARAMETER_GITHUB_APP_KEY_BASE64_NAME]); - expect(mockedCreatAppAuth).toBeCalledTimes(1); const callArgs = mockedCreatAppAuth.mock.calls[0][0] as Record; - expect(callArgs.appId).toBe(parseInt(GITHUB_APP_ID)); + expect(callArgs.appId).toBe(GITHUB_APP_ID); expect(callArgs.createJwt).toBeTypeOf('function'); expect(callArgs.installationId).toBe(installationId); expect(mockedAuth).toBeCalledWith({ type: authType }); @@ -238,12 +186,7 @@ describe('Test createGithubAppAuth', () => { () => mockedRequestInterface as RequestInterface, ); - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: decryptedValue }]); const mockedAuth = vi.fn(); mockedAuth.mockResolvedValue({ token }); // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -255,11 +198,9 @@ describe('Test createGithubAppAuth', () => { const result = await createGithubAppAuth(installationId, githubServerUrl); // Assert - expect(getParameters).toBeCalledWith([PARAMETER_GITHUB_APP_ID_NAME, PARAMETER_GITHUB_APP_KEY_BASE64_NAME]); - expect(mockedCreatAppAuth).toBeCalledTimes(1); const callArgs = mockedCreatAppAuth.mock.calls[0][0] as Record; - expect(callArgs.appId).toBe(parseInt(GITHUB_APP_ID)); + expect(callArgs.appId).toBe(GITHUB_APP_ID); expect(callArgs.createJwt).toBeTypeOf('function'); expect(callArgs.installationId).toBe(installationId); expect(callArgs.request).toBeDefined(); @@ -278,12 +219,7 @@ describe('Test createGithubAppAuth', () => { const installationId = undefined; - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: decryptedValue }]); const mockedAuth = vi.fn(); mockedAuth.mockResolvedValue({ token }); const mockWithHook = Object.assign(mockedAuth, { hook: vi.fn() }); @@ -293,11 +229,9 @@ describe('Test createGithubAppAuth', () => { const result = await createGithubAppAuth(installationId, githubServerUrl); // Assert - expect(getParameters).toBeCalledWith([PARAMETER_GITHUB_APP_ID_NAME, PARAMETER_GITHUB_APP_KEY_BASE64_NAME]); - expect(mockedCreatAppAuth).toBeCalledTimes(1); const callArgs = mockedCreatAppAuth.mock.calls[0][0] as Record; - expect(callArgs.appId).toBe(parseInt(GITHUB_APP_ID)); + expect(callArgs.appId).toBe(GITHUB_APP_ID); expect(callArgs.createJwt).toBeTypeOf('function'); expect(callArgs).not.toHaveProperty('installationId'); expect(callArgs.request).toBeDefined(); @@ -330,98 +264,48 @@ describe('Test throttling retry caps', () => { }); }); -describe('Test getStoredInstallationId', () => { - const decryptedValue = 'decryptedValue'; - const b64 = Buffer.from(decryptedValue, 'binary').toString('base64'); - - beforeEach(() => { - const mockedAuth = vi.fn(); - mockedAuth.mockResolvedValue({ token: 'token' }); - const mockWithHook = Object.assign(mockedAuth, { hook: vi.fn() }); - vi.mocked(createAppAuth).mockReturnValue(mockWithHook); - }); - +describe('Test GitHub App credential accessors', () => { it('returns stored installation ID when configured', async () => { - const installationIdParam = `/actions-runner/${ENVIRONMENT}/github_app_installation_id`; - process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = installationIdParam; - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - [installationIdParam, '12345'], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([ + { appId: GITHUB_APP_ID, privateKey: 'private-key', installationId: 12345 }, + ]); const result = await getStoredInstallationId(0); expect(result).toBe(12345); }); - it('returns undefined when installation ID param is empty', async () => { - process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = ''; - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); - - const result = await getStoredInstallationId(0); - expect(result).toBeUndefined(); - }); - - it('returns undefined when env var is not set', async () => { - delete process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME; - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); + it('returns undefined when the credential has no installation ID', async () => { + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: 'private-key' }]); const result = await getStoredInstallationId(0); expect(result).toBeUndefined(); }); it('returns undefined for out-of-bounds appIndex', async () => { - process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = ''; - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [PARAMETER_GITHUB_APP_ID_NAME, GITHUB_APP_ID], - [PARAMETER_GITHUB_APP_KEY_BASE64_NAME, b64], - ]), - ); + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: 'private-key' }]); const result = await getStoredInstallationId(99); expect(result).toBeUndefined(); }); - it('loads installation IDs for multi-app setup', async () => { - const app1IdParam = `/actions-runner/${ENVIRONMENT}/github_app_id`; - const app2IdParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_id`; - const app1KeyParam = `/actions-runner/${ENVIRONMENT}/github_app_key_base64`; - const app2KeyParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_key_base64`; - const app2InstallParam = `/actions-runner/${ENVIRONMENT}/additional_github_app_0_installation_id`; - - process.env.PARAMETER_GITHUB_APP_ID_NAME = `${app1IdParam}:${app2IdParam}`; - process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = `${app1KeyParam}:${app2KeyParam}`; - process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = `:${app2InstallParam}`; - - mockedGetParameters.mockResolvedValueOnce( - new Map([ - [app1IdParam, '1'], - [app1KeyParam, b64], - [app2IdParam, '2'], - [app2KeyParam, b64], - [app2InstallParam, '67890'], - ]), - ); + it('loads multi-app credentials once and exposes values by index', async () => { + const credentials: GitHubAppCredential[] = [ + { appId: 1, privateKey: 'private-key-1' }, + { appId: 2, privateKey: 'private-key-2', installationId: 67890 }, + ]; + mockCredentialsGet.mockResolvedValueOnce(credentials); + + await expect(getAppCount()).resolves.toBe(2); + await expect(getAppId()).resolves.toBe('1'); + await expect(getAppId(1)).resolves.toBe('2'); + await expect(getStoredInstallationId(0)).resolves.toBeUndefined(); + await expect(getStoredInstallationId(1)).resolves.toBe(67890); + expect(mockCredentialsGet).toHaveBeenCalledOnce(); + }); - // Primary app (index 0) has no stored installation ID - const result0 = await getStoredInstallationId(0); - expect(result0).toBeUndefined(); + it('throws a clear error for an out-of-bounds app ID index', async () => { + mockCredentialsGet.mockResolvedValueOnce([{ appId: GITHUB_APP_ID, privateKey: 'private-key' }]); - // Additional app (index 1) has stored installation ID - const result1 = await getStoredInstallationId(1); - expect(result1).toBe(67890); + await expect(getAppId(99)).rejects.toThrow('GitHub App credential at index 99 not found'); }); }); diff --git a/lambdas/functions/control-plane/src/github/auth.ts b/lambdas/functions/control-plane/src/github/auth.ts index f64ac00b30..e4ade0b38a 100644 --- a/lambdas/functions/control-plane/src/github/auth.ts +++ b/lambdas/functions/control-plane/src/github/auth.ts @@ -22,7 +22,7 @@ import { Octokit } from '@octokit/rest'; import { retry } from '@octokit/plugin-retry'; import { throttling } from '@octokit/plugin-throttling'; import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { getParameters } from '@aws-github-runner/aws-ssm-util'; +import { getGitHubAppCredentialsStore, type GitHubAppCredential } from '@aws-github-runner/storage-providers'; import { EndpointDefaults } from '@octokit/types'; const logger = createChildLogger('gh-auth'); @@ -69,52 +69,10 @@ export function onSecondaryRateLimit( return retryCount < MAX_SECONDARY_RATE_LIMIT_RETRIES; } -interface GitHubAppCredential { - appId: number; - privateKey: string; - installationId?: number; -} - let appCredentialsPromise: Promise | null = null; async function loadAppCredentials(): Promise { - if (!process.env.PARAMETER_GITHUB_APP_ID_NAME) { - throw new Error('Environment variable PARAMETER_GITHUB_APP_ID_NAME is not set'); - } - if (!process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME) { - throw new Error('Environment variable PARAMETER_GITHUB_APP_KEY_BASE64_NAME is not set'); - } - const idParams = process.env.PARAMETER_GITHUB_APP_ID_NAME.split(':').filter(Boolean); - const keyParams = process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME.split(':').filter(Boolean); - const installationIdParams = (process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME || '').split(':'); - if (idParams.length !== keyParams.length) { - throw new Error(`GitHub App parameter count mismatch: ${idParams.length} IDs vs ${keyParams.length} keys`); - } - // Batch fetch all SSM parameters in a single call to reduce API calls - const allParamNames = [...idParams, ...keyParams, ...installationIdParams.filter((p) => p.length > 0)]; - const params = await getParameters(allParamNames); - - const credentials: GitHubAppCredential[] = []; - for (let i = 0; i < idParams.length; i++) { - const appIdValue = params.get(idParams[i]); - if (!appIdValue) { - throw new Error(`Parameter ${idParams[i]} not found`); - } - const appId = parseInt(appIdValue, 10); - const privateKeyBase64 = params.get(keyParams[i]); - if (!privateKeyBase64) { - throw new Error(`Parameter ${keyParams[i]} not found`); - } - // replace literal \n characters with new lines to allow the key to be stored as a - // single line variable. This logic should match how the GitHub Terraform provider - // processes private keys to retain compatibility between the projects - const privateKey = Buffer.from(privateKeyBase64, 'base64').toString().replace(/\\n/g, '\n'); - const installationIdParam = installationIdParams[i]; - const installationIdValue = - installationIdParam && installationIdParam.length > 0 ? params.get(installationIdParam) : undefined; - const installationId = installationIdValue ? parseInt(installationIdValue, 10) : undefined; - credentials.push({ appId, privateKey, installationId }); - } + const credentials = await getGitHubAppCredentialsStore().get(); logger.info(`Loaded ${credentials.length} GitHub App credential(s)`); return credentials; } @@ -137,6 +95,14 @@ export async function getStoredInstallationId(appIndex: number): Promise { + const credential = (await getAppCredentials())[appIndex]; + if (!credential) { + throw new Error(`GitHub App credential at index ${appIndex} not found`); + } + return credential.appId.toString(); +} + export async function createOctokitClient(token: string, ghesApiUrl = ''): Promise { const CustomOctokit = Octokit.plugin(retry, throttling); const ocktokitOptions: OctokitOptions = { diff --git a/lambdas/functions/control-plane/src/github/rate-limit.test.ts b/lambdas/functions/control-plane/src/github/rate-limit.test.ts index d9d18c5921..93e6d24ba0 100644 --- a/lambdas/functions/control-plane/src/github/rate-limit.test.ts +++ b/lambdas/functions/control-plane/src/github/rate-limit.test.ts @@ -1,50 +1,37 @@ -import { ResponseHeaders } from '@octokit/types'; -import { createSingleMetric } from '@aws-github-runner/aws-powertools-util'; import { MetricUnit } from '@aws-lambda-powertools/metrics'; +import { createSingleMetric } from '@aws-github-runner/aws-powertools-util'; +import type { ResponseHeaders } from '@octokit/types'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { getAppId } from './auth'; import { metricGitHubAppRateLimit } from './rate-limit'; -import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; - -process.env.PARAMETER_GITHUB_APP_ID_NAME = 'test'; -vi.mock('@aws-github-runner/aws-ssm-util', async () => { - // Return only what we need without spreading actual - return { - getParameter: vi.fn((name: string) => { - if (name === process.env.PARAMETER_GITHUB_APP_ID_NAME) { - return '1234'; - } else { - return ''; - } - }), - }; -}); -vi.mock('@aws-github-runner/aws-powertools-util', async () => { - // Provide only what's needed without spreading actual - return { - // Mock the logger - logger: { - debug: vi.fn(), - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - }, - // eslint-disable-next-line @typescript-eslint/no-unused-vars - createSingleMetric: vi.fn((name: string, unit: string, value: number, dimensions?: Record) => { - return { - addMetadata: vi.fn(), - }; - }), - }; +vi.mock('./auth', () => ({ + getAppId: vi.fn(), +})); + +vi.mock('@aws-github-runner/aws-powertools-util', () => ({ + logger: { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + createSingleMetric: vi.fn(() => ({ addMetadata: vi.fn() })), +})); + +const cleanEnv = process.env; +const mockedGetAppId = vi.mocked(getAppId); + +beforeEach(() => { + vi.clearAllMocks(); + mockedGetAppId.mockReset(); + mockedGetAppId.mockResolvedValue('1234'); + process.env = { ...cleanEnv }; }); describe('metricGitHubAppRateLimit', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it('should update rate limit metric', async () => { - // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to true + it('updates the rate limit metric', async () => { process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', @@ -53,13 +40,13 @@ describe('metricGitHubAppRateLimit', () => { await metricGitHubAppRateLimit(headers); + expect(mockedGetAppId).toHaveBeenCalledWith(undefined); expect(createSingleMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 10, { AppId: '1234', }); }); - it('should not update rate limit metric', async () => { - // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to false + it('does not update the rate limit metric when disabled', async () => { process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'false'; const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', @@ -68,107 +55,85 @@ describe('metricGitHubAppRateLimit', () => { await metricGitHubAppRateLimit(headers); + expect(mockedGetAppId).not.toHaveBeenCalled(); expect(createSingleMetric).not.toHaveBeenCalled(); }); - it('should not update rate limit metric if headers are undefined', async () => { - // set process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT to true + it('does not update the rate limit metric if headers are undefined', async () => { process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; await metricGitHubAppRateLimit(undefined as unknown as ResponseHeaders); + expect(mockedGetAppId).not.toHaveBeenCalled(); expect(createSingleMetric).not.toHaveBeenCalled(); }); - it('should cache GitHub App ID and only call getParameter once', async () => { - // Reset modules to clear the appIdPromises Map cache - vi.resetModules(); - const { metricGitHubAppRateLimit: freshMetricFunction } = await import('./rate-limit'); - + it('does not update the metric when the app ID lookup fails', async () => { process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; + mockedGetAppId.mockRejectedValueOnce(new Error('credential store unavailable')); const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', 'x-ratelimit-limit': '60', }; - const mockGetParameter = vi.mocked(getParameter); - mockGetParameter.mockClear(); + await expect(metricGitHubAppRateLimit(headers)).resolves.not.toThrow(); - await freshMetricFunction(headers); - await freshMetricFunction(headers); - await freshMetricFunction(headers); - - // getParameter should only be called once due to caching (index 0 cached after first call) - expect(mockGetParameter).toHaveBeenCalledTimes(1); - // split(':')[0] of 'test' is still 'test' - expect(mockGetParameter).toHaveBeenCalledWith(process.env.PARAMETER_GITHUB_APP_ID_NAME); + expect(createSingleMetric).not.toHaveBeenCalled(); }); }); describe('metricGitHubAppRateLimit multi-app', () => { - let freshMetricFunction: typeof metricGitHubAppRateLimit; - let mockGetParam: ReturnType; - - beforeEach(async () => { - // Reset modules to get a clean appIdPromises Map for each test - vi.resetModules(); - - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'app0:app1'; + beforeEach(() => { process.env.ENABLE_METRIC_GITHUB_APP_RATE_LIMIT = 'true'; - - mockGetParam = vi.fn((name: string) => { - if (name === 'app0') return Promise.resolve('1234'); - if (name === 'app1') return Promise.resolve('5678'); - return Promise.resolve(''); + mockedGetAppId.mockImplementation(async (appIndex = 0) => { + if (appIndex === 0) return '1234'; + if (appIndex === 1) return '5678'; + throw new Error(`GitHub App credential at index ${appIndex} not found`); }); - - vi.doMock('@aws-github-runner/aws-ssm-util', () => ({ getParameter: mockGetParam })); - vi.doMock('@aws-github-runner/aws-powertools-util', () => ({ - logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, - createSingleMetric: vi.fn(() => ({ addMetadata: vi.fn() })), - })); - - const mod = await import('./rate-limit'); - freshMetricFunction = mod.metricGitHubAppRateLimit; - }); - - afterEach(() => { - vi.resetModules(); - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'test'; }); - it('should label metric with correct appId for index 0 (primary app)', async () => { - const { createSingleMetric: mockMetric } = await import('@aws-github-runner/aws-powertools-util'); + it('labels the metric with the primary app ID', async () => { const headers: ResponseHeaders = { 'x-ratelimit-remaining': '50', 'x-ratelimit-limit': '5000' }; - await freshMetricFunction(headers, 0); - expect(mockMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 50, { AppId: '1234' }); + + await metricGitHubAppRateLimit(headers, 0); + + expect(mockedGetAppId).toHaveBeenCalledWith(0); + expect(createSingleMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 50, { + AppId: '1234', + }); }); - it('should label metric with correct appId for index 1 (additional app)', async () => { - const { createSingleMetric: mockMetric } = await import('@aws-github-runner/aws-powertools-util'); + it('labels the metric with an additional app ID', async () => { const headers: ResponseHeaders = { 'x-ratelimit-remaining': '100', 'x-ratelimit-limit': '5000' }; - await freshMetricFunction(headers, 1); - expect(mockMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 100, { AppId: '5678' }); + + await metricGitHubAppRateLimit(headers, 1); + + expect(mockedGetAppId).toHaveBeenCalledWith(1); + expect(createSingleMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 100, { + AppId: '5678', + }); }); - it('should default to index 0 when no appIndex is passed', async () => { - const { createSingleMetric: mockMetric } = await import('@aws-github-runner/aws-powertools-util'); + it('defaults to the primary app when no app index is passed', async () => { const headers: ResponseHeaders = { 'x-ratelimit-remaining': '75', 'x-ratelimit-limit': '5000' }; - await freshMetricFunction(headers); - expect(mockMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 75, { AppId: '1234' }); + + await metricGitHubAppRateLimit(headers); + + expect(mockedGetAppId).toHaveBeenCalledWith(undefined); + expect(createSingleMetric).toHaveBeenCalledWith('GitHubAppRateLimitRemaining', MetricUnit.Count, 75, { + AppId: '1234', + }); }); - it('should cache per index and call getParameter separately for each index', async () => { + it('forwards each app index to the shared credential accessor', async () => { const headers: ResponseHeaders = { 'x-ratelimit-remaining': '10', 'x-ratelimit-limit': '5000' }; - // Two calls with index 1, then one with index 0 - await freshMetricFunction(headers, 1); - await freshMetricFunction(headers, 1); - await freshMetricFunction(headers, 0); + await metricGitHubAppRateLimit(headers, 1); + await metricGitHubAppRateLimit(headers, 1); + await metricGitHubAppRateLimit(headers, 0); - // getParameter should be called exactly once per distinct index - expect(mockGetParam).toHaveBeenCalledTimes(2); - expect(mockGetParam).toHaveBeenCalledWith('app1'); - expect(mockGetParam).toHaveBeenCalledWith('app0'); + expect(mockedGetAppId).toHaveBeenNthCalledWith(1, 1); + expect(mockedGetAppId).toHaveBeenNthCalledWith(2, 1); + expect(mockedGetAppId).toHaveBeenNthCalledWith(3, 0); }); }); diff --git a/lambdas/functions/control-plane/src/github/rate-limit.ts b/lambdas/functions/control-plane/src/github/rate-limit.ts index df2372a255..b5559a5d82 100644 --- a/lambdas/functions/control-plane/src/github/rate-limit.ts +++ b/lambdas/functions/control-plane/src/github/rate-limit.ts @@ -2,22 +2,8 @@ import { ResponseHeaders } from '@octokit/types'; import { createSingleMetric, logger } from '@aws-github-runner/aws-powertools-util'; import { MetricUnit } from '@aws-lambda-powertools/metrics'; import yn from 'yn'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -// Cache the app ID per app index to avoid repeated SSM calls across Lambda invocations. -// In multi-app mode PARAMETER_GITHUB_APP_ID_NAME is a ':'-joined list of SSM param names, -// one per app in app-index order; index 0 is the primary app. -const appIdPromises = new Map>(); - -async function getAppId(appIndex = 0): Promise { - let cached = appIdPromises.get(appIndex); - if (!cached) { - const paramName = process.env.PARAMETER_GITHUB_APP_ID_NAME.split(':')[appIndex]; - cached = getParameter(paramName); - appIdPromises.set(appIndex, cached); - } - return cached; -} +import { getAppId } from './auth'; export async function metricGitHubAppRateLimit(headers: ResponseHeaders, appIndex?: number): Promise { try { diff --git a/lambdas/functions/control-plane/src/lambda.test.ts b/lambdas/functions/control-plane/src/lambda.test.ts index 4d970f0f22..0205317728 100644 --- a/lambdas/functions/control-plane/src/lambda.test.ts +++ b/lambdas/functions/control-plane/src/lambda.test.ts @@ -1,12 +1,20 @@ import { captureLambdaHandler, logger } from '@aws-github-runner/aws-powertools-util'; +import { getRunnerConfigStore, type RunnerConfigStore } from '@aws-github-runner/storage-providers'; import { Context, SQSEvent, SQSRecord } from 'aws-lambda'; -import { addMiddleware, adjustPool, scaleDownHandler, scaleUpHandler, ssmHousekeeper, jobRetryCheck } from './lambda'; +import { + addMiddleware, + adjustPool, + jobRetryCheck, + runnerConfigHousekeeper, + scaleDownHandler, + scaleUpHandler, + ssmHousekeeper, +} from './lambda'; import { adjust } from './pool/pool'; import { scaleDown } from './scale-runners/scale-down'; import { scaleUp } from './scale-runners/scale-up'; import type { ActionRequestMessage } from './scale-runners/types'; -import { cleanSSMTokens } from './scale-runners/ssm-housekeeper'; import { checkAndRetryJob } from './scale-runners/job-retry'; import { describe, it, expect, vi, MockedFunction, beforeEach } from 'vitest'; @@ -64,10 +72,16 @@ const context: Context = { vi.mock('./pool/pool'); vi.mock('./scale-runners/scale-down'); vi.mock('./scale-runners/scale-up'); -vi.mock('./scale-runners/ssm-housekeeper'); vi.mock('./scale-runners/job-retry'); vi.mock('@aws-github-runner/aws-powertools-util'); -vi.mock('@aws-github-runner/aws-ssm-util'); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getRunnerConfigStore: vi.fn(), +})); + +const runnerConfigStore = { + create: vi.fn(), + houseKeeper: vi.fn(), +} satisfies RunnerConfigStore; describe('Test scale up lambda wrapper.', () => { it('Do not handle empty record sets.', async () => { @@ -296,22 +310,35 @@ describe('Test middleware', () => { }); }); -describe('Test ssm housekeeper lambda wrapper.', () => { +describe('Test runner config housekeeper lambda wrapper.', () => { + beforeEach(() => { + vi.mocked(getRunnerConfigStore).mockReturnValue(runnerConfigStore); + }); + it('Invoke without errors.', async () => { - vi.mocked(cleanSSMTokens).mockResolvedValue(); + runnerConfigStore.houseKeeper.mockResolvedValue(); + + await expect(runnerConfigHousekeeper({}, context)).resolves.not.toThrow(); + expect(getRunnerConfigStore).toHaveBeenCalledOnce(); + expect(runnerConfigStore.houseKeeper).toHaveBeenCalledOnce(); + }); - process.env.SSM_CLEANUP_CONFIG = JSON.stringify({ - dryRun: false, - minimumDaysOld: 1, - tokenPath: '/path/to/tokens/', + it('Errors not throws.', async () => { + runnerConfigStore.houseKeeper.mockRejectedValue(new Error()); + await expect(runnerConfigHousekeeper({}, context)).resolves.not.toThrow(); + }); + + it('does not catch provider construction errors', async () => { + const error = new Error('Invalid provider configuration'); + vi.mocked(getRunnerConfigStore).mockImplementation(() => { + throw error; }); - await expect(ssmHousekeeper({}, context)).resolves.not.toThrow(); + await expect(runnerConfigHousekeeper({}, context)).rejects.toBe(error); }); - it('Errors not throws.', async () => { - vi.mocked(cleanSSMTokens).mockRejectedValue(new Error()); - await expect(ssmHousekeeper({}, context)).resolves.not.toThrow(); + it('keeps the legacy Terraform handler alias', () => { + expect(ssmHousekeeper).toBe(runnerConfigHousekeeper); }); }); diff --git a/lambdas/functions/control-plane/src/lambda.ts b/lambdas/functions/control-plane/src/lambda.ts index d229a0350e..70331f5858 100644 --- a/lambdas/functions/control-plane/src/lambda.ts +++ b/lambdas/functions/control-plane/src/lambda.ts @@ -1,13 +1,13 @@ import middy from '@middy/core'; import { logger, setContext } from '@aws-github-runner/aws-powertools-util'; import { captureLambdaHandler, tracer } from '@aws-github-runner/aws-powertools-util'; +import { getRunnerConfigStore } from '@aws-github-runner/storage-providers'; import { Context, type SQSBatchItemFailure, type SQSBatchResponse, SQSEvent } from 'aws-lambda'; import { PoolEvent, adjust } from './pool/pool'; import { scaleDown } from './scale-runners/scale-down'; import { scaleUp } from './scale-runners/scale-up'; import type { ActionRequestMessage, ActionRequestMessageSQS } from './scale-runners/types'; -import { SSMCleanupOptions, cleanSSMTokens } from './scale-runners/ssm-housekeeper'; import { checkAndRetryJob } from './scale-runners/job-retry'; export async function scaleUpHandler(event: SQSEvent, context: Context): Promise { @@ -114,22 +114,25 @@ export const addMiddleware = () => { middy(scaleUpHandler).use(handler); middy(scaleDownHandler).use(handler); middy(adjustPool).use(handler); - middy(ssmHousekeeper).use(handler); + middy(runnerConfigHousekeeper).use(handler); }; addMiddleware(); -export async function ssmHousekeeper(event: unknown, context: Context): Promise { +export async function runnerConfigHousekeeper(event: unknown, context: Context): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); - const config = JSON.parse(process.env.SSM_CLEANUP_CONFIG) as SSMCleanupOptions; + const runnerConfigStore = getRunnerConfigStore(); try { - await cleanSSMTokens(config); + await runnerConfigStore.houseKeeper(); } catch (e) { logger.error(`${(e as Error).message}`, { error: e as Error }); } } +/** @deprecated Use runnerConfigHousekeeper. Kept for existing Terraform handler configuration. */ +export const ssmHousekeeper = runnerConfigHousekeeper; + export async function jobRetryCheck(event: SQSEvent, context: Context): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); diff --git a/lambdas/functions/control-plane/src/modules.d.ts b/lambdas/functions/control-plane/src/modules.d.ts index d5157ccb37..af537afcba 100644 --- a/lambdas/functions/control-plane/src/modules.d.ts +++ b/lambdas/functions/control-plane/src/modules.d.ts @@ -13,13 +13,9 @@ declare namespace NodeJS { MINIMUM_RUNNING_TIME_IN_MINUTES: string; PARAMETER_GITHUB_APP_CLIENT_ID_NAME: string; PARAMETER_GITHUB_APP_CLIENT_SECRET_NAME: string; - PARAMETER_GITHUB_APP_ID_NAME: string; - PARAMETER_GITHUB_APP_KEY_BASE64_NAME: string; RUNNER_OWNER: string; COMPUTE_PROVIDER_TYPE?: string; SCALE_DOWN_CONFIG: string; - SSM_TOKEN_PATH: string; - SSM_CLEANUP_CONFIG: string; SUBNET_IDS: string; INSTANCE_TYPES: string; INSTANCE_TARGET_CAPACITY_TYPE: 'on-demand' | 'spot'; diff --git a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts index e519e412e4..a949afae39 100644 --- a/lambdas/functions/control-plane/src/pool/pool-contract.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool-contract.test.ts @@ -20,7 +20,6 @@ vi.mock('../github/auth', () => ({ vi.mock('../scale-runners/github-runner', () => ({ createStartRunnerConfig: vi.fn(), getGitHubEnterpriseApiUrl: vi.fn(), - validateSsmParameterStoreTags: vi.fn(), })); const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); @@ -48,7 +47,6 @@ const computeProviders = providerTypes.map((type) => ({ beforeEach(() => { vi.clearAllMocks(); process.env = { ...cleanEnv }; - mockedAppAuth.mockResolvedValue({ type: 'app', token: 'app-token', appId: 1, expiresAt: 'some-date' }); mockedInstallationAuth.mockResolvedValue({ type: 'token', @@ -62,7 +60,6 @@ beforeEach(() => { }); mockedCreateClient.mockResolvedValue(githubClient); vi.mocked(githubRunner.getGitHubEnterpriseApiUrl).mockReturnValue({ ghesApiUrl: '', ghesBaseUrl: '' }); - vi.mocked(githubRunner.validateSsmParameterStoreTags).mockReturnValue([]); vi.mocked(githubClient.apps.getOrgInstallation).mockResolvedValue({ data: { id: 2 } } as never); vi.mocked(githubClient.paginate).mockResolvedValue([]); }); diff --git a/lambdas/functions/control-plane/src/pool/pool.test.ts b/lambdas/functions/control-plane/src/pool/pool.test.ts index 8372ab6403..05e021bded 100644 --- a/lambdas/functions/control-plane/src/pool/pool.test.ts +++ b/lambdas/functions/control-plane/src/pool/pool.test.ts @@ -1,9 +1,9 @@ -import type { Octokit } from '@octokit/rest'; import { defaultComputeProvider } from '@aws-github-runner/compute-providers/provider-types'; +import type { Octokit } from '@octokit/rest'; import { beforeEach, describe, expect, it, vi } from 'vitest'; -import * as ghAuth from '../github/auth'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; +import * as ghAuth from '../github/auth'; import * as githubRunner from '../scale-runners/github-runner'; import { adjust } from './pool'; import type { PoolComputeProvider } from './pool-provider'; @@ -31,7 +31,6 @@ vi.mock('../scale-runners/github-runner', () => ({ ghesApiUrl: '', ghesBaseUrl: '', }), - validateSsmParameterStoreTags: vi.fn().mockReturnValue([]), })); const mockedAppAuth = vi.mocked(ghAuth.createGithubAppAuth); diff --git a/lambdas/functions/control-plane/src/pool/pool.ts b/lambdas/functions/control-plane/src/pool/pool.ts index da5d2ee9b1..21e91adebc 100644 --- a/lambdas/functions/control-plane/src/pool/pool.ts +++ b/lambdas/functions/control-plane/src/pool/pool.ts @@ -10,7 +10,7 @@ import { getStoredInstallationId, } from '../github/auth'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; -import { getGitHubEnterpriseApiUrl, validateSsmParameterStoreTags } from '../scale-runners/github-runner'; +import { getGitHubEnterpriseApiUrl } from '../scale-runners/github-runner'; import type { RunnerStatus } from './pool-provider'; const logger = createChildLogger('pool'); @@ -31,16 +31,10 @@ export async function adjust(event: PoolEvent): Promise { const runnerGroup = process.env.RUNNER_GROUP_NAME || ''; const runnerNamePrefix = process.env.RUNNER_NAME_PREFIX || ''; const environment = process.env.ENVIRONMENT; - const ssmTokenPath = process.env.SSM_TOKEN_PATH; - const ssmConfigPath = process.env.SSM_CONFIG_PATH || ''; const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); const enableJitConfig = yn(process.env.ENABLE_JIT_CONFIG, { default: ephemeral }); const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false }); const runnerOwner = process.env.RUNNER_OWNER; - const ssmParameterStoreTags: { Key: string; Value: string }[] = - process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== '' - ? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS) - : []; // -1 disables the maximum check, matching the scale-up lambda's semantics. Defaults to unlimited // when unset so the pool keeps its previous behavior on stacks that do not provide the variable. const maximumRunners = parseInt(process.env.RUNNERS_MAXIMUM_COUNT || '-1'); @@ -103,9 +97,6 @@ export async function adjust(event: PoolEvent): Promise { runnerNamePrefix, runnerType: 'Org', disableAutoUpdate: disableAutoUpdate, - ssmTokenPath, - ssmConfigPath, - ssmParameterStoreTags, }, numberOfRunners: topUp, githubInstallationClient, diff --git a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts index 745c66770a..2ec9ee8aaf 100644 --- a/lambdas/functions/control-plane/src/scale-runners/github-runner.ts +++ b/lambdas/functions/control-plane/src/scale-runners/github-runner.ts @@ -1,5 +1,10 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; -import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { + getRunnerGroupCacheStore, + getRunnerConfigStore, + type RunnerConfigMetadata, + type RunnerConfigStore, +} from '@aws-github-runner/storage-providers'; import { Octokit } from '@octokit/rest'; import { getStoredInstallationId } from '../github/auth'; @@ -14,7 +19,7 @@ export interface GitHubRunnerMetadata { } export interface StartRunnerConfigOptions { - getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[]; + getRunnerConfigMetadata?: (runnerId: string) => RunnerConfigMetadata[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } @@ -51,37 +56,6 @@ function quoteShellArg(value: string): string { return `'${value.replace(/'/g, `'\\''`)}'`; } -export function validateSsmParameterStoreTags(tagsJson: string): { Key: string; Value: string }[] { - try { - const tags = JSON.parse(tagsJson); - - if (!Array.isArray(tags)) { - throw new Error('Tags must be an array'); - } - - if (tags.length === 0) { - return []; - } - - tags.forEach((tag, index) => { - if (typeof tag !== 'object' || tag === null) { - throw new Error(`Tag at index ${index} must be an object`); - } - if (!tag.Key || typeof tag.Key !== 'string' || tag.Key.trim() === '') { - throw new Error(`Tag at index ${index} has missing or invalid 'Key' property`); - } - if (!Object.prototype.hasOwnProperty.call(tag, 'Value') || typeof tag.Value !== 'string') { - throw new Error(`Tag at index ${index} has missing or invalid 'Value' property`); - } - }); - - return tags; - } catch (err) { - logger.error('Invalid SSM_PARAMETER_STORE_TAGS format', { error: err }); - throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(err as Error).message}`); - } -} - async function getGithubRunnerRegistrationToken(githubRunnerConfig: CreateGitHubRunnerConfig, ghClient: Octokit) { const registrationToken = githubRunnerConfig.runnerType === 'Org' @@ -186,39 +160,30 @@ export async function getRunnerGroupId( // if the runnerType is Repo, then runnerGroupId is default to 1 let runnerGroupId: number | undefined = 1; if (githubRunnerConfig.runnerType === 'Org' && githubRunnerConfig.runnerGroup !== undefined) { - let runnerGroup: string | undefined; - // check if runner group id is already stored in SSM Parameter Store and - // use it if it exists to avoid API call to GitHub + const runnerGroupCacheStore = getRunnerGroupCacheStore(); + let cachedRunnerGroupId: number | undefined; + // Use a cached runner group id when available to avoid an API call to GitHub. try { - runnerGroup = await getParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - ); + cachedRunnerGroupId = await runnerGroupCacheStore.get(githubRunnerConfig.runnerGroup); } catch (err) { logger.debug('Handling error:', err as Error); - logger.warn( - `SSM Parameter "${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}" - for Runner group ${githubRunnerConfig.runnerGroup} does not exist`, - ); + logger.warn(`Cached id for runner group ${githubRunnerConfig.runnerGroup} does not exist`); } - if (runnerGroup === undefined) { + if (cachedRunnerGroupId === undefined) { // get runner group id from GitHub runnerGroupId = await getRunnerGroupByName(ghClient, githubRunnerConfig); - // store runner group id in SSM + // cache the runner group id try { - await putParameter( - `${githubRunnerConfig.ssmConfigPath}/runner-group/${githubRunnerConfig.runnerGroup}`, - runnerGroupId.toString(), - false, - { - tags: githubRunnerConfig.ssmParameterStoreTags, - }, - ); + await runnerGroupCacheStore.create({ + runnerGroupName: githubRunnerConfig.runnerGroup, + runnerGroupId, + }); } catch (err) { - logger.debug('Error storing runner group id in SSM Parameter Store', err as Error); + logger.debug('Error storing runner group id in cache', err as Error); throw err; } } else { - runnerGroupId = parseInt(runnerGroup); + runnerGroupId = cachedRunnerGroupId; } } return runnerGroupId; @@ -250,18 +215,20 @@ export async function createStartRunnerConfig( ghClient: Octokit, options: StartRunnerConfigOptions = {}, ): Promise { + const runnerConfigStore = getRunnerConfigStore(); if (githubRunnerConfig.enableJitConfig && githubRunnerConfig.ephemeral) { - return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, options); + return await createJitConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); } else { - return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, options); + return await createRegistrationTokenConfig(githubRunnerConfig, runnerIds, ghClient, runnerConfigStore, options); } } -function addDelay(runnerIds: string[]) { +function addDelay(runnerIds: string[], runnerConfigStore: RunnerConfigStore) { const delay = async (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - const ssmParameterStoreMaxThroughput = 40; - const isDelay = runnerIds.length >= ssmParameterStoreMaxThroughput; - return { isDelay, delay }; + const maxWritesPerSecond = runnerConfigStore.maxWritesPerSecond; + const isDelay = maxWritesPerSecond !== undefined && runnerIds.length >= maxWritesPerSecond; + const delayMilliseconds = maxWritesPerSecond === undefined ? 0 : 1000 / maxWritesPerSecond; + return { isDelay, delay, delayMilliseconds }; } /** @@ -273,9 +240,10 @@ async function createRegistrationTokenConfig( githubRunnerConfig: CreateGitHubRunnerConfig, runnerIds: string[], ghClient: Octokit, + runnerConfigStore: RunnerConfigStore, options: StartRunnerConfigOptions, ): Promise { - const { isDelay, delay } = addDelay(runnerIds); + const { isDelay, delay, delayMilliseconds } = addDelay(runnerIds, runnerConfigStore); const token = await getGithubRunnerRegistrationToken(githubRunnerConfig, ghClient); const runnerServiceConfig = generateRunnerServiceConfig(githubRunnerConfig, token); @@ -284,12 +252,13 @@ async function createRegistrationTokenConfig( }); for (const runnerId of runnerIds) { - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerServiceConfig.join(' '), true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], - }); + await runnerConfigStore.create( + { runnerId, value: runnerServiceConfig.join(' ') }, + { metadata: options.getRunnerConfigMetadata?.(runnerId) }, + ); if (isDelay) { - // Delay to prevent AWS ssm rate limits by being within the max throughput limit - await delay(25); + // Delay to stay within the selected store's maximum write throughput. + await delay(delayMilliseconds); } } @@ -306,10 +275,11 @@ async function createJitConfig( githubRunnerConfig: CreateGitHubRunnerConfig, runnerIds: string[], ghClient: Octokit, + runnerConfigStore: RunnerConfigStore, options: StartRunnerConfigOptions, ): Promise { const runnerGroupId = await getRunnerGroupId(githubRunnerConfig, ghClient); - const { isDelay, delay } = addDelay(runnerIds); + const { isDelay, delay, delayMilliseconds } = addDelay(runnerIds, runnerConfigStore); const runnerLabels = githubRunnerConfig.runnerLabels.split(','); const failedRunnerIds: string[] = []; @@ -347,16 +317,17 @@ async function createJitConfig( runnerLabels, }); - // store jit config in ssm parameter store + // Store the JIT config through the selected storage provider. logger.debug('Runner JIT config for ephemeral runner generated.', { instance: runnerId, }); - await putParameter(`${githubRunnerConfig.ssmTokenPath}/${runnerId}`, runnerConfig.data.encoded_jit_config, true, { - tags: [...(options.getSsmParameterTags?.(runnerId) ?? []), ...githubRunnerConfig.ssmParameterStoreTags], - }); + await runnerConfigStore.create( + { runnerId, value: runnerConfig.data.encoded_jit_config }, + { metadata: options.getRunnerConfigMetadata?.(runnerId) }, + ); if (isDelay) { - // Delay to prevent AWS ssm rate limits by being within the max throughput limit - await delay(25); + // Delay to stay within the selected store's maximum write throughput. + await delay(delayMilliseconds); } } catch (error) { failedRunnerIds.push(runnerId); diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts index cccab60a98..c88d70ff2c 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.test.ts @@ -1,9 +1,13 @@ -import { PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; -import { mockClient } from 'aws-sdk-client-mock'; -import 'aws-sdk-client-mock-jest/vitest'; -// Using vi.mocked instead of jest-mock +import { defaultComputeProvider } from '@aws-github-runner/compute-providers/provider-types'; +import { + type RunnerConfigStore, + type RunnerGroupCacheStore, + getRunnerConfigStore, + getRunnerGroupCacheStore, +} from '@aws-github-runner/storage-providers'; +import type { Octokit } from '@octokit/rest'; import nock from 'nock'; -import { performance } from 'perf_hooks'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; import { controlPlaneProviderRegistry } from '../control-plane-providers'; import * as ghAuth from '../github/auth'; @@ -16,10 +20,6 @@ import type { CreateScaleUpRunnersInput, ScaleUpComputeProvider, } from './types'; -import { defaultComputeProvider } from '@aws-github-runner/compute-providers/provider-types'; -import { getParameter } from '@aws-github-runner/aws-ssm-util'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; -import type { Octokit } from '@octokit/rest'; const mockOctokit = { paginate: vi.fn(), @@ -54,8 +54,21 @@ const createRunner = vi.fn<(input: TestRunnerCreationInput) => Promise Promise>(); const mockCreateRunner = vi.mocked(createRunner); const mockListRunners = vi.mocked(listRunners); -const mockSSMClient = mockClient(SSMClient); -const mockSSMgetParameter = vi.mocked(getParameter); +const mockGetRunnerConfigStore = vi.mocked(getRunnerConfigStore); +const mockGetRunnerGroupCacheStore = vi.mocked(getRunnerGroupCacheStore); +const mockRunnerConfigCreate = vi.fn(); +const mockRunnerConfigHouseKeeper = vi.fn(); +const mockRunnerGroupCacheGet = vi.fn(); +const mockRunnerGroupCacheCreate = vi.fn(); +const mockRunnerConfigStore: RunnerConfigStore = { + maxWritesPerSecond: 40, + create: mockRunnerConfigCreate, + houseKeeper: mockRunnerConfigHouseKeeper, +}; +const mockRunnerGroupCacheStore: RunnerGroupCacheStore = { + get: mockRunnerGroupCacheGet, + create: mockRunnerGroupCacheCreate, +}; const mockPublishRetryMessage = vi.mocked(publishRetryMessage); const testProviderState = { provider: 'test' }; const mockComputeProvider = { @@ -86,16 +99,10 @@ vi.mock('../github/auth', async () => ({ getStoredInstallationId: vi.fn().mockResolvedValue(undefined), })); -vi.mock('@aws-github-runner/aws-ssm-util', async () => { - const actual = (await vi.importActual( - '@aws-github-runner/aws-ssm-util', - )) as typeof import('@aws-github-runner/aws-ssm-util'); - - return { - ...actual, - getParameter: vi.fn(), - }; -}); +vi.mock('@aws-github-runner/storage-providers', () => ({ + getRunnerConfigStore: vi.fn(), + getRunnerGroupCacheStore: vi.fn(), +})); vi.mock('./job-retry', () => ({ publishRetryMessage: vi.fn(), @@ -140,7 +147,6 @@ let expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; function setDefaults() { process.env = { ...cleanEnv }; - process.env.PARAMETER_GITHUB_APP_ID_NAME = 'github-app-id'; process.env.GITHUB_APP_KEY_BASE64 = 'TEST_CERTIFICATE_DATA'; process.env.GITHUB_APP_ID = '1337'; process.env.GITHUB_APP_CLIENT_ID = 'TEST_CLIENT_ID'; @@ -168,7 +174,7 @@ async function createTestProviderRunners(input: CreateScaleUpRunnersInput [{ Key: 'RunnerId', Value: runnerId }], + getRunnerConfigMetadata: (runnerId) => [{ key: 'RunnerId', value: runnerId }], }, ); } catch { @@ -187,8 +193,12 @@ beforeEach(() => { vi.resetModules(); vi.clearAllMocks(); setDefaults(); - - defaultSSMGetParameterMockImpl(); + mockGetRunnerConfigStore.mockReturnValue(mockRunnerConfigStore); + mockGetRunnerGroupCacheStore.mockReturnValue(mockRunnerGroupCacheStore); + mockRunnerConfigCreate.mockResolvedValue(); + mockRunnerConfigHouseKeeper.mockResolvedValue(); + mockRunnerGroupCacheGet.mockResolvedValue(1); + mockRunnerGroupCacheCreate.mockResolvedValue(); defaultOctokitMockImpl(); mockedResolveCapability.mockReturnValue(() => mockComputeProvider); @@ -268,12 +278,9 @@ describe('scaleUp with GHES', () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; process.env.RUNNER_NAME_PREFIX = 'unit-test-'; process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; process.env.RUNNER_LABELS = 'label1,label2'; expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); }); it('does not create a token when maximum runners has been reached', async () => { @@ -328,9 +335,7 @@ describe('scaleUp with GHES', () => { it('returns a retryable failure if runner group lookup fails for ephemeral runners', async () => { process.env.RUNNER_GROUP_NAME = 'test-runner-group'; - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); + mockRunnerGroupCacheGet.mockRejectedValue(new Error('Cache entry not found')); await expect(scaleUpModule.scaleUp(TEST_DATA)).resolves.toEqual(['foobar']); @@ -345,24 +350,27 @@ describe('scaleUp with GHES', () => { expect(createRunner).not.toHaveBeenCalled(); }); - it('create SSM parameter for runner group id if it does not exist', async () => { - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); + it('caches the runner group id if it does not exist', async () => { + mockRunnerGroupCacheGet.mockResolvedValue(undefined); + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerGroupCacheGet).toHaveBeenCalledWith('Default'); expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, - Value: '1', - Type: 'String', + expect(mockRunnerGroupCacheCreate).toHaveBeenCalledWith({ + runnerGroupName: 'Default', + runnerGroupId: 1, }); + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(1); }); - it('Does not create SSM parameter for runner group id if it exists', async () => { + it('reuses a cached runner group id', async () => { await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerGroupCacheGet).toHaveBeenCalledWith('Default'); expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); + expect(mockRunnerGroupCacheCreate).not.toHaveBeenCalled(); + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(1); }); it('create start runner config for ephemeral runners ', async () => { @@ -375,17 +383,10 @@ describe('scaleUp with GHES', () => { runner_group_id: 1, labels: ['label1', 'label2'], }); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_ORG', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-12345', value: 'TEST_JIT_CONFIG_ORG' }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); }); it('create start runner config for non-ephemeral runners ', async () => { @@ -394,19 +395,15 @@ describe('scaleUp with GHES', () => { await scaleUpModule.scaleUp(TEST_DATA); expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + - '--labels label1,label2 --runnergroup Default', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { + runnerId: 'i-12345', + value: + '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + + '--labels label1,label2 --runnergroup Default', + }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); }); it('quotes runner labels with semicolon separators in non-ephemeral runner config', async () => { @@ -421,19 +418,15 @@ describe('scaleUp with GHES', () => { }, ]); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + - "--labels 'label1,label2,ghr-provider-capability:intel;amd' --runnergroup Default", - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { + runnerId: 'i-12345', + value: + '--url https://github.enterprise.something/Codertocat --token 1234abcd ' + + "--labels 'label1,label2,ghr-provider-capability:intel;amd' --runnergroup Default", + }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); }); it('should create JIT config for all remaining instances even when GitHub API fails for one instance', async () => { @@ -493,23 +486,18 @@ describe('scaleUp with GHES', () => { labels: ['label1', 'label2'], }); - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-1', - Type: 'SecureString', - Tags: [{ Key: 'RunnerId', Value: 'i-instance-1' }], - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-3', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-3', - Type: 'SecureString', - Tags: [{ Key: 'RunnerId', Value: 'i-instance-3' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-instance-1', value: 'TEST_JIT_CONFIG_unit-test-i-instance-1' }, + { metadata: [{ key: 'RunnerId', value: 'i-instance-1' }] }, + ); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-instance-3', value: 'TEST_JIT_CONFIG_unit-test-i-instance-3' }, + { metadata: [{ key: 'RunnerId', value: 'i-instance-3' }] }, + ); + expect(mockRunnerConfigCreate).not.toHaveBeenCalledWith( + expect.objectContaining({ runnerId: 'i-instance-2' }), + expect.anything(), + ); }); it('should handle retryable errors with error handling logic', async () => { @@ -545,16 +533,14 @@ describe('scaleUp with GHES', () => { await scaleUpModule.scaleUp(TEST_DATA); - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', - Type: 'SecureString', - Tags: [{ Key: 'RunnerId', Value: 'i-instance-2' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-instance-2', value: 'TEST_JIT_CONFIG_unit-test-i-instance-2' }, + { metadata: [{ key: 'RunnerId', value: 'i-instance-2' }] }, + ); + expect(mockRunnerConfigCreate).not.toHaveBeenCalledWith( + expect.objectContaining({ runnerId: 'i-instance-1' }), + expect.anything(), + ); }); it('should handle non-retryable 4xx errors gracefully', async () => { @@ -591,79 +577,62 @@ describe('scaleUp with GHES', () => { await scaleUpModule.scaleUp(TEST_DATA); - expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-2', - Value: 'TEST_JIT_CONFIG_unit-test-i-instance-2', - Type: 'SecureString', - Tags: [{ Key: 'RunnerId', Value: 'i-instance-2' }], - }); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-instance-1', - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-instance-2', value: 'TEST_JIT_CONFIG_unit-test-i-instance-2' }, + { metadata: [{ key: 'RunnerId', value: 'i-instance-2' }] }, + ); + expect(mockRunnerConfigCreate).not.toHaveBeenCalledWith( + expect.objectContaining({ runnerId: 'i-instance-1' }), + expect.anything(), + ); }); it.each(RUNNER_TYPES)( - 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', + 'paces 40 runner-config writes at the store throughput limit for %s runners', async (type: RunnerLifecycle) => { process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; process.env.RUNNERS_MAXIMUM_COUNT = '40'; + const instances = Array.from({ length: 40 }, (_, index) => `i-${index + 1}`); mockCreateRunner.mockImplementation(async () => { return createRunnerResult(instances); }); mockListRunners.mockImplementation(async () => { return []; }); - const startTime = performance.now(); - const instances = [ - 'i-1234', - 'i-5678', - 'i-5567', - 'i-5569', - 'i-5561', - 'i-5560', - 'i-5566', - 'i-5536', - 'i-5526', - 'i-5516', - 'i-122', - 'i-123', - 'i-124', - 'i-125', - 'i-126', - 'i-127', - 'i-128', - 'i-129', - 'i-130', - 'i-131', - 'i-132', - 'i-133', - 'i-134', - 'i-135', - 'i-136', - 'i-137', - 'i-138', - 'i-139', - 'i-140', - 'i-141', - 'i-142', - 'i-143', - 'i-144', - 'i-145', - 'i-146', - 'i-147', - 'i-148', - 'i-149', - 'i-150', - 'i-151', - ]; - await scaleUpModule.scaleUp(TEST_DATA); - const endTime = performance.now(); - expect(endTime - startTime).toBeGreaterThan(1000); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); + + const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout').mockImplementation((callback) => { + callback(); + return 0 as unknown as NodeJS.Timeout; + }); + try { + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(40); + expect(setTimeoutSpy).toHaveBeenCalledTimes(40); + expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 25); + } finally { + setTimeoutSpy.mockRestore(); + } }, - 10000, ); + + it('does not pace 39 runner-config writes below the store throughput limit', async () => { + process.env.ENABLE_EPHEMERAL_RUNNERS = 'false'; + process.env.RUNNERS_MAXIMUM_COUNT = '39'; + const instances = Array.from({ length: 39 }, (_, index) => `i-${index + 1}`); + mockCreateRunner.mockResolvedValue(createRunnerResult(instances)); + mockListRunners.mockResolvedValue([]); + const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout'); + + try { + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(39); + expect(setTimeoutSpy).not.toHaveBeenCalled(); + } finally { + setTimeoutSpy.mockRestore(); + } + }); }); describe('dynamic label groups', () => { @@ -674,7 +643,6 @@ describe('scaleUp with GHES', () => { process.env.RUNNER_LABELS = 'base-label'; process.env.RUNNER_NAME_PREFIX = 'unit-test'; expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); mockResolveLabelsForRunners.mockImplementation(async (labels) => ({ runnerLabels: labels.filter((label) => label.startsWith('ghr-')), @@ -1150,8 +1118,6 @@ describe('scaleUp with public GH', () => { describe('on repo level', () => { beforeEach(() => { - mockSSMClient.reset(); - process.env.ENABLE_ORGANIZATION_RUNNERS = 'false'; process.env.RUNNER_NAME_PREFIX = 'unit-test'; expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; @@ -1195,44 +1161,32 @@ describe('scaleUp with public GH', () => { it('creates a ephemeral runner with JIT config.', async () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; await scaleUpModule.scaleUp(TEST_DATA); expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); expect(createRunner).toBeCalledWith(expectedRunnerParams); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_REPO', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-12345', value: 'TEST_JIT_CONFIG_REPO' }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); + expect(mockGetRunnerGroupCacheStore).not.toHaveBeenCalled(); }); it('creates a ephemeral runner with registration token.', async () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; process.env.ENABLE_JIT_CONFIG = 'false'; process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; await scaleUpModule.scaleUp(TEST_DATA); expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); expect(createRunner).toBeCalledWith(expectedRunnerParams); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { + runnerId: 'i-12345', + value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --ephemeral', + }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); }); it('JIT config is ignored for non-ephemeral runners.', async () => { @@ -1240,22 +1194,18 @@ describe('scaleUp with public GH', () => { process.env.ENABLE_JIT_CONFIG = 'true'; process.env.ENABLE_JOB_QUEUED_CHECK = 'false'; process.env.RUNNER_LABELS = 'jit'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; await scaleUpModule.scaleUp(TEST_DATA); expect(mockOctokit.actions.getJobForWorkflowRun).not.toBeCalled(); expect(createRunner).toBeCalledWith(expectedRunnerParams); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { + runnerId: 'i-12345', + value: '--url https://github.com/Codertocat/hello-world --token 1234abcd --labels jit', + }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); + expect(mockGetRunnerGroupCacheStore).not.toHaveBeenCalled(); }); it('creates a ephemeral runner after checking job is queued.', async () => { @@ -1534,12 +1484,9 @@ describe('scaleUp with Github Data Residency', () => { process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; process.env.RUNNER_NAME_PREFIX = 'unit-test-'; process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; process.env.RUNNER_LABELS = 'label1,label2'; expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); }); it('does not create a token when maximum runners has been reached', async () => { @@ -1582,24 +1529,27 @@ describe('scaleUp with Github Data Residency', () => { expect(createRunner).not.toHaveBeenCalled(); }); - it('create SSM parameter for runner group id if it does not exist', async () => { - mockSSMgetParameter.mockImplementation(async () => { - throw new Error('ParameterNotFound'); - }); + it('caches the runner group id if it does not exist', async () => { + mockRunnerGroupCacheGet.mockResolvedValue(undefined); + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerGroupCacheGet).toHaveBeenCalledWith('Default'); expect(mockOctokit.paginate).toHaveBeenCalledTimes(1); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 2); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`, - Value: '1', - Type: 'String', + expect(mockRunnerGroupCacheCreate).toHaveBeenCalledWith({ + runnerGroupName: 'Default', + runnerGroupId: 1, }); + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(1); }); - it('Does not create SSM parameter for runner group id if it exists', async () => { + it('reuses a cached runner group id', async () => { await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerGroupCacheGet).toHaveBeenCalledWith('Default'); expect(mockOctokit.paginate).toHaveBeenCalledTimes(0); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 1); + expect(mockRunnerGroupCacheCreate).not.toHaveBeenCalled(); + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(1); }); it('create start runner config for ephemeral runners ', async () => { @@ -1612,17 +1562,10 @@ describe('scaleUp with Github Data Residency', () => { runner_group_id: 1, labels: ['label1', 'label2'], }); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: 'TEST_JIT_CONFIG_ORG', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { runnerId: 'i-12345', value: 'TEST_JIT_CONFIG_ORG' }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); }); it('create start runner config for non-ephemeral runners ', async () => { @@ -1631,80 +1574,43 @@ describe('scaleUp with Github Data Residency', () => { await scaleUpModule.scaleUp(TEST_DATA); expect(mockOctokit.actions.generateRunnerJitconfigForOrg).not.toBeCalled(); expect(mockOctokit.actions.createRegistrationTokenForOrg).toBeCalled(); - expect(mockSSMClient).toHaveReceivedNthSpecificCommandWith(1, PutParameterCommand, { - Name: '/github-action-runners/default/runners/config/i-12345', - Value: - '--url https://companyname.ghe.com/Codertocat --token 1234abcd ' + - '--labels label1,label2 --runnergroup Default', - Type: 'SecureString', - Tags: [ - { - Key: 'RunnerId', - Value: 'i-12345', - }, - ], - }); + expect(mockRunnerConfigCreate).toHaveBeenCalledWith( + { + runnerId: 'i-12345', + value: + '--url https://companyname.ghe.com/Codertocat --token 1234abcd ' + + '--labels label1,label2 --runnergroup Default', + }, + { metadata: [{ key: 'RunnerId', value: 'i-12345' }] }, + ); }); it.each(RUNNER_TYPES)( - 'calls create start runner config of 40' + ' instances (ssm rate limit condition) to test time delay ', + 'paces 40 runner-config writes at the store throughput limit for %s runners', async (type: RunnerLifecycle) => { process.env.ENABLE_EPHEMERAL_RUNNERS = type === 'ephemeral' ? 'true' : 'false'; process.env.RUNNERS_MAXIMUM_COUNT = '40'; + const instances = Array.from({ length: 40 }, (_, index) => `i-${index + 1}`); mockCreateRunner.mockImplementation(async () => { return createRunnerResult(instances); }); mockListRunners.mockImplementation(async () => { return []; }); - const startTime = performance.now(); - const instances = [ - 'i-1234', - 'i-5678', - 'i-5567', - 'i-5569', - 'i-5561', - 'i-5560', - 'i-5566', - 'i-5536', - 'i-5526', - 'i-5516', - 'i-122', - 'i-123', - 'i-124', - 'i-125', - 'i-126', - 'i-127', - 'i-128', - 'i-129', - 'i-130', - 'i-131', - 'i-132', - 'i-133', - 'i-134', - 'i-135', - 'i-136', - 'i-137', - 'i-138', - 'i-139', - 'i-140', - 'i-141', - 'i-142', - 'i-143', - 'i-144', - 'i-145', - 'i-146', - 'i-147', - 'i-148', - 'i-149', - 'i-150', - 'i-151', - ]; - await scaleUpModule.scaleUp(TEST_DATA); - const endTime = performance.now(); - expect(endTime - startTime).toBeGreaterThan(1000); - expect(mockSSMClient).toHaveReceivedCommandTimes(PutParameterCommand, 40); + + const setTimeoutSpy = vi.spyOn(globalThis, 'setTimeout').mockImplementation((callback) => { + callback(); + return 0 as unknown as NodeJS.Timeout; + }); + try { + await scaleUpModule.scaleUp(TEST_DATA); + + expect(mockRunnerConfigCreate).toHaveBeenCalledTimes(40); + expect(setTimeoutSpy).toHaveBeenCalledTimes(40); + expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 25); + } finally { + setTimeoutSpy.mockRestore(); + } }, - 10000, ); }); describe('on repo level', () => { @@ -1989,7 +1895,6 @@ describe('Retry mechanism tests', () => { process.env.ENABLE_JOB_QUEUED_CHECK = 'true'; process.env.RUNNERS_MAXIMUM_COUNT = '10'; expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); }); const createTestMessages = ( @@ -2177,11 +2082,8 @@ describe('Multi-app round-robin', () => { process.env.RUNNERS_MAXIMUM_COUNT = '10'; process.env.RUNNER_NAME_PREFIX = 'unit-test-'; process.env.RUNNER_GROUP_NAME = 'Default'; - process.env.SSM_CONFIG_PATH = '/github-action-runners/default/runners/config'; - process.env.SSM_TOKEN_PATH = '/github-action-runners/default/runners/config'; process.env.RUNNER_LABELS = 'label1,label2'; expectedRunnerParams = { ...EXPECTED_RUNNER_PARAMS }; - mockSSMClient.reset(); }); it('passes the same appIndex to createGithubInstallationAuth when multi-app is active', async () => { @@ -2265,7 +2167,7 @@ describe('Multi-app round-robin', () => { }); it('stored installationId takes precedence over webhook payload for additional app', async () => { - // Additional app (index 1) with a pre-configured installation id stored in SSM + // Additional app (index 1) with a pre-configured installation id mockedGetAppCount.mockResolvedValue(2); mockedGetStoredInstallationId.mockResolvedValue(77); mockedAppAuth.mockResolvedValue({ @@ -2335,15 +2237,3 @@ function defaultOctokitMockImpl() { mockOctokit.apps.getOrgInstallation.mockImplementation(() => mockInstallationIdReturnValueOrgs); mockOctokit.apps.getRepoInstallation.mockImplementation(() => mockInstallationIdReturnValueRepos); } - -function defaultSSMGetParameterMockImpl() { - mockSSMgetParameter.mockImplementation(async (name: string) => { - if (name === `${process.env.SSM_CONFIG_PATH}/runner-group/${process.env.RUNNER_GROUP_NAME}`) { - return '1'; - } else if (name === `${process.env.PARAMETER_GITHUB_APP_ID_NAME}`) { - return `${process.env.GITHUB_APP_ID}`; - } else { - throw new Error(`ParameterNotFound: ${name}`); - } - }); -} diff --git a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts index 44d522a1f0..48733c13c9 100644 --- a/lambdas/functions/control-plane/src/scale-runners/scale-up.ts +++ b/lambdas/functions/control-plane/src/scale-runners/scale-up.ts @@ -11,7 +11,6 @@ import { resolveInstallationId, isJobQueued, UnsupportedEventError, - validateSsmParameterStoreTags, } from './github-runner'; import { publishRetryMessage } from './job-retry'; import type { @@ -80,17 +79,11 @@ export async function scaleUp(payloads: ActionRequestMessageSQS[]): Promise { - beforeEach(() => { - mockSSMClient.reset(); - mockSSMClient.on(GetParametersByPathCommand).resolves({ - Parameters: undefined, - }); - mockSSMClient.on(GetParametersByPathCommand, { Path: tokenPath }).resolves({ - Parameters: [ - { - Name: tokenPath + 'i-old-01', - LastModifiedDate: dateOld, - }, - ], - NextToken: 'next', - }); - mockSSMClient.on(GetParametersByPathCommand, { Path: tokenPath, NextToken: 'next' }).resolves({ - Parameters: [ - { - Name: tokenPath + 'i-new-01', - LastModifiedDate: now, - }, - ], - NextToken: undefined, - }); - }); - - it('should delete parameters older then minimumDaysOld', async () => { - await cleanSSMTokens({ - dryRun: false, - minimumDaysOld: deleteAmisOlderThenDays, - tokenPath: tokenPath, - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(GetParametersByPathCommand, { Path: tokenPath }); - expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-old-01' }); - expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-new-01' }); - }); - - it('should not delete when dry run is activated', async () => { - await cleanSSMTokens({ - dryRun: true, - minimumDaysOld: deleteAmisOlderThenDays, - tokenPath: tokenPath, - }); - - expect(mockSSMClient).toHaveReceivedCommandWith(GetParametersByPathCommand, { Path: tokenPath }); - expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-old-01' }); - expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-new-01' }); - }); - - it('should not call delete when no parameters are found.', async () => { - await expect( - cleanSSMTokens({ - dryRun: false, - minimumDaysOld: deleteAmisOlderThenDays, - tokenPath: 'no-exist', - }), - ).resolves.not.toThrow(); - - expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-old-01' }); - expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: tokenPath + 'i-new-01' }); - }); - - it('should not error on delete failure.', async () => { - mockSSMClient.on(DeleteParameterCommand).rejects(new Error('ParameterNotFound')); - - await expect( - cleanSSMTokens({ - dryRun: false, - minimumDaysOld: deleteAmisOlderThenDays, - tokenPath: tokenPath, - }), - ).resolves.not.toThrow(); - }); - - it('should only accept valid options.', async () => { - await expect( - cleanSSMTokens({ - dryRun: false, - minimumDaysOld: undefined as unknown as number, - tokenPath: tokenPath, - }), - ).rejects.toBeInstanceOf(Error); - - await expect( - cleanSSMTokens({ - dryRun: false, - minimumDaysOld: 0, - tokenPath: tokenPath, - }), - ).rejects.toBeInstanceOf(Error); - - await expect( - cleanSSMTokens({ - dryRun: false, - minimumDaysOld: 1, - tokenPath: undefined as unknown as string, - }), - ).rejects.toBeInstanceOf(Error); - }); -}); diff --git a/lambdas/libs/aws-ssm-util/src/index.test.ts b/lambdas/libs/aws-ssm-util/src/index.test.ts index 8a1d8d3864..bf31a60a1f 100644 --- a/lambdas/libs/aws-ssm-util/src/index.test.ts +++ b/lambdas/libs/aws-ssm-util/src/index.test.ts @@ -1,6 +1,9 @@ import { + AddTagsToResourceCommand, + DeleteParameterCommand, GetParameterCommand, GetParameterCommandOutput, + GetParametersByPathCommand, GetParametersCommand, PutParameterCommand, PutParameterCommandOutput, @@ -10,7 +13,17 @@ import 'aws-sdk-client-mock-jest/vitest'; import { mockClient } from 'aws-sdk-client-mock'; import nock from 'nock'; -import { getParameter, getParameters, putParameter, resetSSMClient, ssmClient, SSM_ADVANCED_TIER_THRESHOLD } from '.'; +import { + addParameterTags, + deleteParameter, + getParameter, + getParameters, + getParametersByPath, + putParameter, + resetSSMClient, + ssmClient, + SSM_ADVANCED_TIER_THRESHOLD, +} from '.'; import { describe, it, expect, beforeEach, vi } from 'vitest'; const mockSSMClient = mockClient(SSMClient); @@ -104,6 +117,30 @@ describe('Test getParameter and putParameter', () => { }); }); + it('overwrites a parameter only when explicitly requested', async () => { + mockSSMClient.on(PutParameterCommand).resolves({}); + + await putParameter('testParam', 'updated', false, { overwrite: true }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: 'testParam', + Value: 'updated', + Type: 'String', + Overwrite: true, + }); + }); + + it('rejects tags when overwriting an existing parameter', async () => { + mockSSMClient.resetHistory(); + await expect( + putParameter('testParam', 'updated', false, { + overwrite: true, + tags: [{ Key: 'owner', Value: 'runner' }], + } as never), + ).rejects.toThrow('tags cannot be supplied when overwriting'); + expect(mockSSMClient).not.toHaveReceivedCommand(PutParameterCommand); + }); + it('Puts parameters as SecureString', async () => { // Arrange const parameterValue = 'test'; @@ -127,6 +164,27 @@ describe('Test getParameter and putParameter', () => { }); }); + it('passes tags to the PutParameter command', async () => { + const parameterValue = 'test'; + const parameterName = 'testParam'; + const tags = [{ Key: 'InstanceId', Value: 'i-123' }]; + const output: PutParameterCommandOutput = { + $metadata: { + httpStatusCode: 200, + }, + }; + mockSSMClient.on(PutParameterCommand).resolves(output); + + await putParameter(parameterName, parameterValue, true, { tags }); + + expect(mockSSMClient).toHaveReceivedCommandWith(PutParameterCommand, { + Name: parameterName, + Value: parameterValue, + Type: 'SecureString', + Tags: tags, + }); + }); + it('Gets invalid parameters and returns string', async () => { // Arrange const parameterName = 'invalid'; @@ -256,6 +314,70 @@ describe('Test getParameters (batch)', () => { }); }); +describe('Test direct parameter path operations', () => { + beforeEach(() => { + mockSSMClient.reset(); + }); + + it('paginates direct, non-secret children of a parameter path', async () => { + mockSSMClient + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: undefined, + }) + .resolves({ Parameters: [{ Name: '/metadata/one', Value: '1' }], NextToken: 'page-2' }) + .on(GetParametersByPathCommand, { + Path: '/metadata', + Recursive: false, + WithDecryption: false, + NextToken: 'page-2', + }) + .resolves({ Parameters: [{ Name: '/metadata/two', Value: '2' }] }); + + await expect(getParametersByPath('/metadata')).resolves.toEqual( + new Map([ + ['/metadata/one', '1'], + ['/metadata/two', '2'], + ]), + ); + expect(mockSSMClient).toHaveReceivedCommandTimes(GetParametersByPathCommand, 2); + }); + + it('deletes an exact parameter name', async () => { + mockSSMClient.on(DeleteParameterCommand).resolves({}); + + await deleteParameter('/metadata/one'); + + expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: '/metadata/one' }); + }); + + it('adds tags to an exact parameter name', async () => { + mockSSMClient.on(AddTagsToResourceCommand).resolves({}); + + await addParameterTags('/metadata/one', [{ Key: 'ghr:environment', Value: 'unit-test' }]); + + expect(mockSSMClient).toHaveReceivedCommandWith(AddTagsToResourceCommand, { + ResourceType: 'Parameter', + ResourceId: '/metadata/one', + Tags: [{ Key: 'ghr:environment', Value: 'unit-test' }], + }); + }); + + it('does not call SSM when there are no parameter tags to add', async () => { + await addParameterTags('/metadata/one', []); + + expect(mockSSMClient).not.toHaveReceivedCommand(AddTagsToResourceCommand); + }); + + it('propagates failures when adding parameter tags', async () => { + mockSSMClient.on(AddTagsToResourceCommand).rejects(new Error('AccessDenied')); + + await expect(addParameterTags('/metadata/one', [{ Key: 'Name', Value: 'runner' }])).rejects.toThrow('AccessDenied'); + }); +}); + describe('SSM client configuration', () => { it('configures adaptive retry with a raised attempt cap', async () => { const config = ssmClient().config; diff --git a/lambdas/libs/aws-ssm-util/src/index.ts b/lambdas/libs/aws-ssm-util/src/index.ts index 71b33cbf41..ad448b57ac 100644 --- a/lambdas/libs/aws-ssm-util/src/index.ts +++ b/lambdas/libs/aws-ssm-util/src/index.ts @@ -1,4 +1,12 @@ -import { GetParametersCommand, PutParameterCommand, SSMClient, Tag } from '@aws-sdk/client-ssm'; +import { + AddTagsToResourceCommand, + DeleteParameterCommand, + GetParametersByPathCommand, + GetParametersCommand, + PutParameterCommand, + SSMClient, + Tag, +} from '@aws-sdk/client-ssm'; import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; import { SSMProvider } from '@aws-lambda-powertools/parameters/ssm'; @@ -103,14 +111,68 @@ export async function getParameters(parameter_names: string[]): Promise> { + const result = new Map(); + let nextToken: string | undefined; + + do { + const response = await ssmClient().send( + new GetParametersByPathCommand({ + Path: parameter_path, + Recursive: false, + WithDecryption: false, + NextToken: nextToken, + }), + ); + + for (const parameter of response.Parameters ?? []) { + if (parameter.Name && parameter.Value) { + result.set(parameter.Name, parameter.Value); + } + } + nextToken = response.NextToken; + } while (nextToken); + + return result; +} + +export async function deleteParameter(parameter_name: string): Promise { + await ssmClient().send(new DeleteParameterCommand({ Name: parameter_name })); +} + +export async function addParameterTags(parameter_name: string, tags: Tag[]): Promise { + if (tags.length === 0) return; + + await ssmClient().send( + new AddTagsToResourceCommand({ + ResourceType: 'Parameter', + ResourceId: parameter_name, + Tags: tags, + }), + ); +} + export const SSM_ADVANCED_TIER_THRESHOLD = 4000; +type PutParameterOptions = { overwrite: true; tags?: never } | { overwrite?: false | undefined; tags?: Tag[] }; + export async function putParameter( parameter_name: string, parameter_value: string, secure: boolean, - options: { tags?: Tag[] } = {}, + options: PutParameterOptions = {}, ): Promise { + if (options.overwrite && options.tags !== undefined) { + throw new Error('SSM parameter tags cannot be supplied when overwriting an existing parameter'); + } + const client = ssmClient(); // Determine tier based on parameter_value size @@ -121,6 +183,7 @@ export async function putParameter( Name: parameter_name, Value: parameter_value, Type: secure ? 'SecureString' : 'String', + Overwrite: options.overwrite, Tags: options.tags, Tier: valueSizeBytes >= SSM_ADVANCED_TIER_THRESHOLD ? 'Advanced' : 'Standard', }), diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts index 06a6bb430b..7936317576 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/runner-creation.ts @@ -1,4 +1,8 @@ import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import type { Tag } from '@aws-sdk/client-ec2'; +import { Octokit } from '@octokit/rest'; +import yn from 'yn'; + import type { CreateGitHubRunnerConfig, CreateRunnerResult, @@ -7,10 +11,6 @@ import type { RunnerSource, StartRunnerConfigOptions, } from '../../../../core'; -import { Octokit } from '@octokit/rest'; -import type { Tag } from '@aws-sdk/client-ec2'; -import yn from 'yn'; - import type { Ec2RunnerResourceOperations } from '../runners'; import type { RunnerInputParameters } from '../runners.d'; import { toControlPlaneCreateRunnerResult } from './create-result'; @@ -148,7 +148,7 @@ async function terminateFailedInstances( function createEc2StartRunnerConfigOptions(ec2Operations: Ec2RunnerResourceOperations): StartRunnerConfigOptions { return { - getSsmParameterTags: (instanceId) => [{ Key: 'InstanceId', Value: instanceId }], + getRunnerConfigMetadata: (instanceId) => [{ key: 'InstanceId', value: instanceId }], onJitConfigCreated: async (instanceId, metadata) => await tagEc2RunnerMetadata(ec2Operations, instanceId, metadata), }; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts index 22785e0268..1512955e05 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/control-plane/scale-up.test.ts @@ -51,9 +51,6 @@ function runnerConfig(overrides: Partial = {}): Create runnerOwner, runnerType: 'Org', disableAutoUpdate: false, - ssmTokenPath: '/github-action-runners/default/runners/config', - ssmConfigPath: '/github-action-runners/default/runners/config', - ssmParameterStoreTags: [], ...overrides, }; } @@ -175,7 +172,7 @@ describe('scaleUp with GHES', () => { { Key: 'ghr:runner_labels', Value: 'label1,label2' }, ]); const [, , , options] = mockCreateStartRunnerConfig.mock.calls[0]; - expect(options?.getSsmParameterTags?.('i-12345')).toEqual([{ Key: 'InstanceId', Value: 'i-12345' }]); + expect(options?.getRunnerConfigMetadata?.('i-12345')).toEqual([{ key: 'InstanceId', value: 'i-12345' }]); }); it('chunks comma-joined GitHub runner labels by the EC2 tag value max length', async () => { diff --git a/lambdas/libs/compute-providers/aws/microvm/README.md b/lambdas/libs/compute-providers/aws/microvm/README.md new file mode 100644 index 0000000000..bc672dd2a2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/README.md @@ -0,0 +1,149 @@ +# Lambda MicroVM compute provider + +This provider manages a compatible AWS Lambda MicroVM image through the control-plane Lambda. It currently supports ephemeral JIT runners only. + +The MicroVM image `/run` hook receives this `runHookPayload`: + +```json +{ + "version": 1, + "imageArn": "arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner", + "imageVersion": "12.0", + "runnerConfigSsmPath": "/github-action-runners/example/runners/config", + "runnerTokenSsmPath": "/github-action-runners/example/runners/tokens" +} +``` + +Lambda adds `microvmId` beside that payload. `imageArn` and `imageVersion` are the requested launch values and are included together when an explicit image version is selected. The image must poll the SecureString parameter at `/`, start the GitHub runner with its encoded JIT configuration, delete the parameter after reading it, and exit its lifecycle entrypoint after the job completes. The image separately polls its complete non-secret tag map at `/microvm-metadata/.tags`. The control plane stores the JIT parameter before the provider callback writes the tag map, preventing cleanup from deleting an absent JIT that could otherwise be recreated later. Neither metadata record contains the JIT configuration value. Trusted control-plane cleanup and the fixed lifetime remain termination backstops. + +Runner ownership and lifecycle state are stored separately as non-secret `String` +parameters under `/`. The immutable base +record and independent state parameters prevent concurrent GitHub ID, orphan, +and cleanup updates from overwriting one another. Deleting the JIT SecureString +does not delete this metadata. Use a dedicated metadata prefix that does not +overlap the JIT path, and grant the MicroVM execution role only the exact +value-read access described below, without path-listing permissions. The control +plane retries pending cleanup, removes metadata after termination, and reconciles +expired records during inventory. + +The immutable base metadata parameter carries the same AWS resource tags that +are serialized as a JSON object in the `.tags` parameter. The tag +set starts with `SSM_PARAMETER_STORE_TAGS`, omits `Name`, and derives +`ghr:environment`, `ghr:ssm_config_path`, and `ghr:runner_name_prefix` from +the existing `ENVIRONMENT`, `SSM_CONFIG_PATH`, and `RUNNER_NAME_PREFIX` +settings. The Lambda then adds authoritative runtime tags: +`ghr:Application`, `ghr:created_by`, `ghr:environment`, `ghr:Owner`, +`ghr:Type`, `ghr:microvm_id`, `ghr:microvm_image_arn`, and, when available, +`ghr:microvm_image_version`. After JIT registration, the control plane adds +`ghr:github_runner_id` and base64url-encoded runner-label groups under +`ghr:runner_labels` through `ghr:runner_labels:5`. Runtime-owned values override +configured collisions. The `aws:` tag prefix is reserved and cannot be used for +these SSM parameters. The `.tags` value may use the Parameter Store advanced +tier when its UTF-8 representation is at least 4,000 bytes and is rejected if +the complete value could exceed the 8 KiB Parameter Store limit. + +Final cleanup deletes `/`, the +`.github-runner-id`, `.orphan`, and `.tags` companions, the base ownership +record, and `.cleanup-requested-at` last. The tombstone keeps its original +timestamp through a five-minute grace window so cleanup can repeatedly revoke a +late JIT write before removing every record. Missing parameters are treated as +already cleaned. + +The runner configuration publishes `/enable_cloudwatch` +and, when enabled, `/cloudwatch_agent_config_runner`. +The generated agent configuration reads these image-owned files by default: + +- `/var/log/microvm/internal-services.log` +- `/var/log/microvm/run.log` +- `/opt/actions-runner/_diag/Runner_**.log` + +Their default log-group suffixes are `internal_service`, `run`, and `runner`, +and `{microvm_id}` is an image-expanded log-stream placeholder. The first two +files are part of the MicroVM image contract; the portable lifecycle hook does +not create CloudWatch-specific files. Native RunMicrovm stdout and stderr stay +enabled independently as the early-startup and failure backstop. + +The control-plane Lambda requires these provider environment variables: + +- `MICROVM_IMAGE_ARN` +- `MICROVM_EXECUTION_ROLE_ARN` +- `MICROVM_IMAGE_VERSION` (optional) +- `MICROVM_INGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_EGRESS_NETWORK_CONNECTORS` (optional JSON array or comma-separated list) +- `MICROVM_METADATA_SSM_PATH` (dedicated SSM path for control-plane metadata) +- `MICROVM_LOG_GROUP` (optional) +- `SSM_TOKEN_PATH` (lane-scoped JIT parameter path) + +Each runner is launched with a fixed lifetime of 28,800 seconds (8 hours). + +The control-plane role requires `ssm:GetParametersByPath`, `ssm:GetParameters`, +`ssm:PutParameter`, `ssm:AddTagsToResource`, and `ssm:DeleteParameter` on the +dedicated metadata prefix, plus a separate `ssm:DeleteParameter` grant on the +lane-scoped JIT prefix, and `lambda:ListMicrovms`, `lambda:RunMicrovm`, and +`lambda:TerminateMicrovm` for inventory and lifecycle reconciliation. Restrict +`lambda:RunMicrovm` and `lambda:TerminateMicrovm` to approved image resources; +`lambda:ListMicrovms` does not support resource-level permissions. + +The MicroVM execution role must trust `lambda.amazonaws.com` for both +`sts:AssumeRole` and `sts:TagSession`. Restrict `iam:PassRole` to that exact role +ARN. Network connectors also require `lambda:PassNetworkConnector`; because +that action does not currently support resource-level permissions, enforce the +connector boundary with the explicit dynamic-label allowlist described below. + +All MicroVMs using one execution role, JIT prefix, and metadata prefix share a +trust boundary. Grant that role only `ssm:GetParameter` on the Parameter Store +ARN corresponding to `/microvm-metadata/*` and the exact +CloudWatch configuration parameters, `ssm:GetParameter` and +`ssm:DeleteParameter` on the lane-scoped JIT prefix, and stream-write access to +the provider-managed log groups. The image must address its own metadata with +its AWS-provided `microvmId` and must not receive path-listing access. IAM cannot +bind that ID to the calling MicroVM session, so a MicroVM can read other +metadata records in the same lane if it learns their IDs. Only allow trusted +images and workloads within a shared role, or isolate trust domains with +separate roles, prefixes, and provider deployments. + +## Dynamic labels + +When a runner matcher enables dynamic labels, workflow jobs can override the +following `RunMicrovm` inputs: + +| Label | Override | +| --------------------------------------------- | -------------------------------- | +| `ghr-microvm-egress-network-connectors:` | One egress network connector ARN | +| `ghr-microvm-image-arn:` | MicroVM image ARN | +| `ghr-microvm-image-version:` | MicroVM image version | + +Repeat `ghr-microvm-egress-network-connectors:` to attach multiple +connectors. Specify one ARN per label; `RunMicrovm` accepts at most 10. These +labels replace the compute provider's configured +`MICROVM_EGRESS_NETWORK_CONNECTORS` value for that job. + +Lambda MicroVM does not expose CPU or memory as `RunMicrovm` inputs. Select an +image and version with the required resources instead. Labels such as +`ghr-microvm-memory` are rejected. + +Execution roles, ingress network connectors, logging, idle policy, run hook +payloads, and client tokens remain deployment-controlled. Image ARN, image +version, and egress connector overrides change executable code or the network +boundary, so they are rejected unless `awsDynamicLabelsPolicy` supplies an +explicit `allowed` list for the corresponding key. + +Use the matcher's `awsDynamicLabelsPolicy` to restrict values accepted from +workflow jobs. The MicroVM policy keys are `egress-network-connectors`, +`image-arn`, and `image-version`. For example: + +```json +{ + "restricted_keys": { + "egress-network-connectors": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-*"] + }, + "image-arn": { + "allowed": ["arn:aws:lambda:eu-west-1:123456789012:microvm-image:github-runner-*"] + }, + "image-version": { + "allowed": ["3.*"] + } + } +} +``` diff --git a/lambdas/libs/compute-providers/aws/microvm/control-plane.ts b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts new file mode 100644 index 0000000000..d6287ca1e1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/control-plane.ts @@ -0,0 +1,25 @@ +import type { ComputeProviderPlugin, CreateStartRunnerConfig } from '../../core'; + +import type { ControlPlaneProviderCapabilities, ControlPlaneProviderModule } from '../../contracts'; +import type {} from './src/environment'; +import { createMicrovmPoolProvider } from './src/control-plane/pool'; +import { createMicrovmScaleDownProvider } from './src/control-plane/scale-down'; +import { createMicrovmScaleUpProvider } from './src/control-plane/scale-up'; + +export function createMicrovmControlPlanePlugin( + createStartRunnerConfig: CreateStartRunnerConfig, +): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { + pool: () => createMicrovmPoolProvider(createStartRunnerConfig), + scaleUp: () => createMicrovmScaleUpProvider(createStartRunnerConfig), + scaleDown: createMicrovmScaleDownProvider, + }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmControlPlanePlugin, +} satisfies ControlPlaneProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts new file mode 100644 index 0000000000..e58d73093c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.test.ts @@ -0,0 +1,85 @@ +import { beforeEach, describe, expect, it } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; + +const cleanEnv = process.env; + +beforeEach(() => { + process.env = { ...cleanEnv }; + process.env.MICROVM_IMAGE_ARN = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; + process.env.MICROVM_EXECUTION_ROLE_ARN = 'arn:aws:iam::123456789012:role/microvm-runner'; + process.env.MICROVM_METADATA_SSM_PATH = '/github-action-runners/unit-test/microvm-metadata/'; + process.env.SSM_TOKEN_PATH = '/github-action-runners/unit-test/token/'; + delete process.env.MICROVM_IMAGE_VERSION; + delete process.env.MICROVM_INGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_EGRESS_NETWORK_CONNECTORS; + delete process.env.MICROVM_LOG_GROUP; +}); + +describe('loadMicrovmProviderConfig', () => { + it('loads required values and applies optional defaults', () => { + expect(loadMicrovmProviderConfig()).toEqual({ + imageIdentifier: process.env.MICROVM_IMAGE_ARN, + imageVersion: undefined, + executionRoleArn: process.env.MICROVM_EXECUTION_ROLE_ARN, + ingressNetworkConnectors: undefined, + egressNetworkConnectors: undefined, + metadataSsmPath: '/github-action-runners/unit-test/microvm-metadata', + runnerTokenSsmPath: '/github-action-runners/unit-test/token', + logging: undefined, + }); + }); + + it('loads versions, logging, and either connector list format', () => { + process.env.MICROVM_IMAGE_VERSION = ' 3.0 '; + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS = '["arn:ingress:one","arn:ingress:two"]'; + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = 'arn:egress:one, arn:egress:two'; + process.env.MICROVM_LOG_GROUP = ' /aws/lambda-microvms/runner '; + + expect(loadMicrovmProviderConfig()).toMatchObject({ + imageVersion: '3.0', + ingressNetworkConnectors: ['arn:ingress:one', 'arn:ingress:two'], + egressNetworkConnectors: ['arn:egress:one', 'arn:egress:two'], + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, + }); + }); + + it.each([ + ['MICROVM_IMAGE_ARN', 'MICROVM_IMAGE_ARN'], + ['MICROVM_EXECUTION_ROLE_ARN', 'MICROVM_EXECUTION_ROLE_ARN'], + ['MICROVM_METADATA_SSM_PATH', 'MICROVM_METADATA_SSM_PATH'], + ['SSM_TOKEN_PATH', 'SSM_TOKEN_PATH'], + ])('requires %s', (environmentVariable, expectedName) => { + delete process.env[environmentVariable]; + + expect(() => loadMicrovmProviderConfig()).toThrow( + `${expectedName} must be configured for the MicroVM compute provider`, + ); + }); + + it.each(['[not-json', '[]', '["valid", 2]', 'first,'])('rejects malformed connector lists %s', (connectors) => { + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS = connectors; + + expect(() => loadMicrovmProviderConfig()).toThrow(/MICROVM_EGRESS_NETWORK_CONNECTORS must/); + }); + + it.each(['metadata', '/', '/metadata//nested', '/metadata/../nested', '/metadata/has space'])( + 'rejects malformed metadata SSM path %s', + (metadataPath) => { + process.env.MICROVM_METADATA_SSM_PATH = metadataPath; + + expect(() => loadMicrovmProviderConfig()).toThrow( + 'MICROVM_METADATA_SSM_PATH must be a valid absolute SSM parameter path', + ); + }, + ); + + it.each(['token', '/', '/token//nested', '/token/../nested', '/token/has space'])( + 'rejects malformed JIT SSM path %s', + (tokenPath) => { + process.env.SSM_TOKEN_PATH = tokenPath; + + expect(() => loadMicrovmProviderConfig()).toThrow('SSM_TOKEN_PATH must be a valid absolute SSM parameter path'); + }, + ); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts new file mode 100644 index 0000000000..b86331967b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/config.ts @@ -0,0 +1,78 @@ +import type { Logging, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +export interface MicrovmProviderConfig { + egressNetworkConnectors?: string[]; + executionRoleArn: string; + imageIdentifier: string; + imageVersion?: string; + ingressNetworkConnectors?: string[]; + logging?: Logging; + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + +function requiredEnvironmentValue(name: string, value: string | undefined): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`${name} must be configured for the MicroVM compute provider`); + } + return trimmed; +} + +function optionalEnvironmentValue(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function parseSsmPath(name: string, value: string | undefined): string { + const path = requiredEnvironmentValue(name, value).replace(/\/+$/, ''); + if (path === '' || !/^\/[A-Za-z0-9_.\-/]+$/.test(path) || path.includes('//') || path.split('/').includes('..')) { + throw new Error(`${name} must be a valid absolute SSM parameter path`); + } + return path; +} + +function parseNetworkConnectors(name: string, value: string | undefined): string[] | undefined { + const configuredValue = optionalEnvironmentValue(value); + if (!configuredValue) return undefined; + + let connectors: unknown; + try { + connectors = configuredValue.startsWith('[') + ? JSON.parse(configuredValue) + : configuredValue.split(',').map((connector) => connector.trim()); + } catch (error) { + throw new Error(`${name} must be a JSON array or comma-separated list`, { cause: error }); + } + + if ( + !Array.isArray(connectors) || + connectors.length === 0 || + connectors.some((connector) => typeof connector !== 'string' || connector.trim().length === 0) + ) { + throw new Error(`${name} must contain one or more non-empty connector ARNs`); + } + + return connectors.map((connector) => connector.trim()); +} + +export function loadMicrovmProviderConfig(): MicrovmProviderConfig { + const logGroup = optionalEnvironmentValue(process.env.MICROVM_LOG_GROUP); + + return { + imageIdentifier: requiredEnvironmentValue('MICROVM_IMAGE_ARN', process.env.MICROVM_IMAGE_ARN), + imageVersion: optionalEnvironmentValue(process.env.MICROVM_IMAGE_VERSION), + executionRoleArn: requiredEnvironmentValue('MICROVM_EXECUTION_ROLE_ARN', process.env.MICROVM_EXECUTION_ROLE_ARN), + ingressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_INGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_INGRESS_NETWORK_CONNECTORS, + ), + egressNetworkConnectors: parseNetworkConnectors( + 'MICROVM_EGRESS_NETWORK_CONNECTORS', + process.env.MICROVM_EGRESS_NETWORK_CONNECTORS, + ), + metadataSsmPath: parseSsmPath('MICROVM_METADATA_SSM_PATH', process.env.MICROVM_METADATA_SSM_PATH), + runnerTokenSsmPath: parseSsmPath('SSM_TOKEN_PATH', process.env.SSM_TOKEN_PATH), + logging: logGroup ? ({ cloudWatch: { logGroup } } satisfies RunMicrovmCommandInput['logging']) : undefined, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts new file mode 100644 index 0000000000..09b6a46f8d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/lifetime.ts @@ -0,0 +1 @@ +export const MICROVM_LIFETIME_IN_SECONDS = 28_800; diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts new file mode 100644 index 0000000000..3271fd8b98 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.test.ts @@ -0,0 +1,418 @@ +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { MicrovmProviderConfig } from './config'; +import { + isRetryableMicrovmError, + listMicrovmRunners, + microvmBootTimeExceeded, + runMicrovmRunner, + terminateMicrovm, +} from './microvms'; +import { + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + createMicrovmRunnerMetadata: vi.fn(), + deleteMicrovmRunnerJitConfig: vi.fn(), + listMicrovmRunnerMetadata: vi.fn(), + markMicrovmCleanupPending: vi.fn(), +})); + +const mockMicrovmClient = mockClient(LambdaMicrovmsClient); +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const config: MicrovmProviderConfig = { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + egressNetworkConnectors: ['arn:egress'], + metadataSsmPath, + runnerTokenSsmPath, + logging: { cloudWatch: { logGroup: '/aws/lambda-microvms/runner' } }, +}; +const ssmParameterStoreTags = [{ Key: 'CostCenter', Value: '1234' }]; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-managed', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.0', + createdAt: '2026-08-06T10:00:00.000Z', + expiresAt: '2026-08-06T11:00:00.000Z', + ...overrides, + }; +} + +beforeEach(() => { + mockMicrovmClient.reset(); + vi.clearAllMocks(); + vi.useRealTimers(); + delete process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS; + process.env.AWS_REGION = 'eu-west-1'; + process.env.RUNNER_BOOT_TIME_IN_MINUTES = '5'; + vi.mocked(createMicrovmRunnerMetadata).mockResolvedValue(ssmParameterStoreTags); + vi.mocked(deleteMicrovmRunnerJitConfig).mockResolvedValue(); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ cleanupMicrovmIds: [], metadataById: new Map() }); + vi.mocked(markMicrovmCleanupPending).mockResolvedValue(); +}); + +describe('runMicrovmRunner', () => { + it('launches a runner for the fixed lifetime and records durable ownership metadata', async () => { + process.env.MICROVM_MAXIMUM_DURATION_IN_SECONDS = '1200'; + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-123', imageArn, imageVersion: '3.1' }); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{"version":1}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags, + source: 'scale-up-lambda', + }), + ).resolves.toEqual({ microvmId: 'mvm-123', metadataTags: ssmParameterStoreTags }); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(RunMicrovmCommand, { + imageIdentifier: imageArn, + imageVersion: '3.0', + executionRoleArn: config.executionRoleArn, + egressNetworkConnectors: ['arn:egress'], + maximumDurationInSeconds: 28_800, + logging: config.logging, + runHookPayload: '{"version":1}', + clientToken: expect.any(String), + }); + expect(createMicrovmRunnerMetadata).toHaveBeenCalledWith(metadataSsmPath, { + microvmId: 'mvm-123', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn, + imageVersion: '3.1', + ssmParameterStoreTags, + }); + }); + + it('rejects invalid metadata tags before launching a MicroVM', async () => { + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: imageArn }], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + expect(mockMicrovmClient).not.toHaveReceivedCommand(RunMicrovmCommand); + }); + + it('rejects a launch response without an ID', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({}); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'pool-lambda', + }), + ).rejects.toThrow('RunMicrovm returned no microvmId'); + }); + + it('terminates a new runner when required metadata cannot be recorded', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('metadata failed'); + + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-untracked', + }); + }); + + it('preserves the metadata error when termination also fails', async () => { + mockMicrovmClient.on(RunMicrovmCommand).resolves({ microvmId: 'mvm-untracked', imageArn }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + vi.mocked(createMicrovmRunnerMetadata).mockRejectedValue(new Error('metadata failed')); + + await expect( + runMicrovmRunner({ + config, + environment: 'unit-test', + runHookPayload: '{}', + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: [], + source: 'scale-up-lambda', + }), + ).rejects.toThrow('metadata failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-untracked'); + }); +}); + +describe('listMicrovmRunners', () => { + it('paginates active MicroVMs and filters them by durable metadata', async () => { + const startedAt = new Date('2026-08-06T10:00:00.000Z'); + mockMicrovmClient + .on(ListMicrovmsCommand) + .resolvesOnce({ + nextToken: 'page-2', + items: [ + { microvmId: 'mvm-managed', imageArn, imageVersion: '3.0', startedAt, state: 'RUNNING' }, + { microvmId: 'mvm-terminated', imageArn, imageVersion: '3.0', startedAt, state: 'TERMINATED' }, + ], + }) + .resolvesOnce({ + items: [{ microvmId: 'mvm-other', imageArn, imageVersion: '3.0', startedAt, state: 'PENDING' }], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-managed', metadata({ githubRunnerId: '42', bypassRemoval: true })], + ['mvm-other', metadata({ microvmId: 'mvm-other', runnerOwner: 'Other' })], + ]), + }); + + await expect( + listMicrovmRunners( + { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }, + ssmPaths, + ), + ).resolves.toEqual([ + { + id: 'mvm-managed', + imageArn, + launchTime: startedAt, + owner: 'Codertocat', + type: 'Org', + orphan: false, + githubRunnerId: '42', + bypassRemoval: true, + state: 'RUNNING', + }, + ]); + + expect(mockMicrovmClient).toHaveReceivedNthCommandWith(2, ListMicrovmsCommand, { + maxResults: 50, + nextToken: 'page-2', + }); + expect(listMicrovmRunnerMetadata).toHaveBeenCalledWith( + ssmPaths, + new Map([ + ['mvm-managed', 'RUNNING'], + ['mvm-terminated', 'TERMINATED'], + ['mvm-other', 'PENDING'], + ]), + ); + }); + + it('applies environment, owner, type, and orphan filters after loading metadata', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { + microvmId: 'mvm-filtered', + imageArn, + imageVersion: '3.0', + startedAt: new Date(), + state: 'SUSPENDED', + }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + [ + 'mvm-filtered', + metadata({ microvmId: 'mvm-filtered', environment: 'other', runnerOwner: 'Other', runnerType: 'Repo' }), + ], + ]), + }); + + await expect(listMicrovmRunners({ environment: 'unit-test' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerOwner: 'Codertocat' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ runnerType: 'Org' }, ssmPaths)).resolves.toEqual([]); + await expect(listMicrovmRunners({ orphan: true }, ssmPaths)).resolves.toEqual([]); + }); + + it('fails closed for an image mismatch while ignoring unowned MicroVMs', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-missing', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-mismatch', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + ], + }); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: [], + metadataById: new Map([ + ['mvm-mismatch', metadata({ microvmId: 'mvm-mismatch', imageArn: imageArn.replace(':runner', ':other') })], + ]), + }); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('does not match its metadata'); + }); + + it('attempts every pending cleanup and fails inventory closed when a retry fails', async () => { + const cleanupFailure = new Error('cleanup failed'); + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [ + { microvmId: 'mvm-first', imageArn, imageVersion: '3.0', state: 'RUNNING' }, + { microvmId: 'mvm-second', imageArn, imageVersion: '3.0', state: 'PENDING' }, + ], + }); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-first' }).rejects(cleanupFailure); + mockMicrovmClient.on(TerminateMicrovmCommand, { microvmIdentifier: 'mvm-second' }).resolves({}); + vi.mocked(listMicrovmRunnerMetadata).mockResolvedValue({ + cleanupMicrovmIds: ['mvm-first', 'mvm-second'], + metadataById: new Map(), + }); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('cleanup failed'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-first', + }); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-second', + }); + expect(markMicrovmCleanupPending).toHaveBeenCalledTimes(2); + }); + + it('surfaces metadata lookup failures instead of reporting zero runners', async () => { + mockMicrovmClient.on(ListMicrovmsCommand).resolves({ + items: [{ microvmId: 'mvm-error', imageArn, imageVersion: '3.0', state: 'RUNNING' }], + }); + vi.mocked(listMicrovmRunnerMetadata).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunners({}, ssmPaths)).rejects.toThrow('AccessDenied'); + }); +}); + +describe('MicroVM lifecycle helpers', () => { + it('retains metadata until inventory observes a terminated MicroVM', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await terminateMicrovm('mvm-123', ssmPaths); + + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('retains the tombstone when the MicroVM is already terminated so a late JIT write can be revoked', async () => { + const notFound = Object.assign(new Error('gone'), { name: 'ResourceNotFoundException' }); + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(notFound); + + await expect(terminateMicrovm('mvm-gone', ssmPaths)).resolves.toBeUndefined(); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-gone'); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-gone'); + }); + + it('retains metadata and marks cleanup pending when termination fails', async () => { + mockMicrovmClient.on(TerminateMicrovmCommand).rejects(new Error('terminate failed')); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toThrow('terminate failed'); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('retains the cleanup marker and reports a JIT deletion failure after termination succeeds', async () => { + const error = new Error('JIT cleanup failed'); + vi.mocked(deleteMicrovmRunnerJitConfig).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(markMicrovmCleanupPending).toHaveBeenCalledWith(metadataSsmPath, 'mvm-123'); + }); + + it('still terminates and reports a cleanup-marker failure for retry', async () => { + const error = new Error('metadata cleanup marker failed'); + vi.mocked(markMicrovmCleanupPending).mockRejectedValue(error); + mockMicrovmClient.on(TerminateMicrovmCommand).resolves({}); + + await expect(terminateMicrovm('mvm-123', ssmPaths)).rejects.toBe(error); + expect(deleteMicrovmRunnerJitConfig).toHaveBeenCalledWith(runnerTokenSsmPath, 'mvm-123'); + expect(mockMicrovmClient).toHaveReceivedCommandWith(TerminateMicrovmCommand, { + microvmIdentifier: 'mvm-123', + }); + }); + + it('evaluates the configured boot window', () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-06T10:10:00.000Z')); + + expect(microvmBootTimeExceeded({})).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:06:00.000Z') })).toBe(false); + expect(microvmBootTimeExceeded({ launchTime: new Date('2026-08-06T10:04:00.000Z') })).toBe(true); + }); +}); + +describe('isRetryableMicrovmError', () => { + it.each([ + 'ConflictException', + 'InternalServerException', + 'ServiceQuotaExceededException', + 'ThrottlingException', + 'TooManyUpdates', + ])('classifies %s as retryable', (name) => { + expect(isRetryableMicrovmError(Object.assign(new Error(name), { name }))).toBe(true); + }); + + it('classifies server, throttling, network, and nested failures as retryable', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('server'), { $fault: 'server' }))).toBe(true); + expect(isRetryableMicrovmError(Object.assign(new Error('throttle'), { $metadata: { httpStatusCode: 429 } }))).toBe( + true, + ); + expect(isRetryableMicrovmError(Object.assign(new Error('network'), { code: 'ECONNRESET' }))).toBe(true); + expect( + isRetryableMicrovmError( + Object.assign(new Error('outer'), { cause: Object.assign(new Error(), { code: 'ETIMEDOUT' }) }), + ), + ).toBe(true); + }); + + it('does not retry configuration, unknown, or non-error failures', () => { + expect(isRetryableMicrovmError(Object.assign(new Error('invalid'), { name: 'ValidationException' }))).toBe(false); + expect(isRetryableMicrovmError(new Error('unknown'))).toBe(false); + expect(isRetryableMicrovmError('failure')).toBe(false); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts new file mode 100644 index 0000000000..3293db55ba --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/microvms.ts @@ -0,0 +1,270 @@ +import { randomUUID } from 'node:crypto'; + +import { createChildLogger, getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { + LambdaMicrovmsClient, + ListMicrovmsCommand, + RunMicrovmCommand, + TerminateMicrovmCommand, +} from '@aws-sdk/client-lambda-microvms'; +import type { MicrovmItem, MicrovmState, RunMicrovmCommandInput } from '@aws-sdk/client-lambda-microvms'; + +import type { RunnerSource, ListRunnerFilters, RunnerInfo, RunnerType } from '../../../../core'; +import { loadMicrovmProviderConfig, type MicrovmProviderConfig } from './config'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; +import { + assertValidMicrovmMetadataTags, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + type MicrovmMetadataTag, + type MicrovmSsmPaths, +} from './runner-metadata'; + +const logger = createChildLogger('microvm-runners'); + +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); + +export interface MicrovmRunnerInfo extends RunnerInfo { + imageArn?: string; + state?: MicrovmState; +} + +export interface RunMicrovmRunnerInput { + config: MicrovmProviderConfig; + environment: string; + runHookPayload: string; + runnerOwner: string; + runnerType: RunnerType; + ssmParameterStoreTags: MicrovmMetadataTag[]; + source: RunnerSource; +} + +export interface RunMicrovmRunnerResult { + metadataTags: MicrovmMetadataTag[]; + microvmId: string; +} + +interface AwsErrorLike extends Error { + cause?: unknown; + code?: string; + $fault?: 'client' | 'server'; + $metadata?: { httpStatusCode?: number }; +} + +const RETRYABLE_ERROR_NAMES = new Set([ + 'ConflictException', + 'InternalServerException', + 'RequestTimeout', + 'RequestTimeoutException', + 'ResourceConflictException', + 'ServiceException', + 'ServiceQuotaExceededException', + 'Throttling', + 'ThrottlingException', + 'TooManyUpdates', + 'TooManyRequestsException', +]); + +const RETRYABLE_NETWORK_ERROR_CODES = new Set([ + 'EAI_AGAIN', + 'ECONNREFUSED', + 'ECONNRESET', + 'ENETUNREACH', + 'ENOTFOUND', + 'ETIMEDOUT', +]); + +function microvmClient(): LambdaMicrovmsClient { + return getTracedAWSV3Client(new LambdaMicrovmsClient({ region: process.env.AWS_REGION })); +} + +export async function runMicrovmRunner(input: RunMicrovmRunnerInput): Promise { + assertValidMicrovmMetadataTags({ + microvmId: 'microvm-validation', + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.config.imageIdentifier, + imageVersion: input.config.imageVersion ?? 'version-validation', + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + + const commandInput: RunMicrovmCommandInput = { + imageIdentifier: input.config.imageIdentifier, + imageVersion: input.config.imageVersion, + executionRoleArn: input.config.executionRoleArn, + ingressNetworkConnectors: input.config.ingressNetworkConnectors, + egressNetworkConnectors: input.config.egressNetworkConnectors, + maximumDurationInSeconds: MICROVM_LIFETIME_IN_SECONDS, + logging: input.config.logging, + runHookPayload: input.runHookPayload, + clientToken: randomUUID(), + }; + + logger.debug('Launching Lambda MicroVM runner', { + imageIdentifier: commandInput.imageIdentifier, + imageVersion: commandInput.imageVersion, + maximumDurationInSeconds: commandInput.maximumDurationInSeconds, + }); + + const response = await microvmClient().send(new RunMicrovmCommand(commandInput)); + if (!response.microvmId) { + throw new Error('RunMicrovm returned no microvmId'); + } + + const imageArn = response.imageArn ?? input.config.imageIdentifier; + const imageVersion = response.imageVersion ?? input.config.imageVersion; + + try { + const metadataTags = await createMicrovmRunnerMetadata(input.config.metadataSsmPath, { + microvmId: response.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn, + imageVersion, + ssmParameterStoreTags: input.ssmParameterStoreTags, + }); + return { microvmId: response.microvmId, metadataTags }; + } catch (error) { + logger.error(`Failed to record metadata for new MicroVM runner '${response.microvmId}', terminating it`, { + error, + }); + await terminateMicrovm(response.microvmId, input.config).catch((terminationError) => { + logger.error(`Failed to terminate untracked MicroVM runner '${response.microvmId}'`, { + error: terminationError, + }); + }); + throw error; + } +} + +export async function listMicrovmRunners( + filters: ListRunnerFilters = {}, + paths: MicrovmSsmPaths = loadMicrovmProviderConfig(), +): Promise { + const client = microvmClient(); + const items: MicrovmItem[] = []; + let nextToken: string | undefined; + + do { + const response = await client.send( + new ListMicrovmsCommand({ + maxResults: 50, + nextToken, + }), + ); + items.push(...(response.items ?? [])); + nextToken = response.nextToken; + } while (nextToken); + + const activeItems = items.filter( + (item): item is MicrovmItem & { imageArn: string; microvmId: string; state: MicrovmState } => + Boolean(item.microvmId && item.imageArn && item.state && ACTIVE_STATES.has(item.state)), + ); + const microvmStates = new Map( + items.flatMap((item) => (item.microvmId && item.state ? [[item.microvmId, item.state] as const] : [])), + ); + const { cleanupMicrovmIds, metadataById } = await listMicrovmRunnerMetadata(paths, microvmStates); + + let cleanupError: unknown; + for (const microvmId of cleanupMicrovmIds) { + logger.warn(`Retrying cleanup of MicroVM runner '${microvmId}'`); + try { + await terminateMicrovm(microvmId, paths); + } catch (error) { + cleanupError ??= error; + logger.error(`Failed to retry cleanup of MicroVM runner '${microvmId}'`, { error }); + } + } + if (cleanupError !== undefined) throw cleanupError; + + const runners: MicrovmRunnerInfo[] = []; + for (const item of activeItems) { + const metadata = metadataById.get(item.microvmId); + if (!metadata) continue; + if (metadata.imageArn !== item.imageArn) { + throw new Error(`Active MicroVM runner '${item.microvmId}' has an image that does not match its metadata`); + } + + const orphan = Boolean(metadata.orphan); + if (filters.environment !== undefined && metadata.environment !== filters.environment) continue; + if (filters.runnerType !== undefined && metadata.runnerType !== filters.runnerType) continue; + if (filters.runnerOwner !== undefined && metadata.runnerOwner !== filters.runnerOwner) continue; + if (filters.orphan && !orphan) continue; + + runners.push({ + id: item.microvmId, + imageArn: item.imageArn, + launchTime: item.startedAt, + owner: metadata.runnerOwner, + type: metadata.runnerType, + orphan, + githubRunnerId: metadata.githubRunnerId, + bypassRemoval: metadata.bypassRemoval ?? false, + state: item.state, + }); + } + + return runners; +} + +export async function terminateMicrovm(microvmId: string, paths: MicrovmSsmPaths): Promise { + let cleanupPreparationError: unknown; + try { + await markMicrovmCleanupPending(paths.metadataSsmPath, microvmId); + } catch (error) { + cleanupPreparationError = error; + logger.error(`Failed to mark MicroVM runner '${microvmId}' for cleanup`, { error }); + } + + try { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + } catch (error) { + cleanupPreparationError ??= error; + logger.error(`Failed to delete JIT configuration for MicroVM runner '${microvmId}'`, { error }); + } + + try { + await microvmClient().send(new TerminateMicrovmCommand({ microvmIdentifier: microvmId })); + } catch (error) { + if (error instanceof Error && error.name === 'ResourceNotFoundException') { + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; + return; + } + + throw error; + } + + if (cleanupPreparationError !== undefined) throw cleanupPreparationError; +} + +export function microvmBootTimeExceeded(runner: { launchTime?: Date }): boolean { + if (!runner.launchTime) return false; + + const bootTimeInMinutes = Number(process.env.RUNNER_BOOT_TIME_IN_MINUTES || '5'); + return runner.launchTime.getTime() + bootTimeInMinutes * 60_000 < Date.now(); +} + +export function isRetryableMicrovmError(error: unknown): boolean { + if (!(error instanceof Error)) return false; + + const awsError = error as AwsErrorLike; + if (RETRYABLE_ERROR_NAMES.has(awsError.name)) return true; + + const statusCode = awsError.$metadata?.httpStatusCode; + if ( + awsError.$fault === 'server' || + statusCode === 429 || + (statusCode !== undefined && statusCode >= 500) || + (awsError.code !== undefined && RETRYABLE_NETWORK_ERROR_CODES.has(awsError.code)) + ) { + return true; + } + + return awsError.cause !== undefined && awsError.cause !== error ? isRetryableMicrovmError(awsError.cause) : false; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts new file mode 100644 index 0000000000..719a0058c1 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.test.ts @@ -0,0 +1,109 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import type { MicrovmRunnerInfo } from './microvms'; +import { calculateMicrovmPoolSize, createMicrovmPoolProvider } from './pool'; +import { createMicrovmRunners } from './runner-config'; + +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), +})); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +function runner(id: string, state: MicrovmRunnerInfo['state']): MicrovmRunnerInfo { + return { id, state, owner: 'Codertocat', type: 'Org' }; +} + +function githubRunnerConfig(): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-1'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('calculateMicrovmPoolSize', () => { + it('counts online idle running runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-idle', 'RUNNING')], + new Map([['mvm-idle', { busy: false, status: 'online' }]]), + ), + ).toBe(1); + }); + + it('optionally counts online busy runners', () => { + const runners = [runner('mvm-busy', 'RUNNING')]; + const statuses = new Map([['mvm-busy', { busy: true, status: 'online' }]]); + + expect(calculateMicrovmPoolSize(runners, statuses)).toBe(0); + expect(calculateMicrovmPoolSize(runners, statuses, true)).toBe(1); + }); + + it('counts pending runners only during their boot window', () => { + const runners = [runner('mvm-pending', 'PENDING')]; + vi.mocked(microvmBootTimeExceeded).mockReturnValueOnce(false).mockReturnValueOnce(true); + + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(1); + expect(calculateMicrovmPoolSize(runners, new Map())).toBe(0); + }); + + it('does not count suspended or offline runners', () => { + expect( + calculateMicrovmPoolSize( + [runner('mvm-suspended', 'SUSPENDED'), runner('mvm-offline', 'RUNNING')], + new Map([['mvm-offline', { busy: false, status: 'offline' }]]), + ), + ).toBe(0); + }); +}); + +describe('createMicrovmPoolProvider', () => { + it('lists managed MicroVMs and returns successfully created IDs', async () => { + const provider = createMicrovmPoolProvider(createStartRunnerConfig); + const input = { + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + }; + + await expect(provider.listRunners(input)).resolves.toEqual([]); + expect(listMicrovmRunners).toHaveBeenCalledWith(input); + + await expect( + provider.createRunners({ + githubRunnerConfig: githubRunnerConfig(), + numberOfRunners: 1, + githubInstallationClient: githubClient, + }), + ).resolves.toEqual(['mvm-1']); + expect(createMicrovmRunners).toHaveBeenCalledWith( + expect.any(Object), + 1, + githubClient, + createStartRunnerConfig, + 'pool-lambda', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts new file mode 100644 index 0000000000..8deed5562d --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/pool.ts @@ -0,0 +1,65 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; + +import type { + CreatePoolRunnersInput, + CreateStartRunnerConfig, + ListPoolRunnersInput, + PoolComputeProvider, + RunnerStatus, +} from '../../../../core'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +const logger = createChildLogger('microvm-pool'); + +async function listMicrovmPoolRunners(input: ListPoolRunnersInput): Promise { + return await listMicrovmRunners(input); +} + +async function createMicrovmPoolRunners( + { githubRunnerConfig, numberOfRunners, githubInstallationClient }: CreatePoolRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + const result = await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'pool-lambda', + ); + return result.instances; +} + +export function calculateMicrovmPoolSize( + runners: MicrovmRunnerInfo[], + runnerStatus: Map, + includeBusyRunners = false, +): number { + let availableRunners = 0; + + for (const runner of runners) { + const status = runnerStatus.get(runner.id); + if (runner.state === 'RUNNING' && status?.status === 'online' && (!status.busy || includeBusyRunners)) { + availableRunners++; + logger.debug(`MicroVM runner ${runner.id} is online and counted as part of the pool`); + } else if (runner.state === 'PENDING' && !microvmBootTimeExceeded(runner)) { + availableRunners++; + logger.info(`MicroVM runner ${runner.id} is still booting and counted as part of the pool`); + } else { + logger.debug(`MicroVM runner ${runner.id} is not available and is not counted as part of the pool`); + } + } + + return availableRunners; +} + +export function createMicrovmPoolProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + listRunners: listMicrovmPoolRunners, + countAvailableRunners: calculateMicrovmPoolSize, + createRunners: (input) => createMicrovmPoolRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts new file mode 100644 index 0000000000..6d3f92fc76 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.test.ts @@ -0,0 +1,322 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { createMicrovmRunHookPayload, createMicrovmRunners } from './runner-config'; +import { setMicrovmGithubRunnerMetadata } from './runner-metadata'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + isRetryableMicrovmError: vi.fn(), + runMicrovmRunner: vi.fn(), + terminateMicrovm: vi.fn(), +})); +vi.mock('./runner-metadata', async (importOriginal) => ({ + ...(await importOriginal()), + setMicrovmGithubRunnerMetadata: vi.fn(), +})); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerConfigSsmPath = '/github-action-runners/unit-test/config'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const githubClient = {} as Octokit; +const createStartRunnerConfig = vi.fn(); +const ssmParameterStoreTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:environment', Value: 'caller-cannot-override' }, + { Key: 'ghr:runner_name_prefix', Value: 'caller-cannot-override' }, + { Key: 'ghr:ssm_config_path', Value: 'caller-cannot-override' }, +]; +const microvmMetadataTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: runnerConfigSsmPath }, +]; +const canonicalMetadataTags = [ + ...microvmMetadataTags, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; +const providerConfig = { + imageIdentifier: imageArn, + imageVersion: '2.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; + +function runnerConfig(overrides: Partial = {}): CreateGitHubRunnerConfig { + return { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: 'unit-test-', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, + ...overrides, + }; +} + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + process.env.SSM_CONFIG_PATH = runnerConfigSsmPath; + process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify(ssmParameterStoreTags); + process.env.SSM_TOKEN_PATH = runnerTokenSsmPath; + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(runMicrovmRunner).mockResolvedValue({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }); + vi.mocked(setMicrovmGithubRunnerMetadata).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); + vi.mocked(isRetryableMicrovmError).mockReturnValue(false); + createStartRunnerConfig.mockResolvedValue([]); +}); + +describe('createMicrovmRunHookPayload', () => { + it('contains the image and versioned runner paths', () => { + expect( + JSON.parse( + createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath: '/runner/token', + }), + ), + ).toEqual({ + imageArn, + imageVersion: '2.0', + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath: '/runner/token', + }); + }); + + it('requires the image ARN and version to be provided together', () => { + expect(() => + createMicrovmRunHookPayload({ + imageArn, + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ).toThrow('MicroVM hook payload image ARN and version must be provided together'); + }); + + it('omits image metadata when no explicit image version is selected', () => { + expect(JSON.parse(createMicrovmRunHookPayload({ runnerConfigSsmPath, runnerTokenSsmPath }))).toEqual({ + version: 1, + runnerConfigSsmPath, + runnerTokenSsmPath, + }); + }); +}); + +describe('createMicrovmRunners', () => { + it.each([{ ephemeral: false }, { enableJitConfig: false }])( + 'rejects unsupported runner configuration %j', + async (overrides) => { + await expect( + createMicrovmRunners(runnerConfig(overrides), 2, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 2 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }, + ); + + it('requires an SSM token path', async () => { + process.env.SSM_TOKEN_PATH = ''; + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('requires an SSM config path', async () => { + process.env.SSM_CONFIG_PATH = ''; + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('rejects a metadata path that overlaps the JIT token path', async () => { + vi.mocked(loadMicrovmProviderConfig).mockReturnValue({ + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath: '/github-action-runners/unit-test/token/metadata', + runnerTokenSsmPath, + }); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + expect(runMicrovmRunner).not.toHaveBeenCalled(); + }); + + it('canonicalizes the configuration and token paths before launching or writing JIT configuration', async () => { + process.env.SSM_CONFIG_PATH = `${runnerConfigSsmPath}/`; + process.env.SSM_TOKEN_PATH = `${runnerTokenSsmPath}/`; + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: ['mvm-1'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenCalledWith( + expect.objectContaining({ + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + ssmParameterStoreTags: microvmMetadataTags, + }), + ); + expect(createStartRunnerConfig).toHaveBeenCalledWith(runnerConfig(), ['mvm-1'], githubClient, expect.any(Object)); + }); + + it('classifies invalid provider configuration as non-retryable', async () => { + vi.mocked(loadMicrovmProviderConfig).mockImplementation(() => { + throw new Error('missing image'); + }); + + await expect( + createMicrovmRunners(runnerConfig(), 3, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 3 }); + }); + + it('launches each MicroVM and delivers its JIT configuration', async () => { + vi.mocked(runMicrovmRunner) + .mockResolvedValueOnce({ microvmId: 'mvm-1', metadataTags: canonicalMetadataTags }) + .mockResolvedValueOnce({ microvmId: 'mvm-2', metadataTags: canonicalMetadataTags }); + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { + githubRunnerId: `github-${runnerIds[0]}`, + runnerLabels: ['self-hosted', 'microvm'], + }); + return []; + }); + + await expect( + createMicrovmRunners(runnerConfig(), 2, githubClient, createStartRunnerConfig, 'pool-lambda'), + ).resolves.toEqual({ instances: ['mvm-1', 'mvm-2'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + + expect(runMicrovmRunner).toHaveBeenNthCalledWith(1, { + config: expect.objectContaining({ imageIdentifier: imageArn }), + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload({ + imageArn, + imageVersion: '2.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: microvmMetadataTags, + source: 'pool-lambda', + }); + expect(createStartRunnerConfig).toHaveBeenCalledTimes(2); + const options = createStartRunnerConfig.mock.calls[0][3]; + expect(options?.getRunnerConfigMetadata?.('mvm-1')).toEqual([{ key: 'MicrovmId', value: 'mvm-1' }]); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenNthCalledWith( + 1, + providerConfig, + 'mvm-1', + { + githubRunnerId: 'github-mvm-1', + runnerLabels: ['self-hosted', 'microvm'], + }, + canonicalMetadataTags, + ); + }); + + it('applies supported dynamic labels to the provider configuration', async () => { + const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; + const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + createStartRunnerConfig.mockImplementation(async (_config, runnerIds, _client, options) => { + await options?.onJitConfigCreated?.(runnerIds[0], { githubRunnerId: 'github-mvm-1', runnerLabels: [] }); + return []; + }); + + await createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda', { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }); + + expect(runMicrovmRunner).toHaveBeenCalledWith({ + config: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, + }, + environment: 'unit-test', + runHookPayload: createMicrovmRunHookPayload({ + imageArn: overrideImageArn, + imageVersion: '3.0', + runnerConfigSsmPath, + runnerTokenSsmPath, + }), + runnerOwner: 'Codertocat', + runnerType: 'Org', + ssmParameterStoreTags: microvmMetadataTags, + source: 'scale-up-lambda', + }); + expect(setMicrovmGithubRunnerMetadata).toHaveBeenCalledWith( + { + ...providerConfig, + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + 'mvm-1', + { githubRunnerId: 'github-mvm-1', runnerLabels: [] }, + canonicalMetadataTags, + ); + }); + + it('retries a JIT setup failure even when runner cleanup fails', async () => { + createStartRunnerConfig.mockResolvedValue(['mvm-1']); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); + + it.each([ + [true, { instances: [], retryableErrorCount: 1, nonRetryableErrorCount: 0 }], + [false, { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }], + ])('classifies launch failures with retryable=%s', async (retryable, expected) => { + vi.mocked(runMicrovmRunner).mockRejectedValue(new Error('launch failed')); + vi.mocked(isRetryableMicrovmError).mockReturnValue(retryable); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual(expected); + }); + + it('attempts cleanup when setup throws after launch', async () => { + createStartRunnerConfig.mockRejectedValue(new Error('JIT setup failed')); + vi.mocked(terminateMicrovm).mockRejectedValue(new Error('cleanup failed')); + + await expect( + createMicrovmRunners(runnerConfig(), 1, githubClient, createStartRunnerConfig, 'scale-up-lambda'), + ).resolves.toEqual({ instances: [], retryableErrorCount: 0, nonRetryableErrorCount: 1 }); + + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts new file mode 100644 index 0000000000..9d7154707b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-config.ts @@ -0,0 +1,208 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import type { Octokit } from '@octokit/rest'; + +import type { + CreateGitHubRunnerConfig, + CreateRunnerResult, + CreateStartRunnerConfig, + RunnerSource, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { loadMicrovmProviderConfig } from './config'; +import { isRetryableMicrovmError, runMicrovmRunner, terminateMicrovm } from './microvms'; +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + normalizeMicrovmSsmPath, + type MicrovmMetadataTag, + setMicrovmGithubRunnerMetadata, +} from './runner-metadata'; + +const logger = createChildLogger('microvm-runner-config'); +const MICROVM_METADATA_CONTEXT_TAG_KEYS = new Set([ + 'Name', + 'ghr:environment', + 'ghr:runner_name_prefix', + 'ghr:ssm_config_path', +]); + +export interface MicrovmRunHookPayloadV1 { + imageArn?: string; + imageVersion?: string; + runnerConfigSsmPath: string; + runnerTokenSsmPath: string; + version: 1; +} + +export function createMicrovmRunHookPayload(payload: Omit): string { + const hasImageArn = payload.imageArn !== undefined; + const hasImageVersion = payload.imageVersion !== undefined; + if (hasImageArn !== hasImageVersion) { + throw new Error('MicroVM hook payload image ARN and version must be provided together'); + } + + return JSON.stringify({ + version: 1, + ...(hasImageArn + ? { + imageArn: payload.imageArn, + imageVersion: payload.imageVersion, + } + : {}), + runnerConfigSsmPath: payload.runnerConfigSsmPath, + runnerTokenSsmPath: payload.runnerTokenSsmPath, + } satisfies MicrovmRunHookPayloadV1); +} + +function createMicrovmMetadataTags( + config: CreateGitHubRunnerConfig, + environment: string, + ssmConfigPath: string, + ssmParameterStoreTags: MicrovmMetadataTag[], +): MicrovmMetadataTag[] { + return [ + ...ssmParameterStoreTags.filter((tag) => !MICROVM_METADATA_CONTEXT_TAG_KEYS.has(tag.Key)), + { Key: 'ghr:environment', Value: environment }, + { Key: 'ghr:runner_name_prefix', Value: config.runnerNamePrefix }, + { Key: 'ghr:ssm_config_path', Value: ssmConfigPath }, + ]; +} + +function loadSsmParameterStoreTags(): MicrovmMetadataTag[] { + const encodedTags = process.env.SSM_PARAMETER_STORE_TAGS; + if (encodedTags === undefined || encodedTags.trim() === '') { + return []; + } + + try { + const parsed: unknown = JSON.parse(encodedTags); + if (!Array.isArray(parsed)) { + throw new Error('tags must be an array'); + } + + return parsed.map((tag, index) => { + if ( + tag === null || + typeof tag !== 'object' || + typeof (tag as Record).Key !== 'string' || + typeof (tag as Record).Value !== 'string' + ) { + throw new Error(`tag at index ${index} is invalid`); + } + const candidate = tag as Record; + return { Key: candidate.Key as string, Value: candidate.Value as string }; + }); + } catch (error) { + throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(error as Error).message}`); + } +} + +export async function createMicrovmRunners( + githubRunnerConfig: CreateGitHubRunnerConfig, + numberOfRunners: number, + githubInstallationClient: Octokit, + createStartRunnerConfig: CreateStartRunnerConfig, + source: RunnerSource, + overrides: MicrovmDynamicLabelOverrides = {}, +): Promise { + if (!githubRunnerConfig.ephemeral || !githubRunnerConfig.enableJitConfig) { + logger.error('Lambda MicroVM runners require ephemeral runners with JIT configuration enabled'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + if (!process.env.SSM_TOKEN_PATH?.trim()) { + logger.error('Lambda MicroVM runners require SSM_TOKEN_PATH to deliver JIT configuration'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + if (!process.env.SSM_CONFIG_PATH?.trim()) { + logger.error('Lambda MicroVM runners require SSM_CONFIG_PATH to locate runner metadata'); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + let config; + let normalizedRunnerConfigPath: string; + let normalizedRunnerTokenPath: string; + let ssmParameterStoreTags: MicrovmMetadataTag[]; + try { + config = { ...loadMicrovmProviderConfig(), ...overrides }; + normalizedRunnerConfigPath = normalizeMicrovmSsmPath(process.env.SSM_CONFIG_PATH); + normalizedRunnerTokenPath = normalizeMicrovmSsmPath(process.env.SSM_TOKEN_PATH); + assertMatchingMicrovmRunnerTokenPath(config.runnerTokenSsmPath, normalizedRunnerTokenPath); + assertSeparatedMicrovmMetadataPath(config.metadataSsmPath, config.runnerTokenSsmPath); + ssmParameterStoreTags = loadSsmParameterStoreTags(); + } catch (error) { + logger.error('Invalid Lambda MicroVM provider configuration', { error }); + return { instances: [], retryableErrorCount: 0, nonRetryableErrorCount: numberOfRunners }; + } + + const result: CreateRunnerResult = { + instances: [], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }; + const runHookPayload = createMicrovmRunHookPayload({ + ...(config.imageVersion !== undefined + ? { + imageArn: config.imageIdentifier, + imageVersion: config.imageVersion, + } + : {}), + runnerConfigSsmPath: normalizedRunnerConfigPath, + runnerTokenSsmPath: normalizedRunnerTokenPath, + }); + const environment = process.env.ENVIRONMENT; + const metadataTags = createMicrovmMetadataTags( + githubRunnerConfig, + environment, + normalizedRunnerConfigPath, + ssmParameterStoreTags, + ); + + for (let runnerIndex = 0; runnerIndex < numberOfRunners; runnerIndex++) { + let microvmId: string | undefined; + try { + const runner = await runMicrovmRunner({ + config, + environment, + runHookPayload, + runnerOwner: githubRunnerConfig.runnerOwner, + runnerType: githubRunnerConfig.runnerType, + ssmParameterStoreTags: metadataTags, + source, + }); + microvmId = runner.microvmId; + + const failedRunnerIds = await createStartRunnerConfig(githubRunnerConfig, [microvmId], githubInstallationClient, { + getRunnerConfigMetadata: (runnerId) => [{ key: 'MicrovmId', value: runnerId }], + onJitConfigCreated: async (runnerId, metadata) => { + await setMicrovmGithubRunnerMetadata(config, runnerId, metadata, runner.metadataTags); + }, + }); + + if (failedRunnerIds.includes(microvmId)) { + await terminateMicrovm(microvmId, config).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after JIT configuration failed`, { + error: terminationError, + }); + }); + result.retryableErrorCount++; + } else { + result.instances.push(microvmId); + } + } catch (error) { + if (microvmId) { + await terminateMicrovm(microvmId, config).catch((terminationError) => { + logger.error(`Failed to terminate MicroVM runner '${microvmId}' after setup failed`, { + error: terminationError, + }); + }); + } + + const retryable = isRetryableMicrovmError(error); + logger.error('Failed to create Lambda MicroVM runner', { error, retryable }); + if (retryable) result.retryableErrorCount++; + else result.nonRetryableErrorCount++; + } + } + + return result; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts new file mode 100644 index 0000000000..502fc77d05 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.test.ts @@ -0,0 +1,631 @@ +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + assertMatchingMicrovmRunnerTokenPath, + assertSeparatedMicrovmMetadataPath, + createMicrovmRunnerMetadata, + deleteMicrovmRunnerJitConfig, + deleteMicrovmRunnerSsmState, + listMicrovmRunnerMetadata, + markMicrovmCleanupPending, + microvmMetadataParameterName, + microvmRunnerJitParameterName, + setMicrovmGithubRunnerMetadata, + setMicrovmOrphan, + type MicrovmRunnerMetadata, +} from './runner-metadata'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + addParameterTags: vi.fn(), + deleteParameter: vi.fn(), + getParameters: vi.fn(), + getParametersByPath: vi.fn(), + putParameter: vi.fn(), +})); + +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const ssmPaths = { metadataSsmPath, runnerTokenSsmPath }; +const launchTags = [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, +]; + +function metadata(overrides: Partial = {}): MicrovmRunnerMetadata { + return { + version: 1, + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + createdAt: '2026-08-19T10:00:00.000Z', + expiresAt: '2026-08-19T11:00:00.000Z', + ...overrides, + }; +} + +function states(entries: [string, MicrovmState][]): Map { + return new Map(entries); +} + +beforeEach(() => { + vi.clearAllMocks(); + vi.useRealTimers(); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(addParameterTags).mockResolvedValue(); + vi.mocked(getParameters).mockImplementation(async (names) => new Map([[names[0], '{}']])); + vi.mocked(getParametersByPath).mockResolvedValue(new Map()); + vi.mocked(putParameter).mockResolvedValue(); +}); + +describe('MicroVM metadata paths', () => { + it('uses one base parameter per validated MicroVM ID', () => { + expect(microvmMetadataParameterName(`${metadataSsmPath}/`, 'microvm-123')).toBe(`${metadataSsmPath}/microvm-123`); + expect(() => microvmMetadataParameterName(metadataSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + expect(microvmRunnerJitParameterName(`${runnerTokenSsmPath}/`, 'microvm-123')).toBe( + `${runnerTokenSsmPath}/microvm-123`, + ); + expect(() => microvmRunnerJitParameterName(runnerTokenSsmPath, '../other')).toThrow('Invalid MicroVM identifier'); + }); + + it('requires metadata to use a prefix separate from JIT configuration', () => { + expect(() => + assertSeparatedMicrovmMetadataPath(metadataSsmPath, '/github-action-runners/unit-test/token'), + ).not.toThrow(); + expect(() => assertSeparatedMicrovmMetadataPath('/runner/token/metadata', '/runner/token')).toThrow( + 'must be separate', + ); + expect(() => assertSeparatedMicrovmMetadataPath('/runner', '/runner/token')).toThrow('must be separate'); + expect(() => assertMatchingMicrovmRunnerTokenPath(`${runnerTokenSsmPath}/`, runnerTokenSsmPath)).not.toThrow(); + expect(() => assertMatchingMicrovmRunnerTokenPath('/runner/other-token', runnerTokenSsmPath)).toThrow( + 'must match the runner JIT token path', + ); + }); +}); + +describe('MicroVM metadata lifecycle', () => { + it('creates non-secret, expiring ownership metadata without overwrite', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T10:00:00.000Z')); + + const createdTags = await createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'Name', Value: 'not-used-for-microvm-metadata' }, + { Key: 'ghr:Owner', Value: 'configured-owner-cannot-win' }, + { Key: 'ghr:created_by', Value: 'configured-source-cannot-win' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:github_runner_id', Value: 'configured-id-is-not-launch-metadata' }, + { Key: 'ghr:runner_labels', Value: 'configured-labels-are-not-launch-metadata' }, + ], + }); + + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1`, + JSON.stringify(metadata({ expiresAt: '2026-08-19T18:05:00.000Z' })), + false, + { + tags: [ + { Key: 'CostCenter', Value: '1234' }, + { Key: 'ghr:Owner', Value: 'Codertocat' }, + { Key: 'ghr:created_by', Value: 'scale-up-lambda' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:runner_name_prefix', Value: 'unit-test-' }, + { Key: 'ghr:ssm_config_path', Value: '/github-action-runners/unit-test/config' }, + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:Type', Value: 'Org' }, + { Key: 'ghr:microvm_id', Value: 'mvm-1' }, + { + Key: 'ghr:microvm_image_arn', + Value: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + }, + { Key: 'ghr:microvm_image_version', Value: '3.0' }, + ], + }, + ); + expect(createdTags).toEqual(vi.mocked(putParameter).mock.calls[0][3]?.tags); + }); + + it('rejects reserved tag keys and preserves room for late GitHub metadata', async () => { + const input = { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org' as const, + source: 'scale-up-lambda' as const, + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: [], + }; + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + ssmParameterStoreTags: [{ Key: 'aws:microvm:image-arn', Value: input.imageArn }], + }), + ).rejects.toThrow('AWS-reserved tag prefix'); + + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + ...input, + ssmParameterStoreTags: Array.from({ length: 37 }, (_, index) => ({ + Key: `Custom${index}`, + Value: 'value', + })), + }), + ).rejects.toThrow('cannot have more than 44 launch tags'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('rejects launch tags whose complete serialized metadata could exceed the Parameter Store value limit', async () => { + await expect( + createMicrovmRunnerMetadata(metadataSsmPath, { + microvmId: 'mvm-1', + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + source: 'scale-up-lambda', + imageArn: 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner', + imageVersion: '3.0', + ssmParameterStoreTags: Array.from({ length: 20 }, (_, index) => ({ + Key: `Custom${index}${'k'.repeat(100)}`, + Value: 'v'.repeat(256), + })), + }), + ).rejects.toThrow('cannot exceed 8192 bytes when serialized'); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('loads active metadata and schedules expired or invalid inactive records for two-phase cleanup', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + const active = metadata({ expiresAt: '2026-08-19T12:30:00.000Z' }); + const expiredInactive = metadata({ microvmId: 'mvm-old', expiresAt: '2026-08-19T11:00:00.000Z' }); + const unexpiredInactive = metadata({ microvmId: 'mvm-new', expiresAt: '2026-08-19T12:30:00.000Z' }); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(active)], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.orphan`, 'true'], + [`${metadataSsmPath}/mvm-old`, JSON.stringify(expiredInactive)], + [`${metadataSsmPath}/mvm-new`, JSON.stringify(unexpiredInactive)], + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-old', 'mvm-invalid'], + metadataById: new Map([['mvm-1', { ...active, githubRunnerId: 'github-42', orphan: true }]]), + }); + expect(getParametersByPath).toHaveBeenCalledWith(metadataSsmPath); + expect(deleteParameter).not.toHaveBeenCalled(); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-new`); + }); + + it('fails closed for invalid ownership metadata belonging to an active MicroVM', async () => { + vi.mocked(getParametersByPath).mockResolvedValue(new Map([[`${metadataSsmPath}/mvm-1`, '{not-json']])); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'invalid ownership metadata', + ); + }); + + it('schedules provider-owned metadata with invalid orphan state for two-phase cleanup', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.orphan`, 'invalid'], + ]), + ); + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + }); + + it('propagates metadata path lookup errors so inventory fails closed', async () => { + vi.mocked(getParametersByPath).mockRejectedValue(new Error('AccessDenied')); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow('AccessDenied'); + }); + + it('updates GitHub state and adds late GitHub metadata tags to the base parameter', async () => { + const runnerLabels = ['self-hosted', 'linux', 'env:unit-test']; + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.tags`, expect.any(String), false, { + overwrite: true, + }); + const tagsValue = vi.mocked(putParameter).mock.calls.find(([name]) => name.endsWith('.tags'))?.[1]; + expect(JSON.parse(tagsValue ?? '{}')).toEqual({ + CostCenter: '1234', + 'ghr:Application': 'github-action-runner', + 'ghr:github_runner_id': 'github-42', + 'ghr:microvm_id': 'mvm-1', + 'ghr:runner_labels': `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }); + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify(runnerLabels), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('revokes JIT configuration when cleanup starts before late metadata is recorded', async () => { + vi.mocked(getParameters).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, '{}'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when ownership metadata is already absent', async () => { + vi.mocked(getParameters).mockResolvedValue(new Map()); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('no longer accepting JIT configuration'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('revokes JIT configuration when the post-write ownership fence cannot be read', async () => { + vi.mocked(getParameters).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + expect(putParameter).not.toHaveBeenCalled(); + }); + + it('splits encoded runner labels into SSM-safe tag values', async () => { + const runnerLabels = [`label-${'a'.repeat(140)}`, `label-${'b'.repeat(140)}`]; + + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels, + }, + launchTags, + ); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + { + Key: 'ghr:runner_labels', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[0]]), 'utf8').toString('base64url')}`, + }, + { + Key: 'ghr:runner_labels:2', + Value: `base64url:${Buffer.from(JSON.stringify([runnerLabels[1]]), 'utf8').toString('base64url')}`, + }, + ]); + }); + + it('keeps the GitHub runner ID tag when a runner label is too large', async () => { + await setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: ['x'.repeat(300)], + }, + launchTags, + ); + + expect(addParameterTags).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1`, [ + { Key: 'ghr:github_runner_id', Value: 'github-42' }, + ]); + }); + + it('keeps the durable GitHub runner ID when late metadata tagging fails', async () => { + vi.mocked(addParameterTags).mockRejectedValue(new Error('AccessDenied')); + + await expect( + setMicrovmGithubRunnerMetadata( + ssmPaths, + 'mvm-1', + { + githubRunnerId: 'github-42', + runnerLabels: [], + }, + launchTags, + ), + ).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42', false, { + overwrite: true, + }); + }); + + it('fails JIT setup when the canonical tag-value parameter cannot be written', async () => { + vi.mocked(putParameter).mockImplementation(async (name) => { + if (name.endsWith('.tags')) throw new Error('AccessDenied'); + }); + + await expect( + setMicrovmGithubRunnerMetadata(ssmPaths, 'mvm-1', { githubRunnerId: 'github-42', runnerLabels: [] }, launchTags), + ).rejects.toThrow('AccessDenied'); + expect(addParameterTags).not.toHaveBeenCalled(); + }); + + it('updates orphan state without a shared read-modify-write record', async () => { + await setMicrovmOrphan(metadataSsmPath, 'mvm-1', true); + expect(putParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-1.orphan`, 'true', false, { + overwrite: true, + }); + }); + + it('marks cleanup independently and deletes JIT plus metadata while retaining the tombstone until last', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + await deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1'); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.tags`, + `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + ]); + }); + + it('does not reset the cleanup grace window when its tombstone already exists', async () => { + vi.mocked(putParameter).mockRejectedValueOnce( + Object.assign(new Error('ParameterAlreadyExists'), { __type: 'ParameterAlreadyExists' }), + ); + + await expect(markMicrovmCleanupPending(metadataSsmPath, 'mvm-1')).resolves.toBeUndefined(); + expect(putParameter).toHaveBeenCalledOnce(); + }); + + it('continues deleting metadata when optional parameters are already absent', async () => { + vi.mocked(deleteParameter) + .mockRejectedValueOnce( + Object.assign(new Error('ParameterNotFound'), { + __type: 'ParameterNotFound', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }), + ) + .mockRejectedValueOnce(Object.assign(new Error('missing parameter'), { name: 'ParameterNotFound' })); + + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).resolves.toBeUndefined(); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.github-runner-id`, + `${metadataSsmPath}/mvm-1.orphan`, + `${metadataSsmPath}/mvm-1.tags`, + `${metadataSsmPath}/mvm-1`, + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + ]); + }); + + it('propagates metadata deletion failures other than missing parameters', async () => { + const error = Object.assign(new Error('AccessDeniedException'), { + __type: 'AccessDeniedException', + $fault: 'client', + $metadata: { httpStatusCode: 400 }, + }); + vi.mocked(deleteParameter).mockRejectedValueOnce(error); + + await expect(deleteMicrovmRunnerSsmState(ssmPaths, 'mvm-1')).rejects.toBe(error); + expect(deleteParameter).toHaveBeenCalledTimes(1); + }); + + it('deletes only the lane JIT parameter when revoking pending runner configuration', async () => { + await deleteMicrovmRunnerJitConfig(runnerTokenSsmPath, 'mvm-1'); + + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-1`); + }); + + it('returns tracked and state-only active cleanup requests for termination retry', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-untracked.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + [`${metadataSsmPath}/mvm-terminating.cleanup-requested-at`, '2026-08-19T10:15:00.000Z'], + ]), + ); + + await expect( + listMicrovmRunnerMetadata( + ssmPaths, + states([ + ['mvm-1', 'RUNNING'], + ['mvm-untracked', 'PENDING'], + ['mvm-terminating', 'TERMINATING'], + ]), + ), + ).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1', 'mvm-untracked'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('does not starve cleanup requests when more than one reconciliation batch is pending', async () => { + const cleanupIds = Array.from({ length: 11 }, (_, index) => `mvm-cleanup-${index}`); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map( + cleanupIds.map((microvmId) => [ + `${metadataSsmPath}/${microvmId}.cleanup-requested-at`, + '2026-08-19T10:15:00.000Z', + ]), + ), + ); + + await expect( + listMicrovmRunnerMetadata( + ssmPaths, + states(cleanupIds.map((microvmId): [string, MicrovmState] => [microvmId, 'RUNNING'])), + ), + ).resolves.toEqual({ cleanupMicrovmIds: cleanupIds, metadataById: new Map() }); + }); + + it('keeps cleanup discoverable through the grace window before deleting JIT and every metadata record', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-terminal.github-runner-id`, 'github-42'], + [`${metadataSsmPath}/mvm-missing.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + [`${metadataSsmPath}/mvm-missing.tags`, '{"ghr:microvm_id":"mvm-missing"}'], + [`${metadataSsmPath}/mvm-recent.cleanup-requested-at`, '2026-08-19T11:59:00.000Z'], + [`${metadataSsmPath}/mvm-recent.tags`, '{"ghr:microvm_id":"mvm-recent"}'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-terminal', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-terminal', 'mvm-recent'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + expect(deleteParameter).toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing`); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-missing.tags`); + expect(deleteParameter).toHaveBeenLastCalledWith(`${metadataSsmPath}/mvm-missing.cleanup-requested-at`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-terminal`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${runnerTokenSsmPath}/mvm-recent`); + expect(deleteParameter).not.toHaveBeenCalledWith(`${metadataSsmPath}/mvm-recent`); + }); + + it('deletes invalid ownership metadata after its valid cleanup tombstone ages', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-invalid`, '{not-json'], + [`${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, '2026-08-19T11:54:59.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, new Map())).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(vi.mocked(deleteParameter).mock.calls.map(([name]) => name)).toEqual([ + `${runnerTokenSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.github-runner-id`, + `${metadataSsmPath}/mvm-invalid.orphan`, + `${metadataSsmPath}/mvm-invalid.tags`, + `${metadataSsmPath}/mvm-invalid`, + `${metadataSsmPath}/mvm-invalid.cleanup-requested-at`, + ]); + }); + + it('repairs an invalid cleanup timestamp before recreating the two-phase cleanup marker', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-08-19T12:00:00.000Z')); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, 'not-a-timestamp'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-1'], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith(`${metadataSsmPath}/mvm-1.cleanup-requested-at`); + + await markMicrovmCleanupPending(metadataSsmPath, 'mvm-1'); + expect(putParameter).toHaveBeenCalledWith( + `${metadataSsmPath}/mvm-1.cleanup-requested-at`, + '2026-08-19T12:00:00.000Z', + false, + ); + + vi.clearAllMocks(); + vi.setSystemTime(new Date('2026-08-19T12:06:00.000Z')); + vi.mocked(deleteParameter).mockResolvedValue(); + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([ + [`${metadataSsmPath}/mvm-1`, JSON.stringify(metadata())], + [`${metadataSsmPath}/mvm-1.cleanup-requested-at`, '2026-08-19T12:00:00.000Z'], + ]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: [], + metadataById: new Map(), + }); + expect(deleteParameter).toHaveBeenCalledTimes(6); + }); + + it('marks a terminal tags-only companion for two-phase cleanup instead of deleting it immediately', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-tags-only.tags`, '{"ghr:microvm_id":"mvm-tags-only"}']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-tags-only', 'TERMINATED']]))).resolves.toEqual({ + cleanupMicrovmIds: ['mvm-tags-only'], + metadataById: new Map(), + }); + expect(deleteParameter).not.toHaveBeenCalled(); + }); + + it('fails closed for active state metadata without ownership or a cleanup request', async () => { + vi.mocked(getParametersByPath).mockResolvedValue( + new Map([[`${metadataSsmPath}/mvm-1.github-runner-id`, 'github-42']]), + ); + + await expect(listMicrovmRunnerMetadata(ssmPaths, states([['mvm-1', 'RUNNING']]))).rejects.toThrow( + 'state metadata but no ownership metadata', + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts new file mode 100644 index 0000000000..0bae7a8d80 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/runner-metadata.ts @@ -0,0 +1,594 @@ +import { createChildLogger } from '@aws-github-runner/aws-powertools-util'; +import { + addParameterTags, + deleteParameter, + getParameters, + getParametersByPath, + putParameter, +} from '@aws-github-runner/aws-ssm-util'; +import type { MicrovmState } from '@aws-sdk/client-lambda-microvms'; + +import type { GitHubRunnerMetadata, RunnerSource, RunnerType } from '../../../../core'; +import { MICROVM_LIFETIME_IN_SECONDS } from './lifetime'; + +const logger = createChildLogger('microvm-runner-metadata'); + +const METADATA_VERSION = 1; +const EXPIRATION_GRACE_IN_SECONDS = 300; +const MAX_RECONCILED_RUNNERS = 10; +const MAX_PARAMETER_TAGS = 50; +const MAX_RUNNER_LABEL_TAGS = 5; +const MAX_BASE_PARAMETER_TAGS = MAX_PARAMETER_TAGS - MAX_RUNNER_LABEL_TAGS - 1; +const MAX_TAG_KEY_LENGTH = 128; +const MAX_TAG_VALUE_LENGTH = 256; +const MAX_PARAMETER_VALUE_SIZE_IN_BYTES = 8 * 1024; +const SSM_TAG_VALUE_PATTERN = /^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$/u; +const MICROVM_ID_PATTERN = /^[A-Za-z0-9_-]+$/; +const GITHUB_RUNNER_ID_SUFFIX = '.github-runner-id'; +const ORPHAN_SUFFIX = '.orphan'; +const CLEANUP_REQUESTED_AT_SUFFIX = '.cleanup-requested-at'; +const TAGS_SUFFIX = '.tags'; +const METADATA_COMPANION_SUFFIXES = [ + GITHUB_RUNNER_ID_SUFFIX, + ORPHAN_SUFFIX, + CLEANUP_REQUESTED_AT_SUFFIX, + TAGS_SUFFIX, +] as const; +const ACTIVE_STATES = new Set(['PENDING', 'RUNNING', 'SUSPENDING', 'SUSPENDED']); +export interface MicrovmMetadataTag { + Key: string; + Value: string; +} + +export interface MicrovmSsmPaths { + metadataSsmPath: string; + runnerTokenSsmPath: string; +} + +export interface MicrovmRunnerMetadata { + bypassRemoval?: boolean; + createdAt: string; + environment: string; + expiresAt: string; + githubRunnerId?: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + orphan?: boolean; + runnerOwner: string; + runnerType: RunnerType; + source: RunnerSource; + version: 1; +} + +export interface MicrovmRunnerMetadataInventory { + cleanupMicrovmIds: string[]; + metadataById: Map; +} + +export interface CreateMicrovmRunnerMetadataInput { + environment: string; + imageArn: string; + imageVersion?: string; + microvmId: string; + runnerOwner: string; + runnerType: RunnerType; + ssmParameterStoreTags: MicrovmMetadataTag[]; + source: RunnerSource; +} + +function isProviderOwnedLateTag(key: string): boolean { + return key === 'ghr:github_runner_id' || key === 'ghr:runner_labels' || key.startsWith('ghr:runner_labels:'); +} + +function assertValidParameterTags(tags: MicrovmMetadataTag[]): void { + if (tags.length > MAX_PARAMETER_TAGS) { + throw new Error(`MicroVM metadata cannot have more than ${MAX_PARAMETER_TAGS} tags`); + } + + for (const tag of tags) { + if ( + Array.from(tag.Key).length === 0 || + Array.from(tag.Key).length > MAX_TAG_KEY_LENGTH || + Array.from(tag.Value).length > MAX_TAG_VALUE_LENGTH || + !SSM_TAG_VALUE_PATTERN.test(tag.Key) || + !SSM_TAG_VALUE_PATTERN.test(tag.Value) + ) { + throw new Error(`MicroVM metadata tag '${tag.Key}' does not satisfy SSM tag constraints`); + } + if (tag.Key.toLowerCase().startsWith('aws:')) { + throw new Error(`MicroVM metadata tag '${tag.Key}' uses the AWS-reserved tag prefix`); + } + } +} + +function mergeParameterTags(...tagSets: MicrovmMetadataTag[][]): MicrovmMetadataTag[] { + const tagsByKey = new Map(); + for (const tags of tagSets) { + for (const tag of tags) tagsByKey.set(tag.Key, tag.Value); + } + + return [...tagsByKey].map(([Key, Value]) => ({ Key, Value })); +} + +function serializeParameterTags(tags: MicrovmMetadataTag[]): string { + assertValidParameterTags(tags); + const tagValues: Record = Object.create(null) as Record; + for (const { Key, Value } of [...tags].sort((left, right) => + left.Key < right.Key ? -1 : left.Key > right.Key ? 1 : 0, + )) { + tagValues[Key] = Value; + } + + const value = JSON.stringify(tagValues); + if (Buffer.byteLength(value, 'utf8') > MAX_PARAMETER_VALUE_SIZE_IN_BYTES) { + throw new Error(`MicroVM metadata tags cannot exceed ${MAX_PARAMETER_VALUE_SIZE_IN_BYTES} bytes when serialized`); + } + return value; +} + +function maximumGitHubRunnerMetadataTags(): MicrovmMetadataTag[] { + return [ + { Key: 'ghr:github_runner_id', Value: '0'.repeat(MAX_TAG_VALUE_LENGTH) }, + ...Array.from({ length: MAX_RUNNER_LABEL_TAGS }, (_, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value: '0'.repeat(MAX_TAG_VALUE_LENGTH), + })), + ]; +} + +function createMetadataParameterTags(input: CreateMicrovmRunnerMetadataInput): MicrovmMetadataTag[] { + const configuredTags = mergeParameterTags(input.ssmParameterStoreTags).filter( + (tag) => !isProviderOwnedLateTag(tag.Key) && tag.Key !== 'ghr:microvm_image_version' && tag.Key !== 'Name', + ); + const providerTags: MicrovmMetadataTag[] = [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: input.source }, + { Key: 'ghr:environment', Value: input.environment }, + { Key: 'ghr:Owner', Value: input.runnerOwner }, + { Key: 'ghr:Type', Value: input.runnerType }, + { Key: 'ghr:microvm_id', Value: input.microvmId }, + { Key: 'ghr:microvm_image_arn', Value: input.imageArn }, + ]; + if (input.imageVersion !== undefined) { + providerTags.push({ Key: 'ghr:microvm_image_version', Value: input.imageVersion }); + } + + const tags = mergeParameterTags(configuredTags, providerTags); + assertValidParameterTags(tags); + if (tags.length > MAX_BASE_PARAMETER_TAGS) { + throw new Error( + `MicroVM metadata cannot have more than ${MAX_BASE_PARAMETER_TAGS} launch tags because ${MAX_RUNNER_LABEL_TAGS + 1} tags are reserved for GitHub runner metadata`, + ); + } + serializeParameterTags(mergeParameterTags(tags, maximumGitHubRunnerMetadataTags())); + return tags; +} + +export function assertValidMicrovmMetadataTags(input: CreateMicrovmRunnerMetadataInput): void { + createMetadataParameterTags(input); +} + +function encodeRunnerLabelGroups(labels: string[]): string[] { + const encodedGroups: string[] = []; + let group: string[] = []; + const encode = (values: string[]) => `base64url:${Buffer.from(JSON.stringify(values), 'utf8').toString('base64url')}`; + + for (const label of labels) { + const candidate = [...group, label]; + if (Array.from(encode(candidate)).length <= MAX_TAG_VALUE_LENGTH) { + group = candidate; + continue; + } + if (group.length === 0) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + continue; + } + encodedGroups.push(encode(group)); + group = [label]; + if (Array.from(encode(group)).length > MAX_TAG_VALUE_LENGTH) { + logger.warn('A GitHub runner label was omitted because its encoded value exceeds the SSM tag limit', { + labelLength: Array.from(label).length, + }); + group = []; + } + } + if (group.length > 0) encodedGroups.push(encode(group)); + + if (encodedGroups.length > MAX_RUNNER_LABEL_TAGS) { + logger.warn('GitHub runner label SSM tags were truncated to avoid exceeding the metadata tag budget', { + maxRunnerLabelsTagCount: MAX_RUNNER_LABEL_TAGS, + }); + } + return encodedGroups.slice(0, MAX_RUNNER_LABEL_TAGS); +} + +function createGitHubRunnerMetadataTags(metadata: GitHubRunnerMetadata): MicrovmMetadataTag[] { + const tags: MicrovmMetadataTag[] = [{ Key: 'ghr:github_runner_id', Value: metadata.githubRunnerId }]; + tags.push( + ...encodeRunnerLabelGroups(metadata.runnerLabels).map((Value, index) => ({ + Key: index === 0 ? 'ghr:runner_labels' : `ghr:runner_labels:${index + 1}`, + Value, + })), + ); + assertValidParameterTags(tags); + return tags; +} + +export function normalizeMicrovmSsmPath(path: string): string { + const normalized = path.trim().replace(/\/+$/, ''); + if (!/^\/[A-Za-z0-9_.\-/]+$/.test(normalized) || normalized.includes('//') || normalized.split('/').includes('..')) { + throw new Error(`Invalid SSM parameter path '${path}'`); + } + return normalized; +} + +export function microvmMetadataParameterName(metadataSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(metadataSsmPath)}/${microvmId}`; +} + +export function microvmRunnerJitParameterName(runnerTokenSsmPath: string, microvmId: string): string { + if (!MICROVM_ID_PATTERN.test(microvmId)) { + throw new Error(`Invalid MicroVM identifier '${microvmId}'`); + } + return `${normalizeMicrovmSsmPath(runnerTokenSsmPath)}/${microvmId}`; +} + +function stateParameterName(metadataSsmPath: string, microvmId: string, suffix: string): string { + return `${microvmMetadataParameterName(metadataSsmPath, microvmId)}${suffix}`; +} + +function metadataParameterNames(metadataSsmPath: string, microvmId: string): string[] { + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + return [ + `${baseName}${GITHUB_RUNNER_ID_SUFFIX}`, + `${baseName}${ORPHAN_SUFFIX}`, + `${baseName}${TAGS_SUFFIX}`, + baseName, + `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`, + ]; +} + +export function assertSeparatedMicrovmMetadataPath(metadataSsmPath: string, runnerTokenSsmPath: string): void { + const metadataPath = normalizeMicrovmSsmPath(metadataSsmPath); + const runnerTokenPath = normalizeMicrovmSsmPath(runnerTokenSsmPath); + if ( + metadataPath === runnerTokenPath || + metadataPath.startsWith(`${runnerTokenPath}/`) || + runnerTokenPath.startsWith(`${metadataPath}/`) + ) { + throw new Error('MICROVM_METADATA_SSM_PATH must be separate from the runner JIT token path'); + } +} + +export function assertMatchingMicrovmRunnerTokenPath( + configuredRunnerTokenSsmPath: string, + runnerTokenSsmPath: string, +): void { + if (normalizeMicrovmSsmPath(configuredRunnerTokenSsmPath) !== normalizeMicrovmSsmPath(runnerTokenSsmPath)) { + throw new Error('MicroVM provider SSM_TOKEN_PATH must match the runner JIT token path'); + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isParameterError(error: unknown, type: string): boolean { + return error instanceof Error && (error.name === type || ('__type' in error && error.__type === type)); +} + +function isParameterNotFound(error: unknown): boolean { + return isParameterError(error, 'ParameterNotFound'); +} + +function optionalString(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length > 0); +} + +function optionalBoolean(value: unknown): value is boolean | undefined { + return value === undefined || typeof value === 'boolean'; +} + +function parseMetadata(value: string, expectedMicrovmId: string): MicrovmRunnerMetadata | undefined { + let parsed: unknown; + try { + parsed = JSON.parse(value); + } catch { + return undefined; + } + + if (!isRecord(parsed)) return undefined; + + const createdAt = typeof parsed.createdAt === 'string' ? Date.parse(parsed.createdAt) : Number.NaN; + const expiresAt = typeof parsed.expiresAt === 'string' ? Date.parse(parsed.expiresAt) : Number.NaN; + if ( + parsed.version !== METADATA_VERSION || + parsed.microvmId !== expectedMicrovmId || + typeof parsed.environment !== 'string' || + parsed.environment.length === 0 || + typeof parsed.runnerOwner !== 'string' || + parsed.runnerOwner.length === 0 || + (parsed.runnerType !== 'Org' && parsed.runnerType !== 'Repo') || + (parsed.source !== 'scale-up-lambda' && parsed.source !== 'pool-lambda') || + typeof parsed.imageArn !== 'string' || + parsed.imageArn.length === 0 || + !optionalString(parsed.imageVersion) || + !optionalBoolean(parsed.bypassRemoval) || + !Number.isFinite(createdAt) || + !Number.isFinite(expiresAt) || + expiresAt <= createdAt + ) { + return undefined; + } + + return { + version: METADATA_VERSION, + microvmId: expectedMicrovmId, + environment: parsed.environment, + runnerOwner: parsed.runnerOwner, + runnerType: parsed.runnerType, + source: parsed.source, + imageArn: parsed.imageArn, + imageVersion: parsed.imageVersion, + bypassRemoval: parsed.bypassRemoval, + createdAt: parsed.createdAt as string, + expiresAt: parsed.expiresAt as string, + }; +} + +export async function createMicrovmRunnerMetadata( + metadataSsmPath: string, + input: CreateMicrovmRunnerMetadataInput, +): Promise { + const createdAt = new Date(); + const metadata: MicrovmRunnerMetadata = { + version: METADATA_VERSION, + microvmId: input.microvmId, + environment: input.environment, + runnerOwner: input.runnerOwner, + runnerType: input.runnerType, + source: input.source, + imageArn: input.imageArn, + imageVersion: input.imageVersion, + createdAt: createdAt.toISOString(), + expiresAt: new Date( + createdAt.getTime() + (MICROVM_LIFETIME_IN_SECONDS + EXPIRATION_GRACE_IN_SECONDS) * 1000, + ).toISOString(), + }; + + const metadataTags = createMetadataParameterTags(input); + await putParameter(microvmMetadataParameterName(metadataSsmPath, input.microvmId), JSON.stringify(metadata), false, { + tags: metadataTags, + }); + return metadataTags; +} + +function invalidOrphanState(parameters: Map, baseName: string): boolean { + const orphan = parameters.get(`${baseName}${ORPHAN_SUFFIX}`); + return orphan !== undefined && orphan !== 'true' && orphan !== 'false'; +} + +type CleanupRequestStatus = 'absent' | 'elapsed' | 'invalid' | 'pending'; + +function cleanupRequestStatus(parameters: Map, baseName: string, now: number): CleanupRequestStatus { + const cleanupRequestedAt = parameters.get(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + if (cleanupRequestedAt === undefined) return 'absent'; + const requestedAt = Date.parse(cleanupRequestedAt); + if (!Number.isFinite(requestedAt)) return 'invalid'; + return requestedAt + EXPIRATION_GRACE_IN_SECONDS * 1000 <= now ? 'elapsed' : 'pending'; +} + +export async function listMicrovmRunnerMetadata( + paths: MicrovmSsmPaths, + microvmStates: ReadonlyMap, +): Promise { + const { metadataSsmPath } = paths; + const metadataById = new Map(); + const cleanupMicrovmIds = new Set(); + const parameters = await getParametersByPath(normalizeMicrovmSsmPath(metadataSsmPath)); + const parameterPrefix = `${normalizeMicrovmSsmPath(metadataSsmPath)}/`; + const now = Date.now(); + const metadataBaseIds = new Set(); + const stateParameterIds = new Set(); + const runnersToDelete = new Set(); + + for (const parameterName of parameters.keys()) { + if (!parameterName.startsWith(parameterPrefix)) continue; + for (const suffix of METADATA_COMPANION_SUFFIXES) { + if (!parameterName.endsWith(suffix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length, -suffix.length); + if (MICROVM_ID_PATTERN.test(microvmId)) stateParameterIds.add(microvmId); + break; + } + } + + for (const [parameterName, value] of parameters) { + if (!parameterName.startsWith(parameterPrefix)) continue; + const microvmId = parameterName.slice(parameterPrefix.length); + if (!MICROVM_ID_PATTERN.test(microvmId)) continue; + metadataBaseIds.add(microvmId); + + const state = microvmStates.get(microvmId); + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else { + cleanupMicrovmIds.add(microvmId); + } + continue; + } + + const metadata = parseMetadata(value, microvmId); + if (!metadata) { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has invalid ownership metadata`); + } + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + } + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling invalid MicroVM runner metadata for '${microvmId}' for cleanup`); + continue; + } + + if (cleanupStatus === 'invalid') { + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + logger.warn(`Repairing invalid cleanup request metadata for '${microvmId}'`); + continue; + } + + if (invalidOrphanState(parameters, baseName)) { + cleanupMicrovmIds.add(microvmId); + logger.warn(`Scheduling MicroVM runner metadata for '${microvmId}' with invalid orphan state for cleanup`); + continue; + } + + if (state === 'TERMINATED') { + cleanupMicrovmIds.add(microvmId); + continue; + } + if (state === undefined) { + if (Date.parse(metadata.expiresAt) <= now) cleanupMicrovmIds.add(microvmId); + continue; + } + if (!ACTIVE_STATES.has(state)) continue; + + metadataById.set(microvmId, { + ...metadata, + githubRunnerId: parameters.get(`${baseName}${GITHUB_RUNNER_ID_SUFFIX}`), + orphan: parameters.get(`${baseName}${ORPHAN_SUFFIX}`) === 'true', + }); + } + + for (const microvmId of stateParameterIds) { + if (metadataBaseIds.has(microvmId)) continue; + + const baseName = microvmMetadataParameterName(metadataSsmPath, microvmId); + const state = microvmStates.get(microvmId); + const cleanupStatus = cleanupRequestStatus(parameters, baseName, now); + + if (cleanupStatus === 'pending' || cleanupStatus === 'elapsed') { + if (cleanupStatus === 'elapsed' && (state === undefined || state === 'TERMINATED')) { + runnersToDelete.add(microvmId); + } else if (state === undefined || state === 'TERMINATED' || ACTIVE_STATES.has(state)) { + cleanupMicrovmIds.add(microvmId); + } + continue; + } + + if (cleanupStatus === 'invalid') { + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has an invalid cleanup request timestamp`); + } + await deleteParameterIfPresent(`${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`); + cleanupMicrovmIds.add(microvmId); + continue; + } + + if (state !== undefined && ACTIVE_STATES.has(state)) { + throw new Error(`Active MicroVM runner '${microvmId}' has state metadata but no ownership metadata`); + } + if (state === 'TERMINATED' || state === undefined) { + cleanupMicrovmIds.add(microvmId); + } + } + + for (const microvmId of [...runnersToDelete].slice(0, MAX_RECONCILED_RUNNERS)) { + try { + await deleteMicrovmRunnerSsmState(paths, microvmId); + } catch (error) { + logger.warn(`Failed to delete reconciled MicroVM runner metadata '${microvmId}'`, { error }); + } + } + + return { + cleanupMicrovmIds: [...cleanupMicrovmIds], + metadataById, + }; +} + +export async function setMicrovmGithubRunnerMetadata( + paths: MicrovmSsmPaths, + microvmId: string, + metadata: GitHubRunnerMetadata, + launchTags: MicrovmMetadataTag[], +): Promise { + if (!metadata.githubRunnerId) throw new Error('GitHub runner ID must not be empty'); + const baseName = microvmMetadataParameterName(paths.metadataSsmPath, microvmId); + const cleanupMarkerName = `${baseName}${CLEANUP_REQUESTED_AT_SUFFIX}`; + try { + const parameters = await getParameters([baseName, cleanupMarkerName]); + if (!parameters.has(baseName) || parameters.has(cleanupMarkerName)) { + throw new Error(`MicroVM runner '${microvmId}' is no longer accepting JIT configuration`); + } + } catch (error) { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + throw error; + } + + const githubRunnerTags = createGitHubRunnerMetadataTags(metadata); + const tags = mergeParameterTags(launchTags, githubRunnerTags); + const serializedTags = serializeParameterTags(tags); + await putParameter( + stateParameterName(paths.metadataSsmPath, microvmId, GITHUB_RUNNER_ID_SUFFIX), + metadata.githubRunnerId, + false, + { + overwrite: true, + }, + ); + await putParameter(stateParameterName(paths.metadataSsmPath, microvmId, TAGS_SUFFIX), serializedTags, false, { + overwrite: true, + }); + try { + await addParameterTags(baseName, githubRunnerTags); + } catch (error) { + logger.error(`Failed to tag MicroVM runner '${microvmId}' with GitHub runner metadata`, { error }); + } +} + +export async function setMicrovmOrphan(metadataSsmPath: string, microvmId: string, orphan: boolean): Promise { + await putParameter(stateParameterName(metadataSsmPath, microvmId, ORPHAN_SUFFIX), String(orphan), false, { + overwrite: true, + }); +} + +export async function markMicrovmCleanupPending(metadataSsmPath: string, microvmId: string): Promise { + try { + await putParameter( + stateParameterName(metadataSsmPath, microvmId, CLEANUP_REQUESTED_AT_SUFFIX), + new Date().toISOString(), + false, + ); + } catch (error) { + if (!isParameterError(error, 'ParameterAlreadyExists')) throw error; + } +} + +async function deleteParameterIfPresent(parameterName: string): Promise { + try { + await deleteParameter(parameterName); + } catch (error) { + if (!isParameterNotFound(error)) throw error; + } +} + +export async function deleteMicrovmRunnerJitConfig(runnerTokenSsmPath: string, microvmId: string): Promise { + await deleteParameterIfPresent(microvmRunnerJitParameterName(runnerTokenSsmPath, microvmId)); +} + +export async function deleteMicrovmRunnerSsmState(paths: MicrovmSsmPaths, microvmId: string): Promise { + await deleteMicrovmRunnerJitConfig(paths.runnerTokenSsmPath, microvmId); + for (const parameterName of metadataParameterNames(paths.metadataSsmPath, microvmId)) { + await deleteParameterIfPresent(parameterName); + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts new file mode 100644 index 0000000000..f98eb88628 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.test.ts @@ -0,0 +1,79 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { loadMicrovmProviderConfig } from './config'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { createMicrovmScaleDownProvider } from './scale-down'; +import { setMicrovmOrphan } from './runner-metadata'; + +vi.mock('./config', () => ({ loadMicrovmProviderConfig: vi.fn() })); +vi.mock('./microvms', () => ({ + listMicrovmRunners: vi.fn(), + microvmBootTimeExceeded: vi.fn(), + terminateMicrovm: vi.fn(), +})); +vi.mock('./runner-metadata', () => ({ setMicrovmOrphan: vi.fn() })); + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner'; +const metadataSsmPath = '/github-action-runners/unit-test/microvm-metadata'; +const runnerTokenSsmPath = '/github-action-runners/unit-test/token'; +const providerConfig = { + imageIdentifier: imageArn, + executionRoleArn: 'arn:aws:iam::123456789012:role/microvm-runner', + metadataSsmPath, + runnerTokenSsmPath, +}; + +beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(loadMicrovmProviderConfig).mockReturnValue(providerConfig); + vi.mocked(listMicrovmRunners).mockResolvedValue([]); + vi.mocked(microvmBootTimeExceeded).mockReturnValue(false); + vi.mocked(setMicrovmOrphan).mockResolvedValue(); + vi.mocked(terminateMicrovm).mockResolvedValue(); +}); + +describe('createMicrovmScaleDownProvider', () => { + it('lists active and orphan runners through provider filters', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.list('unit-test'); + await provider.list('unit-test', true); + + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 1, + { + environment: 'unit-test', + orphan: undefined, + }, + providerConfig, + ); + expect(listMicrovmRunners).toHaveBeenNthCalledWith( + 2, + { + environment: 'unit-test', + orphan: true, + }, + providerConfig, + ); + }); + + it('uses durable metadata when marking, unmarking, and terminating runners', async () => { + const provider = createMicrovmScaleDownProvider(); + + await provider.markOrphan('mvm-1'); + await provider.unmarkOrphan('mvm-1'); + await provider.terminate('mvm-1'); + + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(1, metadataSsmPath, 'mvm-1', true); + expect(setMicrovmOrphan).toHaveBeenNthCalledWith(2, metadataSsmPath, 'mvm-1', false); + expect(terminateMicrovm).toHaveBeenCalledWith('mvm-1', providerConfig); + }); + + it('uses the MicroVM boot-time policy', () => { + const provider = createMicrovmScaleDownProvider(); + const runner = { id: 'mvm-1', owner: 'Codertocat', type: 'Org' as const }; + + expect(provider.bootTimeExceeded(runner)).toBe(false); + expect(microvmBootTimeExceeded).toHaveBeenCalledWith(runner); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts new file mode 100644 index 0000000000..82038711df --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-down.ts @@ -0,0 +1,21 @@ +import type { ScaleDownComputeProvider } from '../../../../core'; +import { loadMicrovmProviderConfig } from './config'; +import type { MicrovmRunnerInfo } from './microvms'; +import { listMicrovmRunners, microvmBootTimeExceeded, terminateMicrovm } from './microvms'; +import { setMicrovmOrphan } from './runner-metadata'; + +export function createMicrovmScaleDownProvider(): Omit { + const ssmPaths = () => loadMicrovmProviderConfig(); + + async function list(environment: string, orphan?: boolean): Promise { + return await listMicrovmRunners({ environment, orphan }, ssmPaths()); + } + + return { + list, + bootTimeExceeded: microvmBootTimeExceeded, + markOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, true), + unmarkOrphan: async (id) => await setMicrovmOrphan(ssmPaths().metadataSsmPath, id, false), + terminate: async (id) => await terminateMicrovm(id, ssmPaths()), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts new file mode 100644 index 0000000000..ab3d850b03 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.test.ts @@ -0,0 +1,112 @@ +import type { Octokit } from '@octokit/rest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { CreateGitHubRunnerConfig, CreateStartRunnerConfig } from '../../../../core'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; +import { createMicrovmScaleUpProvider } from './scale-up'; + +vi.mock('./microvms', () => ({ listMicrovmRunners: vi.fn() })); +vi.mock('./runner-config', () => ({ createMicrovmRunners: vi.fn() })); + +const createStartRunnerConfig = vi.fn(); +const githubClient = {} as Octokit; +const overrideImageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const overrideEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const githubRunnerConfig: CreateGitHubRunnerConfig = { + ephemeral: true, + enableJitConfig: true, + runnerLabels: 'self-hosted,linux,arm64,microvm', + runnerGroup: 'Default', + runnerNamePrefix: '', + runnerOwner: 'Codertocat', + runnerType: 'Org', + disableAutoUpdate: true, +}; + +beforeEach(() => { + vi.clearAllMocks(); + process.env.ENVIRONMENT = 'unit-test'; + vi.mocked(listMicrovmRunners).mockResolvedValue([ + { id: 'mvm-current', owner: 'Codertocat', type: 'Org', state: 'RUNNING' }, + ]); + vi.mocked(createMicrovmRunners).mockResolvedValue({ + instances: ['mvm-new'], + retryableErrorCount: 0, + nonRetryableErrorCount: 0, + }); +}); + +describe('createMicrovmScaleUpProvider', () => { + it('resolves supported resource override labels and registers them on the runner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.resolveLabelsForRunners([ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).resolves.toEqual({ + runnerLabels: [ + `ghr-microvm-egress-network-connectors:${overrideEgressConnectorArn}`, + `ghr-microvm-image-arn:${overrideImageArn}`, + 'ghr-microvm-image-version:3.0', + ], + state: { + overrides: { + egressNetworkConnectors: [overrideEgressConnectorArn], + imageIdentifier: overrideImageArn, + imageVersion: '3.0', + }, + }, + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects unsupported MicroVM override label %s at the control-plane boundary', async (label, reason) => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect(provider.resolveLabelsForRunners([label])).rejects.toThrow(reason); + }); + + it('counts managed MicroVMs for the runner owner', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.getCurrentRunners({ overrides: {} }, { runnerOwner: 'Codertocat', runnerType: 'Org' }), + ).resolves.toBe(1); + expect(listMicrovmRunners).toHaveBeenCalledWith({ + environment: 'unit-test', + runnerOwner: 'Codertocat', + runnerType: 'Org', + }); + }); + + it('delegates runner creation to the shared MicroVM lifecycle', async () => { + const provider = createMicrovmScaleUpProvider(createStartRunnerConfig); + + await expect( + provider.createRunners({ + githubRunnerConfig, + numberOfRunners: 1, + githubInstallationClient: githubClient, + state: { overrides: { imageVersion: '3.0' } }, + }), + ).resolves.toEqual({ instances: ['mvm-new'], retryableErrorCount: 0, nonRetryableErrorCount: 0 }); + expect(createMicrovmRunners).toHaveBeenCalledWith( + githubRunnerConfig, + 1, + githubClient, + createStartRunnerConfig, + 'scale-up-lambda', + { imageVersion: '3.0' }, + ); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts new file mode 100644 index 0000000000..a3dcf1219e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/control-plane/scale-up.ts @@ -0,0 +1,77 @@ +import type { + CreateRunnerResult, + CreateScaleUpRunnersInput, + CreateStartRunnerConfig, + CurrentRunnersInput, + RunnerLabelResolution, + ScaleUpComputeProvider, +} from '../../../../core'; +import type { MicrovmDynamicLabelOverrides } from '../dynamic-labels'; +import { parseMicrovmDynamicLabels } from '../dynamic-labels'; +import { listMicrovmRunners } from './microvms'; +import { createMicrovmRunners } from './runner-config'; + +interface MicrovmScaleUpState { + overrides: MicrovmDynamicLabelOverrides; +} + +async function resolveMicrovmLabelsForRunners( + messageLabels: string[], +): Promise> { + const trimmedLabels = messageLabels.map((label) => label.trim()); + const parsed = parseMicrovmDynamicLabels(trimmedLabels); + if (parsed.violations.length > 0) { + throw new Error( + `Invalid MicroVM dynamic labels: ${parsed.violations + .map((violation) => `${violation.label} (${violation.reason})`) + .join(', ')}`, + ); + } + + return { + runnerLabels: trimmedLabels.filter((label) => label.startsWith('ghr-')), + state: { overrides: parsed.overrides }, + }; +} + +async function getCurrentMicrovmRunners( + _state: MicrovmScaleUpState, + { runnerType, runnerOwner }: CurrentRunnersInput, +): Promise { + return ( + await listMicrovmRunners({ + environment: process.env.ENVIRONMENT, + runnerType, + runnerOwner, + }) + ).length; +} + +async function createMicrovmScaleUpRunners( + { + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + state, + }: CreateScaleUpRunnersInput, + createStartRunnerConfig: CreateStartRunnerConfig, +): Promise { + return await createMicrovmRunners( + githubRunnerConfig, + numberOfRunners, + githubInstallationClient, + createStartRunnerConfig, + 'scale-up-lambda', + state.overrides, + ); +} + +export function createMicrovmScaleUpProvider( + createStartRunnerConfig: CreateStartRunnerConfig, +): Omit, 'type'> { + return { + resolveLabelsForRunners: resolveMicrovmLabelsForRunners, + getCurrentRunners: getCurrentMicrovmRunners, + createRunners: (input) => createMicrovmScaleUpRunners(input, createStartRunnerConfig), + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts new file mode 100644 index 0000000000..442986a0f8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from 'vitest'; + +import { parseMicrovmDynamicLabels } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; +const internetEgressConnectorArn = + 'arn:aws:lambda:eu-west-1:aws:network-connector:aws-network-connector:INTERNET_EGRESS'; + +describe('parseMicrovmDynamicLabels', () => { + it('parses every supported RunMicrovm override', () => { + expect( + parseMicrovmDynamicLabels([ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-egress-network-connectors:${internetEgressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).toEqual({ + overrides: { + egressNetworkConnectors: [egressConnectorArn, internetEgressConnectorArn], + imageIdentifier: imageArn, + imageVersion: '3.0', + }, + violations: [], + }); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-egress-network-connectors:not-an-arn', + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn};${internetEgressConnectorArn}`, + 'is not a valid Lambda network connector ARN; specify one ARN per label', + ], + ['ghr-microvm-image-arn:not-an-arn', 'is not a valid customer MicroVM image ARN'], + ['ghr-microvm-image-version:', "key 'image-version' requires a value"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('rejects invalid override %s', (label, reason) => { + const result = parseMicrovmDynamicLabels([label]); + + expect(result.overrides).toEqual({}); + expect(result.violations).toEqual([{ label, reason: expect.stringContaining(reason) }]); + }); + + it('ignores generic dynamic labels', () => { + expect(parseMicrovmDynamicLabels(['ghr-team:platform'])).toEqual({ overrides: {}, violations: [] }); + }); + + it('rejects more than ten egress network connectors', () => { + const labels = Array.from( + { length: 11 }, + (_, index) => + `ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:connector-${index}`, + ); + + const result = parseMicrovmDynamicLabels(labels); + + expect(result.overrides.egressNetworkConnectors).toHaveLength(10); + expect(result.violations).toEqual([ + { + label: labels[10], + reason: 'at most 10 egress network connector labels are supported', + }, + ]); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts new file mode 100644 index 0000000000..2851716149 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/dynamic-labels.ts @@ -0,0 +1,76 @@ +export const MICROVM_DYNAMIC_LABEL_PREFIX = 'ghr-microvm-'; + +const MAXIMUM_EGRESS_NETWORK_CONNECTORS = 10; +const MICROVM_IMAGE_ARN_PATTERN = /^arn:[^:]+:lambda:[^:]+:[0-9]{12}:microvm-image:.+$/; +const MICROVM_NETWORK_CONNECTOR_ARN_PATTERN = + /^arn:aws[a-zA-Z-]*:lambda:[a-z0-9-]+:(?:[0-9]{12}|aws):network-connector:[a-zA-Z0-9_-]+(?::[a-zA-Z0-9_-]+)?$/; + +export interface MicrovmDynamicLabelOverrides { + egressNetworkConnectors?: string[]; + imageIdentifier?: string; + imageVersion?: string; +} + +export interface MicrovmDynamicLabelViolation { + label: string; + reason: string; +} + +export function parseMicrovmDynamicLabels(labels: string[]): { + overrides: MicrovmDynamicLabelOverrides; + violations: MicrovmDynamicLabelViolation[]; +} { + const overrides: MicrovmDynamicLabelOverrides = {}; + const violations: MicrovmDynamicLabelViolation[] = []; + + for (const label of labels) { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) continue; + + const stripped = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length); + const colonIndex = stripped.indexOf(':'); + const key = colonIndex === -1 ? stripped : stripped.slice(0, colonIndex); + const value = colonIndex === -1 ? '' : stripped.slice(colonIndex + 1).trim(); + + if (!value) { + violations.push({ label, reason: `key '${key}' requires a value` }); + continue; + } + + switch (key) { + case 'egress-network-connectors': { + if (!MICROVM_NETWORK_CONNECTOR_ARN_PATTERN.test(value)) { + violations.push({ + label, + reason: `'${value}' is not a valid Lambda network connector ARN; specify one ARN per label`, + }); + break; + } + + const connectors = overrides.egressNetworkConnectors ?? []; + if (connectors.length >= MAXIMUM_EGRESS_NETWORK_CONNECTORS) { + violations.push({ + label, + reason: `at most ${MAXIMUM_EGRESS_NETWORK_CONNECTORS} egress network connector labels are supported`, + }); + } else { + overrides.egressNetworkConnectors = [...connectors, value]; + } + break; + } + case 'image-arn': + if (!MICROVM_IMAGE_ARN_PATTERN.test(value)) { + violations.push({ label, reason: `'${value}' is not a valid customer MicroVM image ARN` }); + } else { + overrides.imageIdentifier = value; + } + break; + case 'image-version': + overrides.imageVersion = value; + break; + default: + violations.push({ label, reason: `key '${key}' is not a supported MicroVM override` }); + } + } + + return { overrides, violations }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts new file mode 100644 index 0000000000..16a38ec54c --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/environment.d.ts @@ -0,0 +1,18 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + MICROVM_EGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_EXECUTION_ROLE_ARN: string; + MICROVM_IMAGE_ARN: string; + MICROVM_IMAGE_VERSION: string | undefined; + MICROVM_INGRESS_NETWORK_CONNECTORS: string | undefined; + MICROVM_LOG_GROUP: string | undefined; + MICROVM_METADATA_SSM_PATH: string; + SSM_CONFIG_PATH: string; + SSM_PARAMETER_STORE_TAGS: string | undefined; + SSM_TOKEN_PATH: string; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts new file mode 100644 index 0000000000..2b7b85d74e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from 'vitest'; + +import type { RunnerMatcherConfig } from '../../../../contracts'; +import { microvmDynamicLabelProvider } from './dynamic-labels'; + +const imageArn = 'arn:aws:lambda:eu-west-1:123456789012:microvm-image:runner-large'; +const egressConnectorArn = 'arn:aws:lambda:eu-west-1:123456789012:network-connector:github-runner-private-egress'; + +describe('microvmDynamicLabelProvider', () => { + it('accepts supported MicroVM overrides', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { allowed: [egressConnectorArn] }, + 'image-arn': { allowed: [imageArn] }, + 'image-version': { allowed: ['3.0'] }, + }, + }; + const dynamicLabels = [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]; + + expect(getViolations(queue, dynamicLabels)).toEqual([]); + }); + + it('requires explicit allowlists for image code and network-boundary overrides', () => { + expect( + getViolations(microvmQueue(), [ + `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + `ghr-microvm-image-arn:${imageArn}`, + 'ghr-microvm-image-version:3.0', + ]), + ).toEqual([ + { + label: `ghr-microvm-egress-network-connectors:${egressConnectorArn}`, + reason: "key 'egress-network-connectors' requires an explicit allowed list", + }, + { + label: `ghr-microvm-image-arn:${imageArn}`, + reason: "key 'image-arn' requires an explicit allowed list", + }, + { + label: 'ghr-microvm-image-version:3.0', + reason: "key 'image-version' requires an explicit allowed list", + }, + ]); + }); + + it.each([ + ['ghr-microvm-memory:8192', "key 'memory' is not a supported MicroVM override"], + [ + 'ghr-microvm-maximum-duration-in-seconds:7200', + "key 'maximum-duration-in-seconds' is not a supported MicroVM override", + ], + ])('preserves the parser violation for %s', (label, reason) => { + expect(getViolations(microvmQueue(), [label])).toEqual([ + { + label, + reason, + }, + ]); + }); + + it('enforces the AWS dynamic-label policy', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { 'image-version': { allowed: ['2.*'] } }, + }; + + expect(getViolations(queue, ['ghr-microvm-image-version:3.0'])).toEqual([ + { + label: 'ghr-microvm-image-version:3.0', + reason: "value '3.0' not in allowed list", + }, + ]); + }); + + it('applies allowed patterns to the complete image ARN', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-arn': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:approved-large', + ]), + ).toEqual([]); + expect( + getViolations(queue, ['ghr-microvm-image-arn:arn:aws:lambda:eu-west-1:123456789012:microvm-image:unapproved']), + ).toHaveLength(1); + }); + + it('applies the policy to each egress connector label', () => { + const queue = microvmQueue(); + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'egress-network-connectors': { + allowed: ['arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-*'], + }, + }, + }; + + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:approved-private', + ]), + ).toEqual([]); + expect( + getViolations(queue, [ + 'ghr-microvm-egress-network-connectors:arn:aws:lambda:eu-west-1:123456789012:network-connector:unapproved', + ]), + ).toHaveLength(1); + }); +}); + +function getViolations(queue: RunnerMatcherConfig, labels: string[]) { + return microvmDynamicLabelProvider.getViolations({ + queue, + labels, + }); +} + +function microvmQueue(): RunnerMatcherConfig { + return { + id: 'microvm', + arn: 'arn:aws:sqs:eu-west-1:123456789012:microvm', + computeProvider: 'microvm', + matcherConfig: { + labelMatchers: [['self-hosted', 'linux', 'arm64', 'microvm']], + exactMatch: false, + enableDynamicLabels: true, + }, + }; +} diff --git a/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts new file mode 100644 index 0000000000..e7c5485617 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/src/webhook/dynamic-labels.ts @@ -0,0 +1,32 @@ +import type { DynamicLabelProvider } from '../../../../contracts'; +import { violationsAgainstAwsDynamicLabelsPolicy } from '../../../dynamic-labels-policy'; +import { MICROVM_DYNAMIC_LABEL_PREFIX, parseMicrovmDynamicLabels } from '../dynamic-labels'; + +const RESOURCE_BOUNDARY_KEYS = new Set(['egress-network-connectors', 'image-arn', 'image-version']); + +function resourceBoundaryViolations( + labels: string[], + policy: Parameters[1], +) { + return labels.flatMap((label) => { + if (!label.startsWith(MICROVM_DYNAMIC_LABEL_PREFIX)) return []; + + const key = label.slice(MICROVM_DYNAMIC_LABEL_PREFIX.length).split(':', 1)[0]; + if (!RESOURCE_BOUNDARY_KEYS.has(key) || policy?.blocked_keys?.includes(key)) return []; + + const allowed = policy?.restricted_keys?.[key]?.allowed; + return allowed && allowed.length > 0 ? [] : [{ label, reason: `key '${key}' requires an explicit allowed list` }]; + }); +} + +export const microvmDynamicLabelProvider: DynamicLabelProvider = { + getViolations: ({ queue, labels }) => [ + ...parseMicrovmDynamicLabels(labels).violations, + ...resourceBoundaryViolations(labels, queue.matcherConfig.awsDynamicLabelsPolicy), + ...violationsAgainstAwsDynamicLabelsPolicy( + labels, + queue.matcherConfig.awsDynamicLabelsPolicy, + MICROVM_DYNAMIC_LABEL_PREFIX, + ), + ], +}; diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts new file mode 100644 index 0000000000..ad3baed78b --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.test.ts @@ -0,0 +1,34 @@ +import { defineWebhookProviderContractTests } from '../../test/webhook-provider-contract'; +import { provider } from './webhook'; + +defineWebhookProviderContractTests({ + provider, + acceptedDynamicLabels: ['ghr-microvm-image-version:3.0'], + configureQueue: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['3.0'] }, + }, + }; + }, + rejectingPolicies: [ + { + name: 'blocked keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + blocked_keys: ['image-version'], + }; + }, + }, + { + name: 'restricted keys', + apply: (queue) => { + queue.matcherConfig.awsDynamicLabelsPolicy = { + restricted_keys: { + 'image-version': { allowed: ['2.*'] }, + }, + }; + }, + }, + ], +}); diff --git a/lambdas/libs/compute-providers/aws/microvm/webhook.ts b/lambdas/libs/compute-providers/aws/microvm/webhook.ts new file mode 100644 index 0000000000..48d603e476 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/microvm/webhook.ts @@ -0,0 +1,16 @@ +import type { ComputeProviderPlugin } from '../../core'; + +import type { WebhookProviderCapabilities, WebhookProviderModule } from '../../contracts'; +import { microvmDynamicLabelProvider } from './src/webhook/dynamic-labels'; + +export function createMicrovmWebhookPlugin(): ComputeProviderPlugin { + return { + type: 'microvm', + capabilities: { dynamicLabels: microvmDynamicLabelProvider }, + }; +} + +export const provider = { + type: 'microvm', + createPlugin: createMicrovmWebhookPlugin, +} satisfies WebhookProviderModule<'microvm'>; diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index c5560942fa..c1037ad268 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -21,9 +21,6 @@ export interface CreateGitHubRunnerConfig { runnerOwner: string; runnerType: RunnerType; disableAutoUpdate: boolean; - ssmTokenPath: string; - ssmConfigPath: string; - ssmParameterStoreTags: { Key: string; Value: string }[]; } export interface GitHubRunnerMetadata { @@ -32,7 +29,7 @@ export interface GitHubRunnerMetadata { } export interface StartRunnerConfigOptions { - getSsmParameterTags?: (runnerId: string) => { Key: string; Value: string }[]; + getRunnerConfigMetadata?: (runnerId: string) => { key: string; value: string }[]; onJitConfigCreated?: (runnerId: string, metadata: GitHubRunnerMetadata) => Promise; } diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index c1806818cc..f9cb1b973a 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -11,7 +11,11 @@ "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", "./aws/ec2/runners": "./aws/ec2/src/runners.ts", - "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts" + "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts", + "./aws/ec2/control-plane/runners": "./aws/ec2/src/control-plane/runners.ts", + "./aws/ec2/control-plane/runner-config": "./aws/ec2/src/control-plane/runner-config.ts", + "./aws/microvm/webhook": "./aws/microvm/webhook.ts", + "./aws/microvm/control-plane": "./aws/microvm/control-plane.ts" }, "type": "module", "license": "MIT", @@ -27,6 +31,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-lambda-microvms": "^3.1074.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/provider-types.ts b/lambdas/libs/compute-providers/provider-types.ts index 64d7be8e5f..087f61de71 100644 --- a/lambdas/libs/compute-providers/provider-types.ts +++ b/lambdas/libs/compute-providers/provider-types.ts @@ -1,4 +1,4 @@ -export const computeProviderTypes = ['ec2'] as const; +export const computeProviderTypes = ['ec2', 'microvm'] as const; export type ComputeProviderType = (typeof computeProviderTypes)[number]; diff --git a/lambdas/libs/compute-providers/providers.config.control-plane.ts b/lambdas/libs/compute-providers/providers.config.control-plane.ts index 55ebaca95e..45a584bc06 100644 --- a/lambdas/libs/compute-providers/providers.config.control-plane.ts +++ b/lambdas/libs/compute-providers/providers.config.control-plane.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/control-plane'; +import { provider as microvm } from './aws/microvm/control-plane'; import type { ControlPlaneProviderModule } from './contracts'; /** Provider plugins included in the control-plane bundle. */ -export const enabledControlPlaneProviders = [ec2] as const satisfies readonly ControlPlaneProviderModule[]; +export const enabledControlPlaneProviders = [ec2, microvm] as const satisfies readonly ControlPlaneProviderModule[]; diff --git a/lambdas/libs/compute-providers/providers.config.webhook.ts b/lambdas/libs/compute-providers/providers.config.webhook.ts index 19c92734da..a4aec0853a 100644 --- a/lambdas/libs/compute-providers/providers.config.webhook.ts +++ b/lambdas/libs/compute-providers/providers.config.webhook.ts @@ -1,5 +1,6 @@ import { provider as ec2 } from './aws/ec2/webhook'; +import { provider as microvm } from './aws/microvm/webhook'; import type { WebhookProviderModule } from './contracts'; /** Provider plugins included in the webhook bundle. */ -export const enabledWebhookProviders = [ec2] as const satisfies readonly WebhookProviderModule[]; +export const enabledWebhookProviders = [ec2, microvm] as const satisfies readonly WebhookProviderModule[]; diff --git a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts index dd4e3097b0..a6d01b7e6e 100644 --- a/lambdas/libs/compute-providers/test/webhook-provider-contract.ts +++ b/lambdas/libs/compute-providers/test/webhook-provider-contract.ts @@ -13,17 +13,25 @@ interface RejectingPolicyCase { interface WebhookProviderContractOptions { provider: WebhookProviderModule; acceptedDynamicLabels: readonly [string, ...string[]]; + configureQueue?(queue: RunnerMatcherConfig): void; rejectingPolicies: readonly [RejectingPolicyCase, ...RejectingPolicyCase[]]; } export function defineWebhookProviderContractTests({ provider, acceptedDynamicLabels, + configureQueue, rejectingPolicies, }: WebhookProviderContractOptions): void { const nonGhrLabels = ['self-hosted', 'linux']; const dynamicLabels = [...acceptedDynamicLabels]; + function configuredRunnerQueue(id: string, computeProvider?: ComputeProviderType): RunnerMatcherConfig { + const queue = runnerQueue(id, computeProvider); + configureQueue?.(queue); + return queue; + } + function expectProviderSelected(queue: RunnerMatcherConfig) { expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toEqual({ queue, @@ -33,11 +41,11 @@ export function defineWebhookProviderContractTests { it('selects an explicitly configured provider through the production registry', () => { - expectProviderSelected(runnerQueue(`${provider.type}-configured`, provider.type)); + expectProviderSelected(configuredRunnerQueue(`${provider.type}-configured`, provider.type)); }); it('skips the provider when dynamic labels are disabled', () => { - const queue = runnerQueue(`${provider.type}-disabled`, provider.type); + const queue = configuredRunnerQueue(`${provider.type}-disabled`, provider.type); queue.matcherConfig.enableDynamicLabels = false; expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); @@ -45,7 +53,7 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-policy-rejected`, provider.type); + const queue = configuredRunnerQueue(`${provider.type}-policy-rejected`, provider.type); policy.apply(queue); expect(selectDynamicLabelQueue([queue], nonGhrLabels, dynamicLabels)).toBeUndefined(); @@ -53,7 +61,7 @@ export function defineWebhookProviderContractTests { - const queue = runnerQueue(`${provider.type}-normalized`); + const queue = configuredRunnerQueue(`${provider.type}-normalized`); (queue as unknown as { computeProvider: string }).computeProvider = ` ${provider.type.toUpperCase()} `; expectProviderSelected(queue); @@ -61,7 +69,7 @@ export function defineWebhookProviderContractTests { - expectProviderSelected(runnerQueue(`${provider.type}-default`)); + expectProviderSelected(configuredRunnerQueue(`${provider.type}-default`)); }); } }); diff --git a/lambdas/libs/storage-providers/aws/ssm/environment.d.ts b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts new file mode 100644 index 0000000000..ba0e7afda0 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/environment.d.ts @@ -0,0 +1,15 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + PARAMETER_GITHUB_APP_ID_NAME?: string; + PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME?: string; + PARAMETER_GITHUB_APP_KEY_BASE64_NAME?: string; + SSM_CONFIG_PATH?: string; + SSM_CLEANUP_CONFIG?: string; + SSM_PARAMETER_STORE_TAGS?: string; + SSM_TOKEN_PATH?: string; + } + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts new file mode 100644 index 0000000000..4ca2e93914 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.test.ts @@ -0,0 +1,154 @@ +import { getParameters } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmGitHubAppCredentialsStore } from './github-app-credentials-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameters: vi.fn(), +})); + +const getParametersMock = vi.mocked(getParameters); +const cleanEnv = process.env; +const primaryIdParameter = '/actions-runner/test/github_app_id'; +const primaryKeyParameter = '/actions-runner/test/github_app_key_base64'; + +describe('aws_ssm GitHub App credentials store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + process.env.PARAMETER_GITHUB_APP_ID_NAME = primaryIdParameter; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = primaryKeyParameter; + delete process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME; + }); + + it('batch reads and maps the primary GitHub App credential', async () => { + const privateKey = 'fake-private-key'; + getParametersMock.mockResolvedValue( + new Map([ + [primaryIdParameter, '123'], + [primaryKeyParameter, Buffer.from(privateKey).toString('base64')], + ]), + ); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).resolves.toEqual([{ appId: 123, privateKey, installationId: undefined }]); + expect(getParametersMock).toHaveBeenCalledOnce(); + expect(getParametersMock).toHaveBeenCalledWith([primaryIdParameter, primaryKeyParameter]); + }); + + it('preserves multi-app order and optional installation-id slots', async () => { + const additionalIdParameter = '/actions-runner/test/additional_github_app_0_id'; + const additionalKeyParameter = '/actions-runner/test/additional_github_app_0_key_base64'; + const additionalInstallationIdParameter = '/actions-runner/test/additional_github_app_0_installation_id'; + process.env.PARAMETER_GITHUB_APP_ID_NAME = `${primaryIdParameter}:${additionalIdParameter}`; + process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME = `${primaryKeyParameter}:${additionalKeyParameter}`; + process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = `:${additionalInstallationIdParameter}`; + getParametersMock.mockResolvedValue( + new Map([ + [primaryIdParameter, '123'], + [primaryKeyParameter, Buffer.from('primary-key').toString('base64')], + [additionalIdParameter, '456'], + [additionalKeyParameter, Buffer.from('additional-key').toString('base64')], + [additionalInstallationIdParameter, '789'], + ]), + ); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).resolves.toEqual([ + { appId: 123, privateKey: 'primary-key', installationId: undefined }, + { appId: 456, privateKey: 'additional-key', installationId: 789 }, + ]); + expect(getParametersMock).toHaveBeenCalledWith([ + primaryIdParameter, + additionalIdParameter, + primaryKeyParameter, + additionalKeyParameter, + additionalInstallationIdParameter, + ]); + }); + + it('decodes literal newline escapes in a base64 private key', async () => { + getParametersMock.mockResolvedValue( + new Map([ + [primaryIdParameter, '123'], + [primaryKeyParameter, Buffer.from('first-line\\nsecond-line').toString('base64')], + ]), + ); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).resolves.toEqual([ + { appId: 123, privateKey: 'first-line\nsecond-line', installationId: undefined }, + ]); + }); + + it('preserves parseInt behavior for stored numeric values', async () => { + const installationIdParameter = '/actions-runner/test/github_app_installation_id'; + process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME = installationIdParameter; + getParametersMock.mockResolvedValue( + new Map([ + [primaryIdParameter, '123app'], + [primaryKeyParameter, Buffer.from('fake-private-key').toString('base64')], + [installationIdParameter, '789installation'], + ]), + ); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).resolves.toEqual([{ appId: 123, privateKey: 'fake-private-key', installationId: 789 }]); + }); + + it.each([ + ['PARAMETER_GITHUB_APP_ID_NAME', undefined], + ['PARAMETER_GITHUB_APP_ID_NAME', ''], + ['PARAMETER_GITHUB_APP_KEY_BASE64_NAME', undefined], + ['PARAMETER_GITHUB_APP_KEY_BASE64_NAME', ''], + ] as const)('rejects missing environment value %s=%j before reading', async (name, value) => { + setEnvironmentValue(name, value); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toThrow(`Environment variable ${name} is not set`); + expect(getParametersMock).not.toHaveBeenCalled(); + }); + + it('rejects mismatched GitHub App id and key parameter counts before reading', async () => { + process.env.PARAMETER_GITHUB_APP_ID_NAME = `${primaryIdParameter}:/additional/id`; + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toThrow('GitHub App parameter count mismatch: 2 IDs vs 1 keys'); + expect(getParametersMock).not.toHaveBeenCalled(); + }); + + it('rejects a missing GitHub App id parameter', async () => { + getParametersMock.mockResolvedValue( + new Map([[primaryKeyParameter, Buffer.from('fake-private-key').toString('base64')]]), + ); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toThrow(`Parameter ${primaryIdParameter} not found`); + }); + + it('rejects a missing GitHub App private-key parameter', async () => { + getParametersMock.mockResolvedValue(new Map([[primaryIdParameter, '123']])); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toThrow(`Parameter ${primaryKeyParameter} not found`); + }); + + it('propagates parameter-store read errors', async () => { + const error = new Error('access denied'); + getParametersMock.mockRejectedValue(error); + const store = createAwsSsmGitHubAppCredentialsStore(); + + await expect(store.get()).rejects.toBe(error); + }); +}); + +function setEnvironmentValue( + name: 'PARAMETER_GITHUB_APP_ID_NAME' | 'PARAMETER_GITHUB_APP_KEY_BASE64_NAME', + value: string | undefined, +): void { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts new file mode 100644 index 0000000000..5e5ca2e501 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/github-app-credentials-store.ts @@ -0,0 +1,61 @@ +import { getParameters } from '@aws-github-runner/aws-ssm-util'; + +import type { GitHubAppCredential, GitHubAppCredentialsStore } from '../../core'; +import type {} from './environment'; + +export function createAwsSsmGitHubAppCredentialsStore(): GitHubAppCredentialsStore { + return new AwsSsmGitHubAppCredentialsStore(); +} + +class AwsSsmGitHubAppCredentialsStore implements GitHubAppCredentialsStore { + async get(): Promise { + if (!process.env.PARAMETER_GITHUB_APP_ID_NAME) { + throw new Error('Environment variable PARAMETER_GITHUB_APP_ID_NAME is not set'); + } + if (!process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME) { + throw new Error('Environment variable PARAMETER_GITHUB_APP_KEY_BASE64_NAME is not set'); + } + + const idParameters = process.env.PARAMETER_GITHUB_APP_ID_NAME.split(':').filter(Boolean); + const keyParameters = process.env.PARAMETER_GITHUB_APP_KEY_BASE64_NAME.split(':').filter(Boolean); + const installationIdParameters = (process.env.PARAMETER_GITHUB_APP_INSTALLATION_ID_NAME || '').split(':'); + if (idParameters.length !== keyParameters.length) { + throw new Error( + `GitHub App parameter count mismatch: ${idParameters.length} IDs vs ${keyParameters.length} keys`, + ); + } + + const parameterNames = [ + ...idParameters, + ...keyParameters, + ...installationIdParameters.filter((parameter) => parameter.length > 0), + ]; + const parameters = await getParameters(parameterNames); + + const credentials: GitHubAppCredential[] = []; + for (let index = 0; index < idParameters.length; index++) { + const appIdValue = parameters.get(idParameters[index]); + if (!appIdValue) { + throw new Error(`Parameter ${idParameters[index]} not found`); + } + + const privateKeyBase64 = parameters.get(keyParameters[index]); + if (!privateKeyBase64) { + throw new Error(`Parameter ${keyParameters[index]} not found`); + } + + const installationIdParameter = installationIdParameters[index]; + const installationIdValue = installationIdParameter ? parameters.get(installationIdParameter) : undefined; + + credentials.push({ + appId: parseInt(appIdValue, 10), + // Match the GitHub Terraform provider's handling of keys stored as a + // single-line base64 value containing literal newline escapes. + privateKey: Buffer.from(privateKeyBase64, 'base64').toString().replace(/\\n/g, '\n'), + installationId: installationIdValue ? parseInt(installationIdValue, 10) : undefined, + }); + } + + return credentials; + } +} diff --git a/lambdas/functions/control-plane/src/local-ssm-housekeeper.ts b/lambdas/libs/storage-providers/aws/ssm/local-runner-config-housekeeper.ts similarity index 50% rename from lambdas/functions/control-plane/src/local-ssm-housekeeper.ts rename to lambdas/libs/storage-providers/aws/ssm/local-runner-config-housekeeper.ts index ec635b13ad..79518a8157 100644 --- a/lambdas/functions/control-plane/src/local-ssm-housekeeper.ts +++ b/lambdas/libs/storage-providers/aws/ssm/local-runner-config-housekeeper.ts @@ -1,11 +1,14 @@ -import { cleanSSMTokens } from './scale-runners/ssm-housekeeper'; +import { createAwsSsmRunnerConfigStore } from './runner-config-store'; export function run(): void { - cleanSSMTokens({ + process.env.SSM_CLEANUP_CONFIG = JSON.stringify({ dryRun: true, minimumDaysOld: 3, tokenPath: '/ghr/my-env/runners/tokens', - }) + }); + + createAwsSsmRunnerConfigStore() + .houseKeeper() .then() .catch((e) => { console.log(e); diff --git a/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts b/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts new file mode 100644 index 0000000000..d35150e10a --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/parameter-store-tags.ts @@ -0,0 +1,42 @@ +interface SsmParameterStoreTag { + Key: string; + Value: string; +} + +export function loadSsmParameterStoreTagsFromEnvironment(): SsmParameterStoreTag[] { + return process.env.SSM_PARAMETER_STORE_TAGS && process.env.SSM_PARAMETER_STORE_TAGS.trim() !== '' + ? validateSsmParameterStoreTags(process.env.SSM_PARAMETER_STORE_TAGS) + : []; +} + +function validateSsmParameterStoreTags(tagsJson: string): SsmParameterStoreTag[] { + try { + const tags: unknown = JSON.parse(tagsJson); + + if (!Array.isArray(tags)) { + throw new Error('Tags must be an array'); + } + + if (tags.length === 0) { + return []; + } + + tags.forEach((tag: unknown, index: number) => { + if (typeof tag !== 'object' || tag === null) { + throw new Error(`Tag at index ${index} must be an object`); + } + + const candidate = tag as Record; + if (!candidate.Key || typeof candidate.Key !== 'string' || candidate.Key.trim() === '') { + throw new Error(`Tag at index ${index} has missing or invalid 'Key' property`); + } + if (!Object.prototype.hasOwnProperty.call(candidate, 'Value') || typeof candidate.Value !== 'string') { + throw new Error(`Tag at index ${index} has missing or invalid 'Value' property`); + } + }); + + return tags as SsmParameterStoreTag[]; + } catch (error) { + throw new Error(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${(error as Error).message}`); + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.test.ts new file mode 100644 index 0000000000..2a5d2217ea --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.test.ts @@ -0,0 +1,220 @@ +import { DeleteParameterCommand, GetParameterCommand, type SSMClient } from '@aws-sdk/client-ssm'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { + AwsSdkSsmRunnerConfigApi, + createAwsSsmRunnerConfigConsumer, + type AwsSsmRunnerConfigApi, +} from './runner-config-consumer'; + +function namedError(name: string, message = 'provider detail'): Error { + const error = new Error(message); + error.name = name; + return error; +} + +describe('AWS SDK SSM runner config API', () => { + it('decrypts the parameter and deletes it with the caller abort signal', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameter: { Value: 'encoded-jit' } }) + .mockResolvedValueOnce({}); + const api = new AwsSdkSsmRunnerConfigApi({ send } as unknown as SSMClient); + const signal = new AbortController().signal; + + await expect(api.getParameter('/runner/tokens/runner-123', signal)).resolves.toBe('encoded-jit'); + await expect(api.deleteParameter('/runner/tokens/runner-123', signal)).resolves.toBeUndefined(); + + expect(send.mock.calls[0][0]).toBeInstanceOf(GetParameterCommand); + expect(send.mock.calls[0][0].input).toEqual({ + Name: '/runner/tokens/runner-123', + WithDecryption: true, + }); + expect(send.mock.calls[0][1]).toEqual({ abortSignal: signal }); + expect(send.mock.calls[1][0]).toBeInstanceOf(DeleteParameterCommand); + expect(send.mock.calls[1][0].input).toEqual({ Name: '/runner/tokens/runner-123' }); + expect(send.mock.calls[1][1]).toEqual({ abortSignal: signal }); + }); +}); + +describe('SSM runner config consumer', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('polls a missing parameter, reads it, and deletes it before returning', async () => { + const getParameter = vi + .fn() + .mockRejectedValueOnce(namedError('ParameterNotFound')) + .mockResolvedValueOnce('encoded-jit'); + const deleteParameter = vi.fn().mockResolvedValue(undefined); + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { + api: { getParameter, deleteParameter }, + callTimeoutMs: 100, + configTimeoutMs: 500, + pollIntervalMs: 1, + }, + ); + + await expect( + consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).resolves.toBe('encoded-jit'); + expect(getParameter).toHaveBeenCalledTimes(2); + expect(deleteParameter).toHaveBeenCalledOnce(); + expect(deleteParameter).toHaveBeenCalledWith('/runner/tokens/runner-123', expect.any(AbortSignal)); + }); + + it('retries a transient delete failure without returning the value early', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue('encoded-jit'), + deleteParameter: vi + .fn() + .mockRejectedValueOnce(namedError('ThrottlingException')) + .mockResolvedValueOnce(undefined), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 2_000, deleteAttempts: 2, pollIntervalMs: 1 }, + ); + + const pending = consumer.consume('runner-123', { + deadlineMs: Date.now() + 3_000, + signal: new AbortController().signal, + }); + await vi.runAllTimersAsync(); + + await expect(pending).resolves.toBe('encoded-jit'); + expect(api.deleteParameter).toHaveBeenCalledTimes(2); + }); + + it('fails closed when another reader deletes the SSM parameter first', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue('encoded-jit'), + deleteParameter: vi.fn().mockRejectedValue(namedError('ParameterNotFound')), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, deleteAttempts: 3, pollIntervalMs: 1 }, + ); + + await expect( + consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('runner configuration could not be deleted from SSM'); + expect(api.deleteParameter).toHaveBeenCalledOnce(); + }); + + it('sanitizes non-retryable provider failures', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockRejectedValue(namedError('AccessDeniedException', 'encoded-jit-secret')), + deleteParameter: vi.fn(), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, pollIntervalMs: 1 }, + ); + + const pending = consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }); + await expect(pending).rejects.toThrow('failed to read runner configuration from SSM'); + await expect(pending).rejects.not.toThrow('encoded-jit-secret'); + expect(api.deleteParameter).not.toHaveBeenCalled(); + }); + + it('rejects an empty SSM parameter value without attempting deletion', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue(''), + deleteParameter: vi.fn(), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, pollIntervalMs: 1 }, + ); + + await expect( + consumer.consume('runner-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('failed to read runner configuration from SSM'); + expect(api.deleteParameter).not.toHaveBeenCalled(); + }); + + it('validates the full parameter name before calling SSM', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn(), + deleteParameter: vi.fn(), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: `/${'x'.repeat(890)}` }, + { api, callTimeoutMs: 100, configTimeoutMs: 100, pollIntervalMs: 1 }, + ); + + await expect( + consumer.consume('runner-1234567890', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toThrow('aws_ssm runner configuration key is invalid'); + expect(api.getParameter).not.toHaveBeenCalled(); + }); + + it('stops a provider call immediately when the caller aborts', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockReturnValue(new Promise(() => undefined)), + deleteParameter: vi.fn(), + }; + const controller = new AbortController(); + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 10_000, configTimeoutMs: 10_000, pollIntervalMs: 1 }, + ); + const pending = consumer.consume('runner-123', { + deadlineMs: Date.now() + 10_000, + signal: controller.signal, + }); + + controller.abort(); + + await expect(pending).rejects.toThrow('runner configuration consumption was cancelled'); + }); + + it('reserves a bounded delete attempt when the value appears near the polling deadline', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + const startedAt = Date.now(); + let deleteStartedAt: number | undefined; + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValueOnce(undefined).mockResolvedValueOnce('encoded-jit'), + deleteParameter: vi.fn().mockImplementation(async () => { + deleteStartedAt = Date.now(); + }), + }; + const consumer = createAwsSsmRunnerConfigConsumer( + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', SSM_TOKEN_PATH: '/runner/tokens' }, + { api, callTimeoutMs: 100, configTimeoutMs: 1_000, pollIntervalMs: 99 }, + ); + + const pending = consumer.consume('runner-123', { + deadlineMs: startedAt + 200, + signal: new AbortController().signal, + }); + await vi.runAllTimersAsync(); + + await expect(pending).resolves.toBe('encoded-jit'); + expect(api.getParameter).toHaveBeenCalledTimes(2); + expect(deleteStartedAt).toBe(startedAt + 99); + expect(deleteStartedAt).toBeLessThanOrEqual(startedAt + 100); + }); +}); diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.ts new file mode 100644 index 0000000000..7736f8e1ee --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-consumer.ts @@ -0,0 +1,167 @@ +import { DeleteParameterCommand, GetParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; + +import type { + AwsSsmRunnerConfigStorageEnvironment, + RunnerConfigConsumer, + RunnerConfigConsumeOptions, +} from '../../core'; +import { + composeSsmParameterName, + delay, + errorName, + isRetryableProviderError, + positiveIntegerOption, + resolvePollingOptions, + throwIfCancelled, + validateConsumeOptions, + withCallDeadline, + type RunnerConfigPollingOptions, +} from '../../runner-config-consumer-common'; + +const DEFAULT_DELETE_ATTEMPTS = 3; + +export interface AwsSsmRunnerConfigApi { + getParameter(name: string, signal: AbortSignal): Promise; + deleteParameter(name: string, signal: AbortSignal): Promise; +} + +export class AwsSdkSsmRunnerConfigApi implements AwsSsmRunnerConfigApi { + private client?: SSMClient; + + public constructor(client?: SSMClient) { + this.client = client; + } + + private getClient(): SSMClient { + // Lifecycle hooks can be snapshotted before their first request. Constructing + // the untraced client here avoids persisting connection state in that snapshot. + this.client ??= new SSMClient({ maxAttempts: 1 }); + return this.client; + } + + public async getParameter(name: string, signal: AbortSignal): Promise { + const response = await this.getClient().send(new GetParameterCommand({ Name: name, WithDecryption: true }), { + abortSignal: signal, + }); + return response.Parameter?.Value; + } + + public async deleteParameter(name: string, signal: AbortSignal): Promise { + await this.getClient().send(new DeleteParameterCommand({ Name: name }), { abortSignal: signal }); + } +} + +export interface AwsSsmRunnerConfigConsumerOptions extends RunnerConfigPollingOptions { + api?: AwsSsmRunnerConfigApi; + deleteAttempts?: number; +} + +export function createAwsSsmRunnerConfigConsumer( + environment: AwsSsmRunnerConfigStorageEnvironment, + options: AwsSsmRunnerConfigConsumerOptions = {}, +): RunnerConfigConsumer { + return new AwsSsmRunnerConfigConsumer( + environment.SSM_TOKEN_PATH, + options.api ?? new AwsSdkSsmRunnerConfigApi(), + options, + ); +} + +class AwsSsmRunnerConfigConsumer implements RunnerConfigConsumer { + private readonly callTimeoutMs: number; + private readonly configTimeoutMs: number; + private readonly deleteAttempts: number; + private readonly pollIntervalMs: number; + + public constructor( + private readonly tokenPath: string, + private readonly api: AwsSsmRunnerConfigApi, + options: AwsSsmRunnerConfigConsumerOptions, + ) { + const polling = resolvePollingOptions(options); + this.callTimeoutMs = polling.callTimeoutMs; + this.configTimeoutMs = polling.configTimeoutMs; + this.pollIntervalMs = polling.pollIntervalMs; + this.deleteAttempts = positiveIntegerOption('deleteAttempts', options.deleteAttempts, DEFAULT_DELETE_ATTEMPTS); + } + + public async consume(runnerId: string, options: RunnerConfigConsumeOptions): Promise { + validateConsumeOptions(options); + const parameterName = composeSsmParameterName(this.tokenPath, runnerId); + const startedAt = Date.now(); + const remainingMs = Math.max(0, options.deadlineMs - startedAt); + // Preserve enough of short hook budgets for at least one bounded delete + // attempt without reviving the old fixed reserve that could consume the + // entire polling window. + const deleteReserveMs = Math.min(this.callTimeoutMs, Math.max(1, Math.floor(remainingMs / 2))); + const pollDeadline = Math.min(startedAt + this.configTimeoutMs, options.deadlineMs - deleteReserveMs); + let runnerConfig: string | undefined; + + while (Date.now() < pollDeadline) { + throwIfCancelled(options.signal); + try { + runnerConfig = await this.read(parameterName, pollDeadline, options.signal); + if (runnerConfig !== undefined) { + if (runnerConfig.length === 0) { + throw new Error('runner configuration record has an invalid value'); + } + break; + } + } catch (error) { + if (options.signal.aborted) { + throw new Error('runner configuration consumption was cancelled'); + } + if (!isSsmNotFound(error) && !isRetryableProviderError(error)) { + throw new Error('failed to read runner configuration from SSM'); + } + } + + const remaining = pollDeadline - Date.now(); + if (remaining > 0) { + await delay(Math.min(this.pollIntervalMs, remaining), options.signal); + } + } + + if (runnerConfig === undefined) { + throw new Error('runner configuration did not become available before the deadline'); + } + + await this.delete(parameterName, options); + return runnerConfig; + } + + private async read(name: string, deadlineMs: number, signal: AbortSignal): Promise { + return withCallDeadline(signal, deadlineMs, this.callTimeoutMs, (callSignal) => + this.api.getParameter(name, callSignal), + ); + } + + private async delete(name: string, options: RunnerConfigConsumeOptions): Promise { + for (let attempt = 1; attempt <= this.deleteAttempts; attempt += 1) { + try { + await withCallDeadline(options.signal, options.deadlineMs, this.callTimeoutMs, (callSignal) => + this.api.deleteParameter(name, callSignal), + ); + return; + } catch (error) { + if (options.signal.aborted) { + throw new Error('runner configuration consumption was cancelled'); + } + if (!isRetryableProviderError(error) || attempt === this.deleteAttempts) { + break; + } + + const remaining = options.deadlineMs - Date.now(); + if (remaining <= 0) { + break; + } + await delay(Math.min(2 ** (attempt - 1) * 1_000, 5_000, remaining), options.signal); + } + } + throw new Error('runner configuration could not be deleted from SSM'); + } +} + +function isSsmNotFound(error: unknown): boolean { + return errorName(error) === 'ParameterNotFound'; +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.test.ts new file mode 100644 index 0000000000..9c837c7fb7 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.test.ts @@ -0,0 +1,97 @@ +import { DeleteParameterCommand, GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { createAwsSsmRunnerConfigStore } from './runner-config-store'; + +const mockSSMClient = mockClient(SSMClient); +const cleanEnv = process.env; +const minimumDaysOld = 1; +const now = new Date(); +const oldDate = new Date(); +oldDate.setDate(oldDate.getDate() - minimumDaysOld - 1); +const tokenPath = '/path/to/tokens/'; + +describe('aws_ssm runner config housekeeper', () => { + beforeEach(() => { + mockSSMClient.reset(); + process.env = { ...cleanEnv }; + delete process.env.SSM_TOKEN_PATH; + process.env.AWS_REGION = 'eu-east-1'; + setCleanupOptions({ dryRun: false, minimumDaysOld, tokenPath }); + + mockSSMClient.on(GetParametersByPathCommand).resolves({ + Parameters: undefined, + }); + mockSSMClient.on(GetParametersByPathCommand, { Path: tokenPath }).resolves({ + Parameters: [ + { + Name: `${tokenPath}i-old-01`, + LastModifiedDate: oldDate, + }, + ], + NextToken: 'next', + }); + mockSSMClient.on(GetParametersByPathCommand, { Path: tokenPath, NextToken: 'next' }).resolves({ + Parameters: [ + { + Name: `${tokenPath}i-new-01`, + LastModifiedDate: now, + }, + ], + NextToken: undefined, + }); + }); + + it('constructs without writer configuration and deletes expired records across pages', async () => { + const store = createAwsSsmRunnerConfigStore(); + + await store.houseKeeper(); + + expect(mockSSMClient).toHaveReceivedCommandWith(GetParametersByPathCommand, { Path: tokenPath }); + expect(mockSSMClient).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: `${tokenPath}i-old-01` }); + expect(mockSSMClient).not.toHaveReceivedCommandWith(DeleteParameterCommand, { Name: `${tokenPath}i-new-01` }); + }); + + it('does not delete records during a dry run', async () => { + setCleanupOptions({ dryRun: true, minimumDaysOld, tokenPath }); + const store = createAwsSsmRunnerConfigStore(); + + await store.houseKeeper(); + + expect(mockSSMClient).toHaveReceivedCommandWith(GetParametersByPathCommand, { Path: tokenPath }); + expect(mockSSMClient).not.toHaveReceivedCommand(DeleteParameterCommand); + }); + + it('does not delete when no records are found', async () => { + setCleanupOptions({ dryRun: false, minimumDaysOld, tokenPath: 'does-not-exist' }); + const store = createAwsSsmRunnerConfigStore(); + + await expect(store.houseKeeper()).resolves.not.toThrow(); + + expect(mockSSMClient).not.toHaveReceivedCommand(DeleteParameterCommand); + }); + + it('continues when deleting an expired record fails', async () => { + mockSSMClient.on(DeleteParameterCommand).rejects(new Error('ParameterNotFound')); + const store = createAwsSsmRunnerConfigStore(); + + await expect(store.houseKeeper()).resolves.not.toThrow(); + }); + + it.each([ + { dryRun: false, minimumDaysOld: undefined as unknown as number, tokenPath }, + { dryRun: false, minimumDaysOld: 0, tokenPath }, + { dryRun: false, minimumDaysOld, tokenPath: undefined as unknown as string }, + ])('rejects invalid cleanup options %#', async (options) => { + setCleanupOptions(options); + const store = createAwsSsmRunnerConfigStore(); + + await expect(store.houseKeeper()).rejects.toBeInstanceOf(Error); + }); +}); + +function setCleanupOptions(options: { dryRun: boolean; minimumDaysOld: number; tokenPath: string }): void { + process.env.SSM_CLEANUP_CONFIG = JSON.stringify(options); +} diff --git a/lambdas/functions/control-plane/src/scale-runners/ssm-housekeeper.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.ts similarity index 91% rename from lambdas/functions/control-plane/src/scale-runners/ssm-housekeeper.ts rename to lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.ts index 857b974a9d..30bc1d20ca 100644 --- a/lambdas/functions/control-plane/src/scale-runners/ssm-housekeeper.ts +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-housekeeper.ts @@ -1,6 +1,5 @@ import { DeleteParameterCommand, GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; -import { logger } from '@aws-github-runner/aws-powertools-util'; -import { getTracedAWSV3Client } from '@aws-github-runner/aws-powertools-util'; +import { getTracedAWSV3Client, logger } from '@aws-github-runner/aws-powertools-util'; export interface SSMCleanupOptions { dryRun: boolean; @@ -36,7 +35,6 @@ export async function cleanSSMTokens(options: SSMCleanupOptions): Promise parameters.NextToken = nextParameters.NextToken; } logger.info(`Found #${parameters.Parameters?.length} parameters in path ${options.tokenPath}`); - logger.debug('Found parameters', { parameters }); // minimumDate = today - minimumDaysOld const minimumDate = new Date(); @@ -47,7 +45,7 @@ export async function cleanSSMTokens(options: SSMCleanupOptions): Promise logger.info(`Deleting parameter ${parameter.Name} with last modified date ${parameter.LastModifiedDate}`); try { if (!options.dryRun) { - // sleep 50ms to avoid rait limit + // sleep 50ms to avoid rate limit await new Promise((resolve) => setTimeout(resolve, 50)); await client.send(new DeleteParameterCommand({ Name: parameter.Name })); } diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts new file mode 100644 index 0000000000..04ecdda05d --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.test.ts @@ -0,0 +1,95 @@ +import { putParameter } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerConfigStore } from './runner-config-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + putParameter: vi.fn(), +})); + +const putParameterMock = vi.mocked(putParameter); +const cleanEnv = process.env; + +describe('aws_ssm runner config store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.SSM_CLEANUP_CONFIG; + delete process.env.SSM_PARAMETER_STORE_TAGS; + process.env.SSM_TOKEN_PATH = '/runner/tokens'; + }); + + it('maps metadata to tags before configured SSM tags', async () => { + process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify([ + { Key: 'Environment', Value: 'test' }, + { Key: 'Team', Value: 'actions' }, + ]); + const store = createAwsSsmRunnerConfigStore(); + + await store.create( + { runnerId: 'i-123', value: 'encoded-jit-config' }, + { metadata: [{ key: 'InstanceId', value: 'i-123' }] }, + ); + + expect(store.maxWritesPerSecond).toBe(40); + expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/i-123', 'encoded-jit-config', true, { + tags: [ + { Key: 'InstanceId', Value: 'i-123' }, + { Key: 'Environment', Value: 'test' }, + { Key: 'Team', Value: 'actions' }, + ], + }); + }); + + it('uses an empty tag list when no tags are configured', async () => { + const store = createAwsSsmRunnerConfigStore(); + + await store.create({ runnerId: 'runner-1', value: 'registration-config' }); + + expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/runner-1', 'registration-config', true, { + tags: [], + }); + }); + + it.each([undefined, '', ' '])('rejects missing or blank SSM_TOKEN_PATH %j before writing', (tokenPath) => { + setTokenPath(tokenPath); + + expect(() => createAwsSsmRunnerConfigStore()).toThrow('Environment variable SSM_TOKEN_PATH is not set'); + expect(putParameterMock).not.toHaveBeenCalled(); + }); + + it.each([ + ['{}', 'Tags must be an array'], + ['[null]', 'Tag at index 0 must be an object'], + [JSON.stringify([{ Key: '', Value: 'test' }]), "Tag at index 0 has missing or invalid 'Key' property"], + [JSON.stringify([{ Key: 'Environment' }]), "Tag at index 0 has missing or invalid 'Value' property"], + ])('rejects invalid legacy SSM parameter tags', (tags, reason) => { + process.env.SSM_PARAMETER_STORE_TAGS = tags; + + expect(() => createAwsSsmRunnerConfigStore()).toThrow(`Failed to parse SSM_PARAMETER_STORE_TAGS: ${reason}`); + expect(putParameterMock).not.toHaveBeenCalled(); + }); + + it('treats a blank legacy tag value as no configured tags', async () => { + process.env.SSM_PARAMETER_STORE_TAGS = ' '; + const store = createAwsSsmRunnerConfigStore(); + + await store.create({ runnerId: 'runner-1', value: 'jit-config' }); + + expect(putParameterMock).toHaveBeenCalledWith('/runner/tokens/runner-1', 'jit-config', true, { tags: [] }); + }); + + it.each(['', '{invalid-json'])('parses cleanup configuration %j during provider construction', (config) => { + process.env.SSM_CLEANUP_CONFIG = config; + + expect(() => createAwsSsmRunnerConfigStore()).toThrow(); + }); +}); + +function setTokenPath(tokenPath: string | undefined): void { + if (tokenPath === undefined) { + delete process.env.SSM_TOKEN_PATH; + } else { + process.env.SSM_TOKEN_PATH = tokenPath; + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts new file mode 100644 index 0000000000..1959a6192a --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-config-store.ts @@ -0,0 +1,58 @@ +import { putParameter } from '@aws-github-runner/aws-ssm-util'; + +import type { RunnerConfigMetadata, RunnerConfigRecord, RunnerConfigStore } from '../../core'; +import type {} from './environment'; +import { loadSsmParameterStoreTagsFromEnvironment } from './parameter-store-tags'; +import { cleanSSMTokens, type SSMCleanupOptions } from './runner-config-housekeeper'; + +interface AwsSsmRunnerConfigStoreConfig { + tokenPath?: string; + parameterStoreTags: { Key: string; Value: string }[]; + cleanupOptions?: SSMCleanupOptions; +} + +export function createAwsSsmRunnerConfigStore(): RunnerConfigStore { + const tokenPath = process.env.SSM_TOKEN_PATH; + const cleanupOptions = + process.env.SSM_CLEANUP_CONFIG !== undefined + ? (JSON.parse(process.env.SSM_CLEANUP_CONFIG) as SSMCleanupOptions) + : undefined; + const hasWriterConfig = tokenPath !== undefined && tokenPath.trim() !== ''; + + if (!hasWriterConfig && cleanupOptions === undefined) { + throw new Error('Environment variable SSM_TOKEN_PATH is not set'); + } + + return new AwsSsmRunnerConfigStore({ + tokenPath: hasWriterConfig ? tokenPath : undefined, + parameterStoreTags: hasWriterConfig ? loadSsmParameterStoreTagsFromEnvironment() : [], + cleanupOptions, + }); +} + +class AwsSsmRunnerConfigStore implements RunnerConfigStore { + readonly maxWritesPerSecond = 40; + + constructor(private readonly config: AwsSsmRunnerConfigStoreConfig) {} + + async create(record: RunnerConfigRecord, options: { metadata?: RunnerConfigMetadata[] } = {}): Promise { + if (!this.config.tokenPath) { + throw new Error('Environment variable SSM_TOKEN_PATH is not set'); + } + + await putParameter(`${this.config.tokenPath}/${record.runnerId}`, record.value, true, { + tags: [ + ...(options.metadata ?? []).map(({ key, value }) => ({ Key: key, Value: value })), + ...this.config.parameterStoreTags, + ], + }); + } + + async houseKeeper(): Promise { + if (!this.config.cleanupOptions) { + throw new Error('Environment variable SSM_CLEANUP_CONFIG is not set'); + } + + await cleanSSMTokens(this.config.cleanupOptions); + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts new file mode 100644 index 0000000000..3943d5a401 --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.test.ts @@ -0,0 +1,72 @@ +import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerGroupCacheStore } from './runner-group-cache-store'; + +vi.mock('@aws-github-runner/aws-ssm-util', () => ({ + getParameter: vi.fn(), + putParameter: vi.fn(), +})); + +const getParameterMock = vi.mocked(getParameter); +const putParameterMock = vi.mocked(putParameter); +const cleanEnv = process.env; + +describe('aws_ssm runner group cache store', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.SSM_PARAMETER_STORE_TAGS; + process.env.SSM_CONFIG_PATH = '/runner/config'; + }); + + it('gets and parses a runner group id from the legacy path', async () => { + getParameterMock.mockResolvedValue('42'); + const store = createAwsSsmRunnerGroupCacheStore(); + + await expect(store.get('Default')).resolves.toBe(42); + expect(getParameterMock).toHaveBeenCalledWith('/runner/config/runner-group/Default'); + }); + + it('preserves the previous parseInt behavior for cached values', async () => { + getParameterMock.mockResolvedValue('42cached'); + const store = createAwsSsmRunnerGroupCacheStore(); + + await expect(store.get('Default')).resolves.toBe(42); + }); + + it('propagates cache read errors', async () => { + const error = new Error('not found'); + getParameterMock.mockRejectedValue(error); + const store = createAwsSsmRunnerGroupCacheStore(); + + await expect(store.get('Default')).rejects.toBe(error); + }); + + it('creates a plaintext parameter at the legacy path with configured tags', async () => { + process.env.SSM_PARAMETER_STORE_TAGS = JSON.stringify([{ Key: 'Environment', Value: 'test' }]); + const store = createAwsSsmRunnerGroupCacheStore(); + + await store.create({ runnerGroupName: 'Default', runnerGroupId: 42 }); + + expect(putParameterMock).toHaveBeenCalledWith('/runner/config/runner-group/Default', '42', false, { + tags: [{ Key: 'Environment', Value: 'test' }], + }); + }); + + it.each([undefined, '', ' '])('rejects missing or blank SSM_CONFIG_PATH %j', (configPath) => { + setConfigPath(configPath); + + expect(() => createAwsSsmRunnerGroupCacheStore()).toThrow('Environment variable SSM_CONFIG_PATH is not set'); + expect(getParameterMock).not.toHaveBeenCalled(); + expect(putParameterMock).not.toHaveBeenCalled(); + }); +}); + +function setConfigPath(configPath: string | undefined): void { + if (configPath === undefined) { + delete process.env.SSM_CONFIG_PATH; + } else { + process.env.SSM_CONFIG_PATH = configPath; + } +} diff --git a/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts new file mode 100644 index 0000000000..f79ee1b9ed --- /dev/null +++ b/lambdas/libs/storage-providers/aws/ssm/runner-group-cache-store.ts @@ -0,0 +1,41 @@ +import { getParameter, putParameter } from '@aws-github-runner/aws-ssm-util'; + +import type { RunnerGroupCacheRecord, RunnerGroupCacheStore } from '../../core'; +import type {} from './environment'; +import { loadSsmParameterStoreTagsFromEnvironment } from './parameter-store-tags'; + +interface AwsSsmRunnerGroupCacheStoreConfig { + configPath: string; + parameterStoreTags: { Key: string; Value: string }[]; +} + +export function createAwsSsmRunnerGroupCacheStore(): RunnerGroupCacheStore { + const configPath = process.env.SSM_CONFIG_PATH; + if (!configPath || configPath.trim() === '') { + throw new Error('Environment variable SSM_CONFIG_PATH is not set'); + } + + return new AwsSsmRunnerGroupCacheStore({ + configPath, + parameterStoreTags: loadSsmParameterStoreTagsFromEnvironment(), + }); +} + +class AwsSsmRunnerGroupCacheStore implements RunnerGroupCacheStore { + constructor(private readonly config: AwsSsmRunnerGroupCacheStoreConfig) {} + + async get(runnerGroupName: string): Promise { + const runnerGroupId = await getParameter(this.parameterName(runnerGroupName)); + return parseInt(runnerGroupId); + } + + async create(record: RunnerGroupCacheRecord): Promise { + await putParameter(this.parameterName(record.runnerGroupName), record.runnerGroupId.toString(), false, { + tags: this.config.parameterStoreTags, + }); + } + + private parameterName(runnerGroupName: string): string { + return `${this.config.configPath}/runner-group/${runnerGroupName}`; + } +} diff --git a/lambdas/libs/storage-providers/core/index.ts b/lambdas/libs/storage-providers/core/index.ts new file mode 100644 index 0000000000..6d590790d3 --- /dev/null +++ b/lambdas/libs/storage-providers/core/index.ts @@ -0,0 +1,53 @@ +export interface GitHubAppCredential { + appId: number; + privateKey: string; + installationId?: number; +} + +export interface GitHubAppCredentialsStore { + get(): Promise; +} + +export interface RunnerConfigMetadata { + key: string; + value: string; +} + +export interface RunnerConfigRecord { + runnerId: string; + value: string; +} + +export interface RunnerConfigStore { + readonly maxWritesPerSecond?: number; + create(record: RunnerConfigRecord, options?: { metadata?: RunnerConfigMetadata[] }): Promise; + houseKeeper(): Promise; +} + +export interface AwsSsmRunnerConfigStorageEnvironment { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm'; + SSM_TOKEN_PATH: string; +} + +export type RunnerConfigStorageContext = AwsSsmRunnerConfigStorageEnvironment; +export type RunnerConfigStorageEnvironment = AwsSsmRunnerConfigStorageEnvironment; + +export interface RunnerConfigConsumeOptions { + /** Absolute Unix time in milliseconds after which the operation must stop. */ + deadlineMs: number; + signal: AbortSignal; +} + +export interface RunnerConfigConsumer { + consume(runnerId: string, options: RunnerConfigConsumeOptions): Promise; +} + +export interface RunnerGroupCacheRecord { + runnerGroupName: string; + runnerGroupId: number; +} + +export interface RunnerGroupCacheStore { + get(runnerGroupName: string): Promise; + create(record: RunnerGroupCacheRecord): Promise; +} diff --git a/lambdas/libs/storage-providers/environment.d.ts b/lambdas/libs/storage-providers/environment.d.ts new file mode 100644 index 0000000000..0f7ade9095 --- /dev/null +++ b/lambdas/libs/storage-providers/environment.d.ts @@ -0,0 +1,9 @@ +export {}; + +declare global { + namespace NodeJS { + interface ProcessEnv { + RUNNER_CONFIG_STORAGE_PROVIDER?: string; + } + } +} diff --git a/lambdas/libs/storage-providers/github-app-credentials.test.ts b/lambdas/libs/storage-providers/github-app-credentials.test.ts new file mode 100644 index 0000000000..fe1870e387 --- /dev/null +++ b/lambdas/libs/storage-providers/github-app-credentials.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import type { GitHubAppCredentialsStore } from './core'; +import { getGitHubAppCredentialsStore, resetGitHubAppCredentialsStore } from './github-app-credentials'; + +vi.mock('./aws/ssm/github-app-credentials-store', () => ({ + createAwsSsmGitHubAppCredentialsStore: vi.fn(), +})); + +const createAwsSsmGitHubAppCredentialsStoreMock = vi.mocked(createAwsSsmGitHubAppCredentialsStore); +const cleanEnv = process.env; + +describe('GitHub App credentials store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetGitHubAppCredentialsStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getGitHubAppCredentialsStore()).toBe(store); + expect(createAwsSsmGitHubAppCredentialsStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getGitHubAppCredentialsStore()).toBe(store); + expect(createAwsSsmGitHubAppCredentialsStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getGitHubAppCredentialsStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmGitHubAppCredentialsStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmGitHubAppCredentialsStoreMock).not.toHaveBeenCalled(); + const first = getGitHubAppCredentialsStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getGitHubAppCredentialsStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmGitHubAppCredentialsStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getGitHubAppCredentialsStore()).toBe(firstStore); + + const secondStore = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsSsmGitHubAppCredentialsStoreMock.mockReturnValue(secondStore); + resetGitHubAppCredentialsStore(); + + expect(getGitHubAppCredentialsStore()).toBe(secondStore); + expect(createAwsSsmGitHubAppCredentialsStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): GitHubAppCredentialsStore { + const store = { get: vi.fn() } satisfies GitHubAppCredentialsStore; + createAwsSsmGitHubAppCredentialsStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/github-app-credentials.ts b/lambdas/libs/storage-providers/github-app-credentials.ts new file mode 100644 index 0000000000..683ab6bb3e --- /dev/null +++ b/lambdas/libs/storage-providers/github-app-credentials.ts @@ -0,0 +1,23 @@ +import { createAwsSsmGitHubAppCredentialsStore } from './aws/ssm/github-app-credentials-store'; +import type { GitHubAppCredentialsStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type GitHubAppCredentialsStoreFactory = () => GitHubAppCredentialsStore; + +const providerFactories = { + aws_ssm: createAwsSsmGitHubAppCredentialsStore, +} as const satisfies Record; + +let githubAppCredentialsStore: GitHubAppCredentialsStore | undefined; + +export function getGitHubAppCredentialsStore(): GitHubAppCredentialsStore { + githubAppCredentialsStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return githubAppCredentialsStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetGitHubAppCredentialsStore(): void { + githubAppCredentialsStore = undefined; +} diff --git a/lambdas/libs/storage-providers/index.ts b/lambdas/libs/storage-providers/index.ts new file mode 100644 index 0000000000..715e7eb61e --- /dev/null +++ b/lambdas/libs/storage-providers/index.ts @@ -0,0 +1,21 @@ +export type { + GitHubAppCredential, + GitHubAppCredentialsStore, + RunnerConfigMetadata, + RunnerConfigRecord, + RunnerConfigStore, + RunnerGroupCacheRecord, + RunnerGroupCacheStore, +} from './core'; +export { getGitHubAppCredentialsStore, resetGitHubAppCredentialsStore } from './github-app-credentials'; +export { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + loadRunnerConfigConsumerConfigFromEnvironment, + loadRunnerConfigStorageContextFromEnvironment, + parseRunnerConfigStorageContext, + runnerConfigStorageEnvironment, + type RunnerConfigConsumerConfig, +} from './runner-config-consumer'; +export { getRunnerConfigStore, resetRunnerConfigStore } from './runner-config'; +export { getRunnerGroupCacheStore, resetRunnerGroupCacheStore } from './runner-group-cache'; diff --git a/lambdas/libs/storage-providers/package.json b/lambdas/libs/storage-providers/package.json new file mode 100644 index 0000000000..15738da9c6 --- /dev/null +++ b/lambdas/libs/storage-providers/package.json @@ -0,0 +1,36 @@ +{ + "name": "@aws-github-runner/storage-providers", + "version": "1.0.0", + "main": "index.ts", + "exports": { + ".": "./index.ts", + "./runner-config-consumer": "./runner-config-consumer.ts" + }, + "type": "module", + "license": "MIT", + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint .", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn format && yarn lint && yarn test" + }, + "devDependencies": { + "aws-sdk-client-mock": "^4.1.0", + "aws-sdk-client-mock-jest": "^4.1.0" + }, + "dependencies": { + "@aws-github-runner/aws-powertools-util": "*", + "@aws-github-runner/aws-ssm-util": "*", + "@aws-sdk/client-ssm": "^3.1009.0" + }, + "nx": { + "includedScripts": [ + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/libs/storage-providers/provider.ts b/lambdas/libs/storage-providers/provider.ts new file mode 100644 index 0000000000..82b2c7895b --- /dev/null +++ b/lambdas/libs/storage-providers/provider.ts @@ -0,0 +1,26 @@ +export const runnerConfigStorageProviders = ['aws_ssm'] as const; + +export type RunnerConfigStorageProvider = (typeof runnerConfigStorageProviders)[number]; + +const defaultProvider = 'aws_ssm' satisfies RunnerConfigStorageProvider; + +export function resolveRunnerConfigStorageProvider(provider: unknown): RunnerConfigStorageProvider { + if (provider === undefined) { + return defaultProvider; + } + + if (typeof provider !== 'string') { + throw new Error(`Unsupported runner config storage provider '${String(provider)}'`); + } + + const normalizedProvider = provider.trim().toLowerCase(); + if (normalizedProvider === '') { + return defaultProvider; + } + + if (!runnerConfigStorageProviders.includes(normalizedProvider as RunnerConfigStorageProvider)) { + throw new Error(`Unsupported runner config storage provider '${String(provider)}'`); + } + + return normalizedProvider as RunnerConfigStorageProvider; +} diff --git a/lambdas/libs/storage-providers/runner-config-consumer-common.ts b/lambdas/libs/storage-providers/runner-config-consumer-common.ts new file mode 100644 index 0000000000..114bf38ad4 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer-common.ts @@ -0,0 +1,220 @@ +import type { RunnerConfigConsumeOptions } from './core'; + +export const DEFAULT_CALL_TIMEOUT_MS = 5_000; +export const DEFAULT_CONFIG_TIMEOUT_MS = 40_000; +export const DEFAULT_POLL_INTERVAL_MS = 2_000; + +const RUNNER_ID_PATTERN = /^[A-Za-z0-9_.-]{1,256}$/; +const SSM_PARAMETER_PATH_PATTERN = /^\/[A-Za-z0-9_.\-/]+$/; +// AWS counts the partition/region/account ARN prefix toward its 1,011-character +// limit. Leave ample room for that deployment-specific prefix. +const MAX_SSM_PARAMETER_NAME_LENGTH = 900; + +const RETRYABLE_ERROR_NAMES = new Set([ + 'AbortError', + 'ConnectionError', + 'InternalServerException', + 'ProvisionedThroughputExceededException', + 'RequestLimitExceeded', + 'RequestTimeout', + 'ServiceUnavailable', + 'ThrottlingException', + 'TimeoutError', +]); + +export interface RunnerConfigPollingOptions { + callTimeoutMs?: number; + configTimeoutMs?: number; + pollIntervalMs?: number; +} + +export interface ResolvedRunnerConfigPollingOptions { + callTimeoutMs: number; + configTimeoutMs: number; + pollIntervalMs: number; +} + +class RunnerConfigCallDeadlineError extends Error { + public constructor() { + super('runner configuration provider call exceeded its deadline'); + this.name = 'RunnerConfigCallDeadlineError'; + } +} + +export function resolvePollingOptions(options: RunnerConfigPollingOptions): ResolvedRunnerConfigPollingOptions { + return { + callTimeoutMs: positiveIntegerOption('callTimeoutMs', options.callTimeoutMs, DEFAULT_CALL_TIMEOUT_MS), + configTimeoutMs: positiveIntegerOption('configTimeoutMs', options.configTimeoutMs, DEFAULT_CONFIG_TIMEOUT_MS), + pollIntervalMs: positiveIntegerOption('pollIntervalMs', options.pollIntervalMs, DEFAULT_POLL_INTERVAL_MS), + }; +} + +export function positiveIntegerOption(name: string, value: number | undefined, fallback: number): number { + if (value === undefined) { + return fallback; + } + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`); + } + return value; +} + +export function validateRunnerId(runnerId: string): void { + if (!RUNNER_ID_PATTERN.test(runnerId)) { + throw new Error('runnerId is invalid'); + } +} + +export function canonicalSsmTokenPath(tokenPath: string): string { + if (tokenPath.includes('//')) { + throw new Error('aws_ssm tokenPath is invalid'); + } + const canonical = tokenPath.endsWith('/') ? tokenPath.slice(0, -1) : tokenPath; + const segments = canonical.split('/').slice(1); + if ( + canonical.length === 0 || + canonical.length > MAX_SSM_PARAMETER_NAME_LENGTH || + !SSM_PARAMETER_PATH_PATTERN.test(canonical) || + segments.length > 14 || + segments.some((segment) => segment === '' || segment === '.' || segment === '..') || + /^(aws|ssm)/i.test(segments[0] ?? '') + ) { + throw new Error('aws_ssm tokenPath is invalid'); + } + return canonical; +} + +export function composeSsmParameterName(tokenPath: string, runnerId: string): string { + validateRunnerId(runnerId); + const parameterName = `${canonicalSsmTokenPath(tokenPath)}/${runnerId}`; + const segments = parameterName.split('/').slice(1); + if (parameterName.length > MAX_SSM_PARAMETER_NAME_LENGTH || segments.length > 15) { + throw new Error('aws_ssm runner configuration key is invalid'); + } + return parameterName; +} + +export function validateConsumeOptions(options: RunnerConfigConsumeOptions): void { + if (!Number.isSafeInteger(options.deadlineMs) || options.deadlineMs <= 0) { + throw new Error('deadlineMs must be a positive integer'); + } + if ( + options.signal === null || + typeof options.signal !== 'object' || + typeof options.signal.aborted !== 'boolean' || + typeof options.signal.addEventListener !== 'function' || + typeof options.signal.removeEventListener !== 'function' + ) { + throw new Error('signal must be an AbortSignal'); + } +} + +export function errorName(error: unknown): string { + if (error !== null && typeof error === 'object' && 'name' in error && typeof error.name === 'string') { + return error.name; + } + return 'UnknownError'; +} + +function httpStatus(error: unknown): number | undefined { + if ( + error !== null && + typeof error === 'object' && + '$metadata' in error && + error.$metadata !== null && + typeof error.$metadata === 'object' && + 'httpStatusCode' in error.$metadata && + typeof error.$metadata.httpStatusCode === 'number' + ) { + return error.$metadata.httpStatusCode; + } + return undefined; +} + +export function isRetryableProviderError(error: unknown): boolean { + const status = httpStatus(error); + return ( + error instanceof RunnerConfigCallDeadlineError || + RETRYABLE_ERROR_NAMES.has(errorName(error)) || + (status !== undefined && status >= 500) + ); +} + +export function delay(ms: number, signal: AbortSignal): Promise { + if (signal.aborted) { + return Promise.reject(new Error('runner configuration consumption was cancelled')); + } + + return new Promise((resolve, reject) => { + let settled = false; + const cleanup = (): void => signal.removeEventListener('abort', cancel); + const finish = (): void => { + if (settled) { + return; + } + settled = true; + cleanup(); + resolve(); + }; + const timer = setTimeout(finish, ms); + const cancel = (): void => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + cleanup(); + reject(new Error('runner configuration consumption was cancelled')); + }; + signal.addEventListener('abort', cancel, { once: true }); + }); +} + +export async function withCallDeadline( + parentSignal: AbortSignal, + deadlineMs: number, + callTimeoutMs: number, + operation: (signal: AbortSignal) => Promise, +): Promise { + if (parentSignal.aborted) { + throw new Error('runner configuration consumption was cancelled'); + } + + const remaining = deadlineMs - Date.now(); + if (remaining <= 0) { + throw new RunnerConfigCallDeadlineError(); + } + + const controller = new AbortController(); + let cancel!: () => void; + let timeout: ReturnType | undefined; + const deadline = new Promise((_resolve, reject) => { + cancel = (): void => { + reject(new Error('runner configuration consumption was cancelled')); + controller.abort(); + }; + parentSignal.addEventListener('abort', cancel, { once: true }); + timeout = setTimeout( + () => { + reject(new RunnerConfigCallDeadlineError()); + controller.abort(); + }, + Math.max(1, Math.min(remaining, callTimeoutMs)), + ); + }); + + try { + return await Promise.race([operation(controller.signal), deadline]); + } finally { + if (timeout !== undefined) { + clearTimeout(timeout); + } + parentSignal.removeEventListener('abort', cancel); + } +} + +export function throwIfCancelled(signal: AbortSignal): void { + if (signal.aborted) { + throw new Error('runner configuration consumption was cancelled'); + } +} diff --git a/lambdas/libs/storage-providers/runner-config-consumer-subpath.test.ts b/lambdas/libs/storage-providers/runner-config-consumer-subpath.test.ts new file mode 100644 index 0000000000..1ed795a5d1 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer-subpath.test.ts @@ -0,0 +1,31 @@ +import { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + loadRunnerConfigStorageContextFromEnvironment, + parseRunnerConfigStorageContext, + type RunnerConfigConsumeOptions, + type RunnerConfigConsumer, + type RunnerConfigStorageContext, + type RunnerConfigStorageEnvironment, +} from '@aws-github-runner/storage-providers/runner-config-consumer'; +import { describe, expect, it } from 'vitest'; + +describe('runner config consumer package subpath', () => { + it('exposes the portable environment round-trip and consumer contract', () => { + const context: RunnerConfigStorageContext = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/runner/tokens', + }; + const options: RunnerConfigConsumeOptions = { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }; + const exported: RunnerConfigStorageEnvironment = exportRunnerConfigStorageEnvironment(context, {}); + const consumerFactory: () => RunnerConfigConsumer = createRunnerConfigConsumerFromEnvironment; + + expect(parseRunnerConfigStorageContext(context)).toEqual(context); + expect(loadRunnerConfigStorageContextFromEnvironment(exported)).toEqual(context); + expect(consumerFactory).toBeTypeOf('function'); + expect(options.signal.aborted).toBe(false); + }); +}); diff --git a/lambdas/libs/storage-providers/runner-config-consumer.test.ts b/lambdas/libs/storage-providers/runner-config-consumer.test.ts new file mode 100644 index 0000000000..e2cc0c6ed4 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer.test.ts @@ -0,0 +1,185 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { AwsSdkSsmRunnerConfigApi, type AwsSsmRunnerConfigApi } from './aws/ssm/runner-config-consumer'; +import { + createRunnerConfigConsumerFromEnvironment, + exportRunnerConfigStorageEnvironment, + loadRunnerConfigConsumerConfigFromEnvironment, + loadRunnerConfigStorageContextFromEnvironment, + parseRunnerConfigStorageContext, + runnerConfigStorageEnvironment, +} from './runner-config-consumer'; + +const ssmContext = { + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: '/runner/tokens', +} as const; +describe('runner config storage context', () => { + it('parses and freezes an exact SSM environment map while canonicalizing one trailing slash', () => { + const context = parseRunnerConfigStorageContext({ ...ssmContext, SSM_TOKEN_PATH: '/runner/tokens/' }); + + expect(context).toEqual(ssmContext); + expect(Object.isFrozen(context)).toBe(true); + expect(runnerConfigStorageEnvironment(context)).toEqual(ssmContext); + }); + + it.each([ + null, + [], + 'aws_ssm', + { RUNNER_CONFIG_STORAGE_PROVIDER: 'AWS_SSM', SSM_TOKEN_PATH: '/runner/tokens' }, + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm' }, + { ...ssmContext, unexpected: true }, + { ...ssmContext, AWS_ACCESS_KEY_ID: 'payload-must-not-export-credentials' }, + { ...ssmContext, RUNNER_CONFIG_TIMEOUT_SECONDS: '60' }, + { ...ssmContext, RUNNER_CONFIG_DYNAMODB_RUNNER_STATE_TABLE_NAME: 'runner-state' }, + { ...ssmContext, SSM_TOKEN_PATH: '/runner//tokens' }, + { ...ssmContext, SSM_TOKEN_PATH: '/runner/tokens//' }, + { ...ssmContext, SSM_TOKEN_PATH: '/awsParameters/tokens' }, + { ...ssmContext, SSM_TOKEN_PATH: '/ssm-private/tokens' }, + { ...ssmContext, SSM_TOKEN_PATH: '/runner/../tokens' }, + { RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_dynamodb', SSM_TOKEN_PATH: '/runner/tokens' }, + { provider: 'aws_dynamodb', tableName: 'runner-state' }, + ])('rejects a non-allowlisted or incomplete storage context %#', (value) => { + expect(() => parseRunnerConfigStorageContext(value)).toThrow(); + }); + + it('rejects symbol fields that would be hidden by JSON-style key enumeration', () => { + const context = { ...ssmContext }; + Object.defineProperty(context, Symbol('unexpected'), { value: true }); + + expect(() => parseRunnerConfigStorageContext(context)).toThrow('storage context is invalid'); + }); + + it.each([ + [ + { + ...ssmContext, + AWS_ACCESS_KEY_ID: 'producer-only', + RUNNER_CONFIG_TIMEOUT_SECONDS: '30', + UNRELATED: 'kept-out', + }, + ssmContext, + ], + ])('selects only the chosen provider locator from a broader producer environment %#', (environment, expected) => { + expect(loadRunnerConfigStorageContextFromEnvironment(environment)).toEqual(expected); + }); + + it('defaults a legacy producer environment with only SSM_TOKEN_PATH to aws_ssm', () => { + expect(loadRunnerConfigStorageContextFromEnvironment({ SSM_TOKEN_PATH: '/runner/tokens' })).toEqual(ssmContext); + }); + + it('round-trips each producer environment through payload context and hook environment export', () => { + for (const producerEnvironment of [ssmContext]) { + const payloadContext = loadRunnerConfigStorageContextFromEnvironment(producerEnvironment); + const hookEnvironment: Record = { + SSM_TOKEN_PATH: '/stale/path', + UNRELATED: 'preserved', + }; + + expect(exportRunnerConfigStorageEnvironment(payloadContext, hookEnvironment)).toEqual(payloadContext); + expect(loadRunnerConfigStorageContextFromEnvironment(hookEnvironment)).toEqual(payloadContext); + expect(hookEnvironment.UNRELATED).toBe('preserved'); + expect(hookEnvironment.SSM_TOKEN_PATH).toBe(ssmContext.SSM_TOKEN_PATH); + } + }); + + it('does not mutate a target when context validation fails', () => { + const target = { ...ssmContext } as Record; + + expect(() => + exportRunnerConfigStorageEnvironment({ ...ssmContext, unexpected: 'forbidden' } as never, target), + ).toThrow(); + expect(target).toEqual(ssmContext); + }); +}); + +describe('runner config consumer environment factory', () => { + afterEach(() => { + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it('creates an injected SSM consumer from exported environment', async () => { + const api: AwsSsmRunnerConfigApi = { + getParameter: vi.fn().mockResolvedValue('encoded-jit'), + deleteParameter: vi.fn().mockResolvedValue(undefined), + }; + const consumer = createRunnerConfigConsumerFromEnvironment(ssmContext, { + awsSsmApi: api, + callTimeoutMs: 100, + configTimeoutMs: 100, + pollIntervalMs: 1, + }); + + await expect( + consumer.consume('microvm-123', { + deadlineMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).resolves.toBe('encoded-jit'); + expect(api.getParameter).toHaveBeenCalledWith('/runner/tokens/microvm-123', expect.any(AbortSignal)); + expect(api.deleteParameter).toHaveBeenCalledWith('/runner/tokens/microvm-123', expect.any(AbortSignal)); + }); + + it('loads timing defaults and overrides from the supplied factory environment', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00.000Z')); + vi.spyOn(AwsSdkSsmRunnerConfigApi.prototype, 'getParameter').mockResolvedValue(undefined); + const consumer = createRunnerConfigConsumerFromEnvironment( + { + ...ssmContext, + RUNNER_CONFIG_TIMEOUT_SECONDS: '1', + RUNNER_CONFIG_POLL_SECONDS: '1', + }, + undefined, + ); + const startedAt = Date.now(); + const pending = consumer.consume('microvm-123', { + deadlineMs: startedAt + 10_000, + signal: new AbortController().signal, + }); + const rejection = expect(pending).rejects.toThrow( + 'runner configuration did not become available before the deadline', + ); + + await vi.runAllTimersAsync(); + + await rejection; + expect(Date.now() - startedAt).toBe(1_000); + }); + + it('loads source-compatible timing defaults', () => { + expect(loadRunnerConfigConsumerConfigFromEnvironment({})).toEqual({ + callTimeoutMs: 5_000, + configTimeoutMs: 20_000, + deleteAttempts: 3, + pollIntervalMs: 2_000, + }); + }); + + it('loads bounded timing overrides from an environment', () => { + expect( + loadRunnerConfigConsumerConfigFromEnvironment({ + AWS_SDK_CALL_TIMEOUT_SECONDS: '7', + RUNNER_CONFIG_TIMEOUT_SECONDS: '31', + RUNNER_CONFIG_DELETE_ATTEMPTS: '4', + RUNNER_CONFIG_POLL_SECONDS: '3', + }), + ).toEqual({ + callTimeoutMs: 7_000, + configTimeoutMs: 31_000, + deleteAttempts: 4, + pollIntervalMs: 3_000, + }); + }); + + it.each([ + ['AWS_SDK_CALL_TIMEOUT_SECONDS', '0', 'callTimeoutMs', 5_000], + ['RUNNER_CONFIG_TIMEOUT_SECONDS', '61', 'configTimeoutMs', 20_000], + ['RUNNER_CONFIG_DELETE_ATTEMPTS', '11', 'deleteAttempts', 3], + ['RUNNER_CONFIG_POLL_SECONDS', 'not-a-number', 'pollIntervalMs', 2_000], + ])('falls back for invalid %s=%j', (name, value, property, expected) => { + expect(loadRunnerConfigConsumerConfigFromEnvironment({ [name]: value })).toHaveProperty(property, expected); + }); +}); diff --git a/lambdas/libs/storage-providers/runner-config-consumer.ts b/lambdas/libs/storage-providers/runner-config-consumer.ts new file mode 100644 index 0000000000..ef2c587eb4 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config-consumer.ts @@ -0,0 +1,145 @@ +import { createAwsSsmRunnerConfigConsumer, type AwsSsmRunnerConfigApi } from './aws/ssm/runner-config-consumer'; +import type { RunnerConfigConsumer, RunnerConfigStorageContext, RunnerConfigStorageEnvironment } from './core'; +import { canonicalSsmTokenPath, type RunnerConfigPollingOptions } from './runner-config-consumer-common'; + +export type { + AwsSsmRunnerConfigStorageEnvironment, + RunnerConfigConsumeOptions, + RunnerConfigConsumer, + RunnerConfigStorageContext, + RunnerConfigStorageEnvironment, +} from './core'; +export type { AwsSsmRunnerConfigApi } from './aws/ssm/runner-config-consumer'; + +const STORAGE_PROVIDER_ENVIRONMENT_VARIABLE = 'RUNNER_CONFIG_STORAGE_PROVIDER'; +const SSM_TOKEN_PATH_ENVIRONMENT_VARIABLE = 'SSM_TOKEN_PATH'; +const STORAGE_ENVIRONMENT_VARIABLES = [ + STORAGE_PROVIDER_ENVIRONMENT_VARIABLE, + SSM_TOKEN_PATH_ENVIRONMENT_VARIABLE, +] as const; + +type Environment = Readonly>; +type MutableEnvironment = Record; + +export interface RunnerConfigConsumerConfig extends RunnerConfigPollingOptions { + awsSsmApi?: AwsSsmRunnerConfigApi; + deleteAttempts?: number; +} + +export function parseRunnerConfigStorageContext(value: unknown): RunnerConfigStorageContext { + if (!isPlainObject(value) || typeof value.RUNNER_CONFIG_STORAGE_PROVIDER !== 'string') { + throw new Error('runner configuration storage context is invalid'); + } + + if (value.RUNNER_CONFIG_STORAGE_PROVIDER === 'aws_ssm') { + if ( + !hasExactKeys(value, [STORAGE_PROVIDER_ENVIRONMENT_VARIABLE, SSM_TOKEN_PATH_ENVIRONMENT_VARIABLE]) || + typeof value.SSM_TOKEN_PATH !== 'string' + ) { + throw new Error('aws_ssm runner configuration storage context is invalid'); + } + return Object.freeze({ + RUNNER_CONFIG_STORAGE_PROVIDER: 'aws_ssm', + SSM_TOKEN_PATH: canonicalSsmTokenPath(value.SSM_TOKEN_PATH), + }); + } + + throw new Error('runner configuration storage provider is unsupported'); +} + +/** Selects only the consumer-safe storage variables from a broader producer environment. */ +export function loadRunnerConfigStorageContextFromEnvironment( + environment: Environment = process.env, +): RunnerConfigStorageContext { + const configuredProvider = environment[STORAGE_PROVIDER_ENVIRONMENT_VARIABLE]; + const provider = + configuredProvider === undefined || configuredProvider.trim() === '' ? 'aws_ssm' : configuredProvider; + + if (provider === 'aws_ssm') { + return parseRunnerConfigStorageContext({ + RUNNER_CONFIG_STORAGE_PROVIDER: provider, + SSM_TOKEN_PATH: environment[SSM_TOKEN_PATH_ENVIRONMENT_VARIABLE], + }); + } + throw new Error('runner configuration storage provider is unsupported'); +} + +/** Revalidates a payload context and returns its exact environment-variable map. */ +export function runnerConfigStorageEnvironment(context: RunnerConfigStorageContext): RunnerConfigStorageEnvironment { + return parseRunnerConfigStorageContext(context); +} + +/** + * Exports only allowlisted storage variables. Provider locators from a previous + * run are cleared so they cannot influence the selected consumer. + */ +export function exportRunnerConfigStorageEnvironment( + context: RunnerConfigStorageContext, + target: MutableEnvironment = process.env, +): RunnerConfigStorageEnvironment { + const environment = runnerConfigStorageEnvironment(context); + for (const name of STORAGE_ENVIRONMENT_VARIABLES) { + delete target[name]; + } + for (const [name, value] of Object.entries(environment)) { + target[name] = value; + } + return environment; +} + +export function createRunnerConfigConsumerFromEnvironment( + environment: Environment = process.env, + config?: RunnerConfigConsumerConfig, +): RunnerConfigConsumer { + const storage = loadRunnerConfigStorageContextFromEnvironment(environment); + const resolvedConfig = config ?? loadRunnerConfigConsumerConfigFromEnvironment(environment); + return createAwsSsmRunnerConfigConsumer(storage, { + api: resolvedConfig.awsSsmApi, + callTimeoutMs: resolvedConfig.callTimeoutMs, + configTimeoutMs: resolvedConfig.configTimeoutMs, + deleteAttempts: resolvedConfig.deleteAttempts, + pollIntervalMs: resolvedConfig.pollIntervalMs, + }); +} + +export function loadRunnerConfigConsumerConfigFromEnvironment( + environment: Environment = process.env, +): RunnerConfigConsumerConfig { + return { + callTimeoutMs: secondsEnvironmentValue(environment, 'AWS_SDK_CALL_TIMEOUT_SECONDS', 5) * 1_000, + configTimeoutMs: secondsEnvironmentValue(environment, 'RUNNER_CONFIG_TIMEOUT_SECONDS', 20) * 1_000, + deleteAttempts: positiveIntegerEnvironmentValue(environment, 'RUNNER_CONFIG_DELETE_ATTEMPTS', 3, 10), + pollIntervalMs: secondsEnvironmentValue(environment, 'RUNNER_CONFIG_POLL_SECONDS', 2) * 1_000, + }; +} + +function secondsEnvironmentValue(environment: Environment, name: string, fallback: number): number { + return positiveIntegerEnvironmentValue(environment, name, fallback, 60); +} + +function positiveIntegerEnvironmentValue( + environment: Environment, + name: string, + fallback: number, + maximum: number, +): number { + const value = environment[name]; + if (value === undefined || !/^\d+$/.test(value)) { + return fallback; + } + const parsed = Number(value); + return Number.isSafeInteger(parsed) && parsed > 0 && parsed <= maximum ? parsed : fallback; +} + +function isPlainObject(value: unknown): value is Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + const prototype = Object.getPrototypeOf(value) as unknown; + return prototype === Object.prototype || prototype === null; +} + +function hasExactKeys(value: object, expected: readonly string[]): boolean { + const keys = Reflect.ownKeys(value); + return keys.length === expected.length && keys.every((key) => typeof key === 'string' && expected.includes(key)); +} diff --git a/lambdas/libs/storage-providers/runner-config.test.ts b/lambdas/libs/storage-providers/runner-config.test.ts new file mode 100644 index 0000000000..95ba3787dd --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config.test.ts @@ -0,0 +1,84 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import type { RunnerConfigStore } from './core'; +import { getRunnerConfigStore, resetRunnerConfigStore } from './runner-config'; + +vi.mock('./aws/ssm/runner-config-store', () => ({ + createAwsSsmRunnerConfigStore: vi.fn(), +})); + +const createAwsSsmRunnerConfigStoreMock = vi.mocked(createAwsSsmRunnerConfigStore); +const cleanEnv = process.env; + +describe('runner config store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerConfigStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerConfigStore()).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + expect(() => getRunnerConfigStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerConfigStoreMock).not.toHaveBeenCalled(); + const first = getRunnerConfigStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerConfigStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerConfigStore()).toBe(firstStore); + + const secondStore = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsSsmRunnerConfigStoreMock.mockReturnValue(secondStore); + resetRunnerConfigStore(); + + expect(getRunnerConfigStore()).toBe(secondStore); + expect(createAwsSsmRunnerConfigStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerConfigStore { + const store = { create: vi.fn(), houseKeeper: vi.fn() } satisfies RunnerConfigStore; + createAwsSsmRunnerConfigStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-config.ts b/lambdas/libs/storage-providers/runner-config.ts new file mode 100644 index 0000000000..180c808d78 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-config.ts @@ -0,0 +1,23 @@ +import { createAwsSsmRunnerConfigStore } from './aws/ssm/runner-config-store'; +import type { RunnerConfigStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerConfigStoreFactory = () => RunnerConfigStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerConfigStore, +} as const satisfies Record; + +let runnerConfigStore: RunnerConfigStore | undefined; + +export function getRunnerConfigStore(): RunnerConfigStore { + runnerConfigStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerConfigStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerConfigStore(): void { + runnerConfigStore = undefined; +} diff --git a/lambdas/libs/storage-providers/runner-group-cache.test.ts b/lambdas/libs/storage-providers/runner-group-cache.test.ts new file mode 100644 index 0000000000..e67e307905 --- /dev/null +++ b/lambdas/libs/storage-providers/runner-group-cache.test.ts @@ -0,0 +1,83 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { RunnerGroupCacheStore } from './core'; +import { getRunnerGroupCacheStore, resetRunnerGroupCacheStore } from './runner-group-cache'; + +vi.mock('./aws/ssm/runner-group-cache-store', () => ({ + createAwsSsmRunnerGroupCacheStore: vi.fn(), +})); + +const createAwsSsmRunnerGroupCacheStoreMock = vi.mocked(createAwsSsmRunnerGroupCacheStore); +const cleanEnv = process.env; + +describe('runner group cache store selection', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env = { ...cleanEnv }; + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + resetRunnerGroupCacheStore(); + }); + + it.each([undefined, '', ' '])('uses aws_ssm for default selector input %j', (provider) => { + setProvider(provider); + const store = stubStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + }); + + it.each(['aws_ssm', ' AWS_SSM '])('uses aws_ssm for explicit selector input %j', (provider) => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + const store = stubStore(); + + expect(getRunnerGroupCacheStore()).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + }); + + it('rejects an unsupported provider on first use', () => { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + + expect(() => getRunnerGroupCacheStore()).toThrow("Unsupported runner config storage provider 'not-registered'"); + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + }); + + it('selects lazily and caches the created store', () => { + const store = stubStore(); + + expect(createAwsSsmRunnerGroupCacheStoreMock).not.toHaveBeenCalled(); + const first = getRunnerGroupCacheStore(); + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = 'not-registered'; + const second = getRunnerGroupCacheStore(); + + expect(first).toBe(store); + expect(second).toBe(store); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledOnce(); + }); + + it('selects again after the test reset', () => { + const firstStore = stubStore(); + expect(getRunnerGroupCacheStore()).toBe(firstStore); + + const secondStore = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsSsmRunnerGroupCacheStoreMock.mockReturnValue(secondStore); + resetRunnerGroupCacheStore(); + + expect(getRunnerGroupCacheStore()).toBe(secondStore); + expect(createAwsSsmRunnerGroupCacheStoreMock).toHaveBeenCalledTimes(2); + }); +}); + +function setProvider(provider: string | undefined): void { + if (provider === undefined) { + delete process.env.RUNNER_CONFIG_STORAGE_PROVIDER; + } else { + process.env.RUNNER_CONFIG_STORAGE_PROVIDER = provider; + } +} + +function stubStore(): RunnerGroupCacheStore { + const store = { get: vi.fn(), create: vi.fn() } satisfies RunnerGroupCacheStore; + createAwsSsmRunnerGroupCacheStoreMock.mockReturnValue(store); + return store; +} diff --git a/lambdas/libs/storage-providers/runner-group-cache.ts b/lambdas/libs/storage-providers/runner-group-cache.ts new file mode 100644 index 0000000000..a28b00f48e --- /dev/null +++ b/lambdas/libs/storage-providers/runner-group-cache.ts @@ -0,0 +1,23 @@ +import { createAwsSsmRunnerGroupCacheStore } from './aws/ssm/runner-group-cache-store'; +import type { RunnerGroupCacheStore } from './core'; +import type {} from './environment'; +import { resolveRunnerConfigStorageProvider, type RunnerConfigStorageProvider } from './provider'; + +type RunnerGroupCacheStoreFactory = () => RunnerGroupCacheStore; + +const providerFactories = { + aws_ssm: createAwsSsmRunnerGroupCacheStore, +} as const satisfies Record; + +let runnerGroupCacheStore: RunnerGroupCacheStore | undefined; + +export function getRunnerGroupCacheStore(): RunnerGroupCacheStore { + runnerGroupCacheStore ??= + providerFactories[resolveRunnerConfigStorageProvider(process.env.RUNNER_CONFIG_STORAGE_PROVIDER)](); + return runnerGroupCacheStore; +} + +// Test-only reset for cases that need to exercise first-use environment selection. +export function resetRunnerGroupCacheStore(): void { + runnerGroupCacheStore = undefined; +} diff --git a/lambdas/libs/storage-providers/tsconfig.json b/lambdas/libs/storage-providers/tsconfig.json new file mode 100644 index 0000000000..139069a7cf --- /dev/null +++ b/lambdas/libs/storage-providers/tsconfig.json @@ -0,0 +1,5 @@ +{ + "extends": "../../tsconfig.json", + "include": ["**/*.ts"], + "exclude": ["**/*.test.ts", "vitest.config.ts"] +} diff --git a/lambdas/libs/storage-providers/vitest.config.ts b/lambdas/libs/storage-providers/vitest.config.ts new file mode 100644 index 0000000000..5ced0aaf52 --- /dev/null +++ b/lambdas/libs/storage-providers/vitest.config.ts @@ -0,0 +1,24 @@ +import { resolve } from 'path'; + +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + test: { + setupFiles: [resolve(__dirname, '../../aws-vitest-setup.ts')], + coverage: { + include: [ + 'index.ts', + 'provider.ts', + 'github-app-credentials.ts', + 'runner-config.ts', + 'runner-config-consumer.ts', + 'runner-config-consumer-common.ts', + 'runner-group-cache.ts', + 'core/**/*.ts', + 'aws/**/*.ts', + ], + exclude: ['**/*.test.ts', '**/*.d.ts'], + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index 58fc0bfc78..91062e1519 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -138,6 +138,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-lambda-microvms": "npm:^3.1074.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -152,8 +153,8 @@ __metadata: resolution: "@aws-github-runner/control-plane@workspace:functions/control-plane" dependencies: "@aws-github-runner/aws-powertools-util": "npm:*" - "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/storage-providers": "npm:*" "@aws-lambda-powertools/parameters": "npm:^2.31.0" "@aws-sdk/client-ec2": "npm:^3.1009.0" "@aws-sdk/client-sqs": "npm:^3.1009.0" @@ -199,6 +200,18 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/storage-providers@npm:*, @aws-github-runner/storage-providers@workspace:libs/storage-providers": + version: 0.0.0-use.local + resolution: "@aws-github-runner/storage-providers@workspace:libs/storage-providers" + dependencies: + "@aws-github-runner/aws-powertools-util": "npm:*" + "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-sdk/client-ssm": "npm:^3.1009.0" + aws-sdk-client-mock: "npm:^4.1.0" + aws-sdk-client-mock-jest: "npm:^4.1.0" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -429,6 +442,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-lambda-microvms@npm:^3.1074.0": + version: 3.1104.0 + resolution: "@aws-sdk/client-lambda-microvms@npm:3.1104.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-node": "npm:^3.972.78" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/219ad52f822def4caa4a20d8d91d46a1b78e6726363a145be86c37cdeef4e4c13653e8a59ada67154146c6c2554e2c12944efad35c689850bf6f72f2d55246f4 + languageName: node + linkType: hard + "@aws-sdk/client-s3@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/client-s3@npm:3.1014.0" @@ -610,6 +639,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/core@npm:^3.977.6": + version: 3.977.6 + resolution: "@aws-sdk/core@npm:3.977.6" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@aws-sdk/xml-builder": "npm:^3.972.37" + "@aws/lambda-invoke-store": "npm:^0.3.0" + "@smithy/core": "npm:^3.31.1" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + bowser: "npm:^2.11.0" + tslib: "npm:^2.6.2" + checksum: 10c0/4d743603bb41aeed426e2928be0947202191c341f9fbefe9ea347b0b4b7154b1ea94189d01c8abf3b03b9635449e2e7c268bd67379ea2294b9f49a61b909b9af + languageName: node + linkType: hard + "@aws-sdk/crc64-nvme@npm:^3.972.5": version: 3.972.5 resolution: "@aws-sdk/crc64-nvme@npm:3.972.5" @@ -633,6 +678,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-env@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-env@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/547bcac01ac0912d0e42bb11f7d51bafcf2eaab1db35a098bea2be322211a86457ea60455a5294e58081c32376c240b07e66f81946a9be30e9722f723c6eaac2 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-http@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-http@npm:3.972.23" @@ -651,6 +709,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-http@npm:^3.972.69": + version: 3.972.69 + resolution: "@aws-sdk/credential-provider-http@npm:3.972.69" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/6e4cf9628919163a2a9784bf8618bc85a8c0ba7056813bedb9758c04eb3b36663f5099cfad329f89ac86c4e408bb3d0698ee7cff7e4a61c8a0335ab98078d678 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-ini@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-ini@npm:3.972.23" @@ -673,6 +746,27 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-ini@npm:^3.973.12": + version: 3.973.12 + resolution: "@aws-sdk/credential-provider-ini@npm:3.973.12" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-login": "npm:^3.972.74" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/84646fee1c61e31b2052d902559ecf163c1d00558ecdc21d77b396250527348b9ebba324d0bf8ffee4b3e45476c691de502e6faad3d39d1f7420eee5d326c7c5 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-login@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-login@npm:3.972.23" @@ -689,6 +783,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-login@npm:^3.972.74": + version: 3.972.74 + resolution: "@aws-sdk/credential-provider-login@npm:3.972.74" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/1ab9996accb61bccbdaefae023e9befab9e5062435370a37f672089457dc13c485d9d2fee6926381672bdb32143c00266b1aa5913b18ef4873584907835b3a92 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-node@npm:^3.972.24": version: 3.972.24 resolution: "@aws-sdk/credential-provider-node@npm:3.972.24" @@ -709,6 +817,25 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-node@npm:^3.972.78": + version: 3.972.78 + resolution: "@aws-sdk/credential-provider-node@npm:3.972.78" + dependencies: + "@aws-sdk/credential-provider-env": "npm:^3.972.67" + "@aws-sdk/credential-provider-http": "npm:^3.972.69" + "@aws-sdk/credential-provider-ini": "npm:^3.973.12" + "@aws-sdk/credential-provider-process": "npm:^3.972.67" + "@aws-sdk/credential-provider-sso": "npm:^3.973.11" + "@aws-sdk/credential-provider-web-identity": "npm:^3.972.73" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/credential-provider-imds": "npm:^4.4.16" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2b6e5bd455a3c2b530a884a0c5919bb7d2d91941b655a56351b957de038d318c1d42b86674e20893ee7dab6db6ea32c4653bce9b1d3ca98ac803d6b49a948343 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-process@npm:^3.972.21": version: 3.972.21 resolution: "@aws-sdk/credential-provider-process@npm:3.972.21" @@ -723,6 +850,19 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-process@npm:^3.972.67": + version: 3.972.67 + resolution: "@aws-sdk/credential-provider-process@npm:3.972.67" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/0381c39f171df2119791b03647545ab5084f6a8d2c227c5d3c5bfa9db027d0566102b6322bc09075c0779b0f0aa88ae1ca7bbdc773d8415d8dd8f163e69e45ea + languageName: node + linkType: hard + "@aws-sdk/credential-provider-sso@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-sso@npm:3.972.23" @@ -739,6 +879,21 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-sso@npm:^3.973.11": + version: 3.973.11 + resolution: "@aws-sdk/credential-provider-sso@npm:3.973.11" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/token-providers": "npm:3.1103.0" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d6df0ae72009c2f74f1c7f12e41c0a7b395ba1860d4f9f1554fd8fbc3b5f0c1be64c83aacd2b329561e27844520ae0b57bb268265f5e7850b1d0fb769455d7a1 + languageName: node + linkType: hard + "@aws-sdk/credential-provider-web-identity@npm:^3.972.23": version: 3.972.23 resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.23" @@ -754,6 +909,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/credential-provider-web-identity@npm:^3.972.73": + version: 3.972.73 + resolution: "@aws-sdk/credential-provider-web-identity@npm:3.972.73" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/a7bee06b4200ff04141d4ce07d49d69f24b55b57f3aab929b445a9d9ea70f043d0b09fca8b95872d2b92df6837b771aeb14b03ca784d17ce1ee871b14173558c + languageName: node + linkType: hard + "@aws-sdk/lib-storage@npm:^3.1009.0": version: 3.1014.0 resolution: "@aws-sdk/lib-storage@npm:3.1014.0" @@ -992,6 +1161,22 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/nested-clients@npm:^3.997.41": + version: 3.997.41 + resolution: "@aws-sdk/nested-clients@npm:3.997.41" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/signature-v4-multi-region": "npm:^3.996.43" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/fetch-http-handler": "npm:^5.6.13" + "@smithy/node-http-handler": "npm:^4.9.13" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/fe1a84bb58675a24ecd0ce3b7bcaf1a456494f10c1a9dd5b55bd268be6713f83bd3c3d3dadee6bb51a53bc24ee18b2b0882d6741bcbabc224feeae97454fdb4a + languageName: node + linkType: hard + "@aws-sdk/region-config-resolver@npm:^3.972.9": version: 3.972.9 resolution: "@aws-sdk/region-config-resolver@npm:3.972.9" @@ -1019,6 +1204,18 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/signature-v4-multi-region@npm:^3.996.43": + version: 3.996.43 + resolution: "@aws-sdk/signature-v4-multi-region@npm:3.996.43" + dependencies: + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/signature-v4": "npm:^5.6.12" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/268608dd5624c6377243903d588b9c13b8de3f3f3e6bea68fc684d125bc92a991fd15a67cb178d1a7a599d0415ce5283f44ba6b96d14909b185d7ff26a9d979b + languageName: node + linkType: hard + "@aws-sdk/token-providers@npm:3.1014.0": version: 3.1014.0 resolution: "@aws-sdk/token-providers@npm:3.1014.0" @@ -1034,6 +1231,20 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/token-providers@npm:3.1103.0": + version: 3.1103.0 + resolution: "@aws-sdk/token-providers@npm:3.1103.0" + dependencies: + "@aws-sdk/core": "npm:^3.977.6" + "@aws-sdk/nested-clients": "npm:^3.997.41" + "@aws-sdk/types": "npm:^3.974.2" + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/5f86aa221e537b8a3fd11ed76ac025935f8859cc62b3af293abd759b8ca3aa390c17f6716723c08056b3373997a17bbdee2ff568eab5049bf0a372d35893b48d + languageName: node + linkType: hard + "@aws-sdk/types@npm:^3.222.0, @aws-sdk/types@npm:^3.4.1, @aws-sdk/types@npm:^3.973.6": version: 3.973.6 resolution: "@aws-sdk/types@npm:3.973.6" @@ -1044,6 +1255,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/types@npm:^3.974.2": + version: 3.974.2 + resolution: "@aws-sdk/types@npm:3.974.2" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b5ce05e8a4160c545edce1e8527e8ac490be7a6651c736f6811190b5d31d5682699889d51186ab0600df756679bebd2df9d650a17f577523441df803c4fb5777 + languageName: node + linkType: hard + "@aws-sdk/util-arn-parser@npm:^3.972.3": version: 3.972.3 resolution: "@aws-sdk/util-arn-parser@npm:3.972.3" @@ -1129,6 +1350,16 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/xml-builder@npm:^3.972.37": + version: 3.972.37 + resolution: "@aws-sdk/xml-builder@npm:3.972.37" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/738f9302f495b3b95602641166a4182244add6e9e079201dba7e8994657dd442df0e4cea3355aa8c7d7f08efb385decaaf0b543f03efdb291c118536f36ac1a1 + languageName: node + linkType: hard + "@aws/lambda-invoke-store@npm:0.2.3, @aws/lambda-invoke-store@npm:^0.2.2": version: 0.2.3 resolution: "@aws/lambda-invoke-store@npm:0.2.3" @@ -1136,6 +1367,13 @@ __metadata: languageName: node linkType: hard +"@aws/lambda-invoke-store@npm:^0.3.0": + version: 0.3.0 + resolution: "@aws/lambda-invoke-store@npm:0.3.0" + checksum: 10c0/b4a2e6b3b5397bc606053e64270d26dc5c886336f88a98cad587b1592eec17058f8fb172f1827a9f0e591f3595cf8f01575c8c9b36cde38c06456f8a65204046 + languageName: node + linkType: hard + "@babel/code-frame@npm:^7.0.0, @babel/code-frame@npm:^7.12.13, @babel/code-frame@npm:^7.28.6, @babel/code-frame@npm:^7.29.0": version: 7.29.0 resolution: "@babel/code-frame@npm:7.29.0" @@ -4556,6 +4794,16 @@ __metadata: languageName: node linkType: hard +"@smithy/core@npm:^3.31.1": + version: 3.31.1 + resolution: "@smithy/core@npm:3.31.1" + dependencies: + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/b953c792dea2c13249b58c1799e4d6aaf21eb1a61e203b83e8e3a9156bebe14ca0585f0ca1ffdf65a193294dddff92a06fbe5c3fbd63ff0c174c88130b47a128 + languageName: node + linkType: hard + "@smithy/credential-provider-imds@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/credential-provider-imds@npm:4.2.12" @@ -4569,6 +4817,17 @@ __metadata: languageName: node linkType: hard +"@smithy/credential-provider-imds@npm:^4.4.16": + version: 4.4.16 + resolution: "@smithy/credential-provider-imds@npm:4.4.16" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/d03687efbbd1f95e77b7dcb639f24f1600671929627cd743f7acf9640238746664e91f955026f22e235603e10537d46e31fa60f231adbdf37457e53720bc80f9 + languageName: node + linkType: hard + "@smithy/eventstream-codec@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/eventstream-codec@npm:4.2.12" @@ -4637,6 +4896,17 @@ __metadata: languageName: node linkType: hard +"@smithy/fetch-http-handler@npm:^5.6.13": + version: 5.6.13 + resolution: "@smithy/fetch-http-handler@npm:5.6.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/028ba8794a6c487ebefae7f40d0124f70e51a1f4e0e465457845c1a44fd607320cd3c64d4a961f159aef59470f0fd43f0d2011b44ee5ef753b7e1dccbdf32ca3 + languageName: node + linkType: hard + "@smithy/hash-blob-browser@npm:^4.2.13": version: 4.2.13 resolution: "@smithy/hash-blob-browser@npm:4.2.13" @@ -4802,6 +5072,17 @@ __metadata: languageName: node linkType: hard +"@smithy/node-http-handler@npm:^4.9.13": + version: 4.9.13 + resolution: "@smithy/node-http-handler@npm:4.9.13" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/2f1cdef7a300ad49c3bb698c2ca4773af5e9202d291cfcd855c1b21ab08b3c4ddf56f3722d3251db4e9b7ac39ec1ebc551b156abf3fa70f74c5491bec421f6b5 + languageName: node + linkType: hard + "@smithy/property-provider@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/property-provider@npm:4.2.12" @@ -4887,6 +5168,17 @@ __metadata: languageName: node linkType: hard +"@smithy/signature-v4@npm:^5.6.12": + version: 5.6.12 + resolution: "@smithy/signature-v4@npm:5.6.12" + dependencies: + "@smithy/core": "npm:^3.31.1" + "@smithy/types": "npm:^4.16.1" + tslib: "npm:^2.6.2" + checksum: 10c0/33656a41ad61dee16209703cb96b46b29014b3c4fad23bfbb90cdb5415ac06c6577b2bfff958ef9e6c19091364945135a0370b12ddc2daed557c903846e81fe7 + languageName: node + linkType: hard + "@smithy/smithy-client@npm:^4.12.7": version: 4.12.7 resolution: "@smithy/smithy-client@npm:4.12.7" @@ -4920,6 +5212,15 @@ __metadata: languageName: node linkType: hard +"@smithy/types@npm:^4.16.1": + version: 4.16.1 + resolution: "@smithy/types@npm:4.16.1" + dependencies: + tslib: "npm:^2.6.2" + checksum: 10c0/e024d9d148deca7bd21d032a9316db109bbe7cf256ffbb8d3981655b9f4f7695c08ec9b87f5a8cf1442e783ba26cb27e4f09603c5bfa3ba1e526c41b1b3e94d2 + languageName: node + linkType: hard + "@smithy/url-parser@npm:^4.2.12": version: 4.2.12 resolution: "@smithy/url-parser@npm:4.2.12"