From c2cf7aad1e32b306e51c4b9c1dea426eeb113605 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Sat, 15 Aug 2026 04:46:16 +0200 Subject: [PATCH 1/2] fix(termination-watcher): preserve custom environment variables --- .github/workflows/terraform.yml | 1 + modules/termination-watcher/main.tf | 7 +- .../tests/termination-watcher.tftest.hcl | 87 +++++++++++++++++++ 3 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 modules/termination-watcher/tests/termination-watcher.tftest.hcl diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index b21ecc544f..c081a9d99a 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -214,6 +214,7 @@ jobs: matrix: module: - modules/runners + - modules/termination-watcher defaults: run: working-directory: ${{ matrix.module }} diff --git a/modules/termination-watcher/main.tf b/modules/termination-watcher/main.tf index 919ba3a3e5..9bded18ed3 100644 --- a/modules/termination-watcher/main.tf +++ b/modules/termination-watcher/main.tf @@ -18,16 +18,11 @@ locals { var.config.github_app_parameters.key_base64.arn, ] : [] - environment_variables = { - ENABLE_METRICS_SPOT_WARNING = var.config.metrics != null ? var.config.metrics.enable && var.config.metrics.metric.enable_spot_termination_warning : false - TAG_FILTERS = jsonencode(var.config.tag_filters) - } - config = merge(var.config, { name = local.name, handler = "index.interruptionWarning", zip = local.lambda_zip, - environment_variables = local.environment_variables + environment_variables = var.config.environment_variables metrics_namespace = var.config.metrics.namespace _deregistration_env_vars = local.deregistration_env_vars _ssm_parameter_arns = local.ssm_parameter_arns diff --git a/modules/termination-watcher/tests/termination-watcher.tftest.hcl b/modules/termination-watcher/tests/termination-watcher.tftest.hcl new file mode 100644 index 0000000000..def0ddf4c6 --- /dev/null +++ b/modules/termination-watcher/tests/termination-watcher.tftest.hcl @@ -0,0 +1,87 @@ +mock_provider "aws" { + mock_data "aws_iam_policy_document" { + defaults = { + json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}" + } + } + + mock_resource "aws_sqs_queue" { + defaults = { + arn = "arn:aws:sqs:eu-west-1:123456789012:termination-watcher-test" + url = "https://sqs.eu-west-1.amazonaws.com/123456789012/termination-watcher-test" + } + } +} + +variables { + config = { + prefix = "termination-watcher-test" + aws_partition = "aws" + + s3_bucket = "lambda-artifacts" + s3_key = "termination-watcher.zip" + + environment_variables = { + CUSTOM_ENV = "preserved" + } + + tag_filters = { + "ghr:environment" = "test" + } + + metrics = { + enable = true + namespace = "TerminationWatcherTest" + metric = { + enable_spot_termination = true + enable_spot_termination_warning = true + } + } + + enable_runner_deregistration = true + github_app_parameters = { + id = { + name = "/github-runner/app-id" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id" + } + key_base64 = { + name = "/github-runner/key-base64" + arn = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64" + } + } + } +} + +run "preserves_component_environment_variables" { + command = plan + + assert { + condition = alltrue([ + output.spot_termination_notification.lambda.function.environment[0].variables["CUSTOM_ENV"] == "preserved", + output.spot_termination_handler.lambda.function.environment[0].variables["CUSTOM_ENV"] == "preserved", + output.deregister_retry.lambda.environment[0].variables["CUSTOM_ENV"] == "preserved", + ]) + error_message = "Caller-provided environment variables must reach every termination-watcher Lambda." + } + + assert { + condition = ( + contains(keys(output.spot_termination_notification.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_WARNING") + && !contains(keys(output.spot_termination_notification.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_TERMINATION") + && contains(keys(output.spot_termination_handler.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_TERMINATION") + && !contains(keys(output.spot_termination_handler.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_WARNING") + && !contains(keys(output.deregister_retry.lambda.environment[0].variables), "ENABLE_METRICS_SPOT_WARNING") + && !contains(keys(output.deregister_retry.lambda.environment[0].variables), "ENABLE_METRICS_SPOT_TERMINATION") + ) + error_message = "Generated metric variables must remain scoped to their owning Lambda." + } + + assert { + condition = alltrue([ + output.spot_termination_notification.lambda.function.environment[0].variables["TAG_FILTERS"] == jsonencode(var.config.tag_filters), + output.spot_termination_handler.lambda.function.environment[0].variables["TAG_FILTERS"] == jsonencode(var.config.tag_filters), + output.deregister_retry.lambda.environment[0].variables["TAG_FILTERS"] == jsonencode(var.config.tag_filters), + ]) + error_message = "Every termination-watcher Lambda must retain TAG_FILTERS." + } +} From 453cbd43563eae084c5427942bb0cc9883876544 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Sat, 15 Aug 2026 04:51:17 +0200 Subject: [PATCH 2/2] fix(termination-watcher): support encrypted app parameters --- main.tf | 1 + modules/multi-runner/termination-watcher.tf | 1 + modules/termination-watcher/README.md | 2 +- .../termination-watcher/deregister-retry.tf | 5 +++ modules/termination-watcher/main.tf | 1 + .../termination-watcher/notification/main.tf | 5 +++ .../termination-watcher/termination/main.tf | 5 +++ .../tests/termination-watcher.tftest.hcl | 34 ++++++++++++++++++- modules/termination-watcher/variables.tf | 2 ++ 9 files changed, 54 insertions(+), 2 deletions(-) diff --git a/main.tf b/main.tf index cad9b66c58..290f80a1d5 100644 --- a/main.tf +++ b/main.tf @@ -398,6 +398,7 @@ locals { log_level = var.log_level log_class = var.log_class logging_kms_key_id = var.logging_kms_key_id + ssm_kms_key_id = var.kms_key_arn logging_retention_in_days = var.logging_retention_in_days role_path = var.role_path role_permissions_boundary = var.role_permissions_boundary diff --git a/modules/multi-runner/termination-watcher.tf b/modules/multi-runner/termination-watcher.tf index 750db361bf..6c08765e25 100644 --- a/modules/multi-runner/termination-watcher.tf +++ b/modules/multi-runner/termination-watcher.tf @@ -11,6 +11,7 @@ locals { log_level = var.log_level log_class = var.log_class logging_kms_key_id = var.logging_kms_key_id + ssm_kms_key_id = var.kms_key_arn logging_retention_in_days = var.logging_retention_in_days role_path = var.role_path role_permissions_boundary = var.role_permissions_boundary diff --git a/modules/termination-watcher/README.md b/modules/termination-watcher/README.md index 4cdf37f13b..ab751ddc8a 100644 --- a/modules/termination-watcher/README.md +++ b/modules/termination-watcher/README.md @@ -94,7 +94,7 @@ yarn run dist | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [config](#input\_config) | Configuration for the spot termination watcher.

`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
'features': Features to enable the different lambda functions to handle spot termination events.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners.
`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | 
object({
    aws_partition         = optional(string, null)
    architecture          = optional(string, null)
    environment_variables = optional(map(string), {})
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    lambda_tags               = optional(map(string), {})
    log_level                 = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    logging_kms_key_id        = optional(string, null)
    logging_retention_in_days = optional(number, null)
    memory_size               = optional(number, null)
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_spot_termination         = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
    prefix = optional(string, null)
    principals = optional(list(object({
      type        = string
      identifiers = list(string)
    })), [])
    role_path                 = optional(string, null)
    role_permissions_boundary = optional(string, null)
    runtime                   = optional(string, null)
    s3_bucket                 = optional(string, null)
    s3_key                    = optional(string, null)
    s3_object_version         = optional(string, null)
    security_group_ids        = optional(list(string), [])
    subnet_ids                = optional(list(string), [])
    tag_filters               = optional(map(string), null)
    tags                      = optional(map(string), {})
    timeout                   = optional(number, null)
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    zip                          = optional(string, null)
    enable_runner_deregistration = optional(bool, false)
    github_app_parameters = optional(object({
      id         = map(string)
      key_base64 = map(string)
    }), null)
    ghes_url = optional(string, null)
  })
| n/a | yes | +| [config](#input\_config) | Configuration for the spot termination watcher.

`aws_partition`: Partition for the base arn if not 'aws'
`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.
`environment_variables`: Environment variables for the lambda.
'features': Features to enable the different lambda functions to handle spot termination events.
`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.
`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.
`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.
`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with
`ssm_kms_key_id`: Optional KMS key ARN used to decrypt GitHub App parameters while deregistering runners. The ARN may be unknown until apply.
`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.
`memory_size`: Memory size limit in MB of the lambda.
`prefix`: The prefix used for naming resources.
`role_path`: The path that will be added to the role, if not set the environment name will be used.
`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.
`runtime`: AWS Lambda runtime.
`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.
`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.
`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.
`security_group_ids`: List of security group IDs associated with the Lambda function.
`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.
`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.
`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.
`timeout`: Time out of the lambda in seconds.
`tracing_config`: Configuration for lambda tracing.
`zip`: File location of the lambda zip file.
`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.
`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners.
`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | 
object({
    aws_partition         = optional(string, null)
    architecture          = optional(string, null)
    environment_variables = optional(map(string), {})
    features = optional(object({
      enable_spot_termination_handler              = optional(bool, true)
      enable_spot_termination_notification_watcher = optional(bool, true)
    }), {})
    lambda_tags               = optional(map(string), {})
    log_level                 = optional(string, null)
    log_class                 = optional(string, "STANDARD")
    logging_kms_key_id        = optional(string, null)
    ssm_kms_key_id            = optional(string, null)
    logging_retention_in_days = optional(number, null)
    memory_size               = optional(number, null)
    metrics = optional(object({
      enable    = optional(bool, false)
      namespace = optional(string, "GitHub Runners")
      metric = optional(object({
        enable_spot_termination         = optional(bool, true)
        enable_spot_termination_warning = optional(bool, true)
      }), {})
    }), {})
    prefix = optional(string, null)
    principals = optional(list(object({
      type        = string
      identifiers = list(string)
    })), [])
    role_path                 = optional(string, null)
    role_permissions_boundary = optional(string, null)
    runtime                   = optional(string, null)
    s3_bucket                 = optional(string, null)
    s3_key                    = optional(string, null)
    s3_object_version         = optional(string, null)
    security_group_ids        = optional(list(string), [])
    subnet_ids                = optional(list(string), [])
    tag_filters               = optional(map(string), null)
    tags                      = optional(map(string), {})
    timeout                   = optional(number, null)
    tracing_config = optional(object({
      mode                  = optional(string, null)
      capture_http_requests = optional(bool, false)
      capture_error         = optional(bool, false)
    }), {})
    zip                          = optional(string, null)
    enable_runner_deregistration = optional(bool, false)
    github_app_parameters = optional(object({
      id         = map(string)
      key_base64 = map(string)
    }), null)
    ghes_url = optional(string, null)
  })
| n/a | yes | ## Outputs diff --git a/modules/termination-watcher/deregister-retry.tf b/modules/termination-watcher/deregister-retry.tf index 921d7abf5e..a856274b0f 100644 --- a/modules/termination-watcher/deregister-retry.tf +++ b/modules/termination-watcher/deregister-retry.tf @@ -101,6 +101,11 @@ resource "aws_iam_role_policy" "deregister_retry_ssm" { Effect = "Allow" Action = ["ssm:GetParameter"] Resource = local.ssm_parameter_arns + }, + { + Effect = "Allow" + Action = ["kms:Decrypt"] + Resource = [local.config._ssm_kms_key_id] } ] }) diff --git a/modules/termination-watcher/main.tf b/modules/termination-watcher/main.tf index 9bded18ed3..145e19380b 100644 --- a/modules/termination-watcher/main.tf +++ b/modules/termination-watcher/main.tf @@ -26,6 +26,7 @@ locals { metrics_namespace = var.config.metrics.namespace _deregistration_env_vars = local.deregistration_env_vars _ssm_parameter_arns = local.ssm_parameter_arns + _ssm_kms_key_id = coalesce(var.config.ssm_kms_key_id, "arn:${coalesce(var.config.aws_partition, "aws")}:kms:*:000000000000:key/00000000-0000-0000-0000-000000000000") _enable_runner_deregistration = local.enable_runner_deregistration }) } diff --git a/modules/termination-watcher/notification/main.tf b/modules/termination-watcher/notification/main.tf index 735c34126b..7683a760f9 100644 --- a/modules/termination-watcher/notification/main.tf +++ b/modules/termination-watcher/notification/main.tf @@ -102,6 +102,11 @@ resource "aws_iam_role_policy" "ssm_policy" { Effect = "Allow" Action = ["ssm:GetParameter"] Resource = var.config._ssm_parameter_arns + }, + { + Effect = "Allow" + Action = ["kms:Decrypt"] + Resource = [var.config._ssm_kms_key_id] } ] }) diff --git a/modules/termination-watcher/termination/main.tf b/modules/termination-watcher/termination/main.tf index f43b61775a..d96e6f820e 100644 --- a/modules/termination-watcher/termination/main.tf +++ b/modules/termination-watcher/termination/main.tf @@ -66,6 +66,11 @@ resource "aws_iam_role_policy" "ssm_policy" { Effect = "Allow" Action = ["ssm:GetParameter"] Resource = var.config._ssm_parameter_arns + }, + { + Effect = "Allow" + Action = ["kms:Decrypt"] + Resource = [var.config._ssm_kms_key_id] } ] }) diff --git a/modules/termination-watcher/tests/termination-watcher.tftest.hcl b/modules/termination-watcher/tests/termination-watcher.tftest.hcl index def0ddf4c6..d7714240d8 100644 --- a/modules/termination-watcher/tests/termination-watcher.tftest.hcl +++ b/modules/termination-watcher/tests/termination-watcher.tftest.hcl @@ -39,6 +39,7 @@ variables { } enable_runner_deregistration = true + ssm_kms_key_id = "arn:aws:kms:eu-west-1:123456789012:key/termination-watcher-test" github_app_parameters = { id = { name = "/github-runner/app-id" @@ -52,7 +53,7 @@ variables { } } -run "preserves_component_environment_variables" { +run "preserves_environment_and_configures_kms_access" { command = plan assert { @@ -84,4 +85,35 @@ run "preserves_component_environment_variables" { ]) error_message = "Every termination-watcher Lambda must retain TAG_FILTERS." } + + assert { + condition = local.config._ssm_kms_key_id == var.config.ssm_kms_key_id + error_message = "The configured Parameter Store KMS key must reach the canonical watcher configuration." + } + + assert { + condition = anytrue([ + for statement in jsondecode(aws_iam_role_policy.deregister_retry_ssm[0].policy).Statement : + contains(statement.Action, "kms:Decrypt") + && contains(statement.Resource, var.config.ssm_kms_key_id) + ]) + error_message = "The deregistration-retry role must receive KMS decrypt access scoped to the configured key." + } +} + +run "uses_inert_kms_arn_when_unset" { + command = plan + + variables { + config = { + prefix = "termination-watcher-no-kms" + s3_bucket = "lambda-artifacts" + s3_key = "termination-watcher.zip" + } + } + + assert { + condition = local.config._ssm_kms_key_id == "arn:aws:kms:*:000000000000:key/00000000-0000-0000-0000-000000000000" + error_message = "An unset Parameter Store KMS key must retain a static, inert IAM resource shape." + } } diff --git a/modules/termination-watcher/variables.tf b/modules/termination-watcher/variables.tf index a72bf74916..4d60abc5e8 100644 --- a/modules/termination-watcher/variables.tf +++ b/modules/termination-watcher/variables.tf @@ -11,6 +11,7 @@ variable "config" { `log_level`: Logging level for lambda logging. Valid values are 'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'. `log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`. `logging_kms_key_id`: Specifies the kms key id to encrypt the logs with + `ssm_kms_key_id`: Optional KMS key ARN used to decrypt GitHub App parameters while deregistering runners. The ARN may be unknown until apply. `logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653. `memory_size`: Memory size limit in MB of the lambda. `prefix`: The prefix used for naming resources. @@ -43,6 +44,7 @@ variable "config" { log_level = optional(string, null) log_class = optional(string, "STANDARD") logging_kms_key_id = optional(string, null) + ssm_kms_key_id = optional(string, null) logging_retention_in_days = optional(number, null) memory_size = optional(number, null) metrics = optional(object({