fix(termination-watcher): preserve env vars and KMS access

.github/workflows/terraform.yml

@@ -214,6 +214,7 @@ jobs:
       matrix:
         module:
           - modules/runners
+          - modules/termination-watcher
     defaults:
       run:
         working-directory: ${{ matrix.module }}

main.tf

@@ -398,6 +398,7 @@ locals {
     log_level                    = var.log_level
     log_class                    = var.log_class
     logging_kms_key_id           = var.logging_kms_key_id
+    ssm_kms_key_id               = var.kms_key_arn
     logging_retention_in_days    = var.logging_retention_in_days
     role_path                    = var.role_path
     role_permissions_boundary    = var.role_permissions_boundary

modules/multi-runner/termination-watcher.tf

@@ -11,6 +11,7 @@ locals {
     log_level                    = var.log_level
     log_class                    = var.log_class
     logging_kms_key_id           = var.logging_kms_key_id
+    ssm_kms_key_id               = var.kms_key_arn
     logging_retention_in_days    = var.logging_retention_in_days
     role_path                    = var.role_path
     role_permissions_boundary    = var.role_permissions_boundary

modules/termination-watcher/README.md

@@ -94,7 +94,7 @@ yarn run dist
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_config"></a> [config](#input\_config) | Configuration for the spot termination watcher.<br/><br/>`aws_partition`: Partition for the base arn if not 'aws'<br/>`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.<br/>`environment_variables`: Environment variables for the lambda.<br/>'features': Features to enable the different lambda functions to handle spot termination events.<br/>`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.<br/>`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.<br/>`log_level`: Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.<br/>`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.<br/>`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with<br/>`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.<br/>`memory_size`: Memory size limit in MB of the lambda.<br/>`prefix`: The prefix used for naming resources.<br/>`role_path`: The path that will be added to the role, if not set the environment name will be used.<br/>`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.<br/>`runtime`: AWS Lambda runtime.<br/>`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.<br/>`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.<br/>`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.<br/>`security_group_ids`: List of security group IDs associated with the Lambda function.<br/>`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.<br/>`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.<br/>`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.<br/>`timeout`: Time out of the lambda in seconds.<br/>`tracing_config`: Configuration for lambda tracing.<br/>`zip`: File location of the lambda zip file.<br/>`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.<br/>`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners.<br/>`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | <pre>object({<br/>    aws_partition         = optional(string, null)<br/>    architecture          = optional(string, null)<br/>    environment_variables = optional(map(string), {})<br/>    features = optional(object({<br/>      enable_spot_termination_handler              = optional(bool, true)<br/>      enable_spot_termination_notification_watcher = optional(bool, true)<br/>    }), {})<br/>    lambda_tags               = optional(map(string), {})<br/>    log_level                 = optional(string, null)<br/>    log_class                 = optional(string, "STANDARD")<br/>    logging_kms_key_id        = optional(string, null)<br/>    logging_retention_in_days = optional(number, null)<br/>    memory_size               = optional(number, null)<br/>    metrics = optional(object({<br/>      enable    = optional(bool, false)<br/>      namespace = optional(string, "GitHub Runners")<br/>      metric = optional(object({<br/>        enable_spot_termination         = optional(bool, true)<br/>        enable_spot_termination_warning = optional(bool, true)<br/>      }), {})<br/>    }), {})<br/>    prefix = optional(string, null)<br/>    principals = optional(list(object({<br/>      type        = string<br/>      identifiers = list(string)<br/>    })), [])<br/>    role_path                 = optional(string, null)<br/>    role_permissions_boundary = optional(string, null)<br/>    runtime                   = optional(string, null)<br/>    s3_bucket                 = optional(string, null)<br/>    s3_key                    = optional(string, null)<br/>    s3_object_version         = optional(string, null)<br/>    security_group_ids        = optional(list(string), [])<br/>    subnet_ids                = optional(list(string), [])<br/>    tag_filters               = optional(map(string), null)<br/>    tags                      = optional(map(string), {})<br/>    timeout                   = optional(number, null)<br/>    tracing_config = optional(object({<br/>      mode                  = optional(string, null)<br/>      capture_http_requests = optional(bool, false)<br/>      capture_error         = optional(bool, false)<br/>    }), {})<br/>    zip                          = optional(string, null)<br/>    enable_runner_deregistration = optional(bool, false)<br/>    github_app_parameters = optional(object({<br/>      id         = map(string)<br/>      key_base64 = map(string)<br/>    }), null)<br/>    ghes_url = optional(string, null)<br/>  })</pre> | n/a | yes |
+| <a name="input_config"></a> [config](#input\_config) | Configuration for the spot termination watcher.<br/><br/>`aws_partition`: Partition for the base arn if not 'aws'<br/>`architecture`: AWS Lambda architecture. Lambda functions using Graviton processors ('arm64') tend to have better price/performance than 'x86\_64' functions.<br/>`environment_variables`: Environment variables for the lambda.<br/>'features': Features to enable the different lambda functions to handle spot termination events.<br/>`lambda_principals`: Add extra principals to the role created for execution of the lambda, e.g. for local testing.<br/>`lambda_tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.<br/>`log_level`: Logging level for lambda logging. Valid values are  'silly', 'trace', 'debug', 'info', 'warn', 'error', 'fatal'.<br/>`log_class`: The log class of the CloudWatch log group. Valid values are `STANDARD` or `INFREQUENT_ACCESS`.<br/>`logging_kms_key_id`: Specifies the kms key id to encrypt the logs with<br/>`ssm_kms_key_id`: Optional KMS key ARN used to decrypt GitHub App parameters while deregistering runners. The ARN may be unknown until apply.<br/>`logging_retention_in_days`: Specifies the number of days you want to retain log events for the lambda log group. Possible values are: 0, 1, 3, 5, 7, 14, 30, 60, 90, 120, 150, 180, 365, 400, 545, 731, 1827, and 3653.<br/>`memory_size`: Memory size limit in MB of the lambda.<br/>`prefix`: The prefix used for naming resources.<br/>`role_path`: The path that will be added to the role, if not set the environment name will be used.<br/>`role_permissions_boundary`: Permissions boundary that will be added to the created role for the lambda.<br/>`runtime`: AWS Lambda runtime.<br/>`s3_bucket`: S3 bucket from which to specify lambda functions. This is an alternative to providing local files directly.<br/>`s3_key`: S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas.<br/>`s3_object_version`: S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket.<br/>`security_group_ids`: List of security group IDs associated with the Lambda function.<br/>`subnet_ids`: List of subnets in which the action runners will be launched, the subnets needs to be subnets in the `vpc_id`.<br/>`tag_filters`: Map of tags that will be used to filter the resources to be tracked. Only for which all tags are present and starting with the same value as the value in the map will be tracked.<br/>`tags`: Map of tags that will be added to created resources. By default resources will be tagged with name and environment.<br/>`timeout`: Time out of the lambda in seconds.<br/>`tracing_config`: Configuration for lambda tracing.<br/>`zip`: File location of the lambda zip file.<br/>`enable_runner_deregistration`: Enable or disable deregistering the runner from GitHub when its EC2 instance is terminated.<br/>`github_app_parameters`: GitHub App SSM parameters (`id` and `key_base64`, each a map of `arn`/`name`) used to authenticate to GitHub when deregistering runners.<br/>`ghes_url`: GitHub Enterprise Server URL used to target the GHES API when deregistering runners. Leave `null` for github.com. | <pre>object({<br/>    aws_partition         = optional(string, null)<br/>    architecture          = optional(string, null)<br/>    environment_variables = optional(map(string), {})<br/>    features = optional(object({<br/>      enable_spot_termination_handler              = optional(bool, true)<br/>      enable_spot_termination_notification_watcher = optional(bool, true)<br/>    }), {})<br/>    lambda_tags               = optional(map(string), {})<br/>    log_level                 = optional(string, null)<br/>    log_class                 = optional(string, "STANDARD")<br/>    logging_kms_key_id        = optional(string, null)<br/>    ssm_kms_key_id            = optional(string, null)<br/>    logging_retention_in_days = optional(number, null)<br/>    memory_size               = optional(number, null)<br/>    metrics = optional(object({<br/>      enable    = optional(bool, false)<br/>      namespace = optional(string, "GitHub Runners")<br/>      metric = optional(object({<br/>        enable_spot_termination         = optional(bool, true)<br/>        enable_spot_termination_warning = optional(bool, true)<br/>      }), {})<br/>    }), {})<br/>    prefix = optional(string, null)<br/>    principals = optional(list(object({<br/>      type        = string<br/>      identifiers = list(string)<br/>    })), [])<br/>    role_path                 = optional(string, null)<br/>    role_permissions_boundary = optional(string, null)<br/>    runtime                   = optional(string, null)<br/>    s3_bucket                 = optional(string, null)<br/>    s3_key                    = optional(string, null)<br/>    s3_object_version         = optional(string, null)<br/>    security_group_ids        = optional(list(string), [])<br/>    subnet_ids                = optional(list(string), [])<br/>    tag_filters               = optional(map(string), null)<br/>    tags                      = optional(map(string), {})<br/>    timeout                   = optional(number, null)<br/>    tracing_config = optional(object({<br/>      mode                  = optional(string, null)<br/>      capture_http_requests = optional(bool, false)<br/>      capture_error         = optional(bool, false)<br/>    }), {})<br/>    zip                          = optional(string, null)<br/>    enable_runner_deregistration = optional(bool, false)<br/>    github_app_parameters = optional(object({<br/>      id         = map(string)<br/>      key_base64 = map(string)<br/>    }), null)<br/>    ghes_url = optional(string, null)<br/>  })</pre> | n/a | yes |
 
 ## Outputs
 

modules/termination-watcher/deregister-retry.tf

@@ -101,6 +101,11 @@ resource "aws_iam_role_policy" "deregister_retry_ssm" {
         Effect   = "Allow"
         Action   = ["ssm:GetParameter"]
         Resource = local.ssm_parameter_arns
+      },
+      {
+        Effect   = "Allow"
+        Action   = ["kms:Decrypt"]
+        Resource = [local.config._ssm_kms_key_id]
       }
     ]
   })

modules/termination-watcher/main.tf

@@ -18,19 +18,15 @@ locals {
     var.config.github_app_parameters.key_base64.arn,
   ] : []
 
-  environment_variables = {
-    ENABLE_METRICS_SPOT_WARNING = var.config.metrics != null ? var.config.metrics.enable && var.config.metrics.metric.enable_spot_termination_warning : false
-    TAG_FILTERS                 = jsonencode(var.config.tag_filters)
-  }
-
   config = merge(var.config, {
     name                          = local.name,
     handler                       = "index.interruptionWarning",
     zip                           = local.lambda_zip,
-    environment_variables         = local.environment_variables
+    environment_variables         = var.config.environment_variables
     metrics_namespace             = var.config.metrics.namespace
     _deregistration_env_vars      = local.deregistration_env_vars
     _ssm_parameter_arns           = local.ssm_parameter_arns
+    _ssm_kms_key_id               = coalesce(var.config.ssm_kms_key_id, "arn:${coalesce(var.config.aws_partition, "aws")}:kms:*:000000000000:key/00000000-0000-0000-0000-000000000000")
     _enable_runner_deregistration = local.enable_runner_deregistration
   })
 }

modules/termination-watcher/notification/main.tf

@@ -102,6 +102,11 @@ resource "aws_iam_role_policy" "ssm_policy" {
         Effect   = "Allow"
         Action   = ["ssm:GetParameter"]
         Resource = var.config._ssm_parameter_arns
+      },
+      {
+        Effect   = "Allow"
+        Action   = ["kms:Decrypt"]
+        Resource = [var.config._ssm_kms_key_id]
       }
     ]
   })

modules/termination-watcher/termination/main.tf

@@ -66,6 +66,11 @@ resource "aws_iam_role_policy" "ssm_policy" {
         Effect   = "Allow"
         Action   = ["ssm:GetParameter"]
         Resource = var.config._ssm_parameter_arns
+      },
+      {
+        Effect   = "Allow"
+        Action   = ["kms:Decrypt"]
+        Resource = [var.config._ssm_kms_key_id]
       }
     ]
   })

modules/termination-watcher/tests/termination-watcher.tftest.hcl

@@ -0,0 +1,119 @@
+mock_provider "aws" {
+  mock_data "aws_iam_policy_document" {
+    defaults = {
+      json = "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"Service\":\"lambda.amazonaws.com\"},\"Action\":\"sts:AssumeRole\"}]}"
+    }
+  }
+
+  mock_resource "aws_sqs_queue" {
+    defaults = {
+      arn = "arn:aws:sqs:eu-west-1:123456789012:termination-watcher-test"
+      url = "https://sqs.eu-west-1.amazonaws.com/123456789012/termination-watcher-test"
+    }
+  }
+}
+
+variables {
+  config = {
+    prefix        = "termination-watcher-test"
+    aws_partition = "aws"
+
+    s3_bucket = "lambda-artifacts"
+    s3_key    = "termination-watcher.zip"
+
+    environment_variables = {
+      CUSTOM_ENV = "preserved"
+    }
+
+    tag_filters = {
+      "ghr:environment" = "test"
+    }
+
+    metrics = {
+      enable    = true
+      namespace = "TerminationWatcherTest"
+      metric = {
+        enable_spot_termination         = true
+        enable_spot_termination_warning = true
+      }
+    }
+
+    enable_runner_deregistration = true
+    ssm_kms_key_id               = "arn:aws:kms:eu-west-1:123456789012:key/termination-watcher-test"
+    github_app_parameters = {
+      id = {
+        name = "/github-runner/app-id"
+        arn  = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/app-id"
+      }
+      key_base64 = {
+        name = "/github-runner/key-base64"
+        arn  = "arn:aws:ssm:eu-west-1:123456789012:parameter/github-runner/key-base64"
+      }
+    }
+  }
+}
+
+run "preserves_environment_and_configures_kms_access" {
+  command = plan
+
+  assert {
+    condition = alltrue([
+      output.spot_termination_notification.lambda.function.environment[0].variables["CUSTOM_ENV"] == "preserved",
+      output.spot_termination_handler.lambda.function.environment[0].variables["CUSTOM_ENV"] == "preserved",
+      output.deregister_retry.lambda.environment[0].variables["CUSTOM_ENV"] == "preserved",
+    ])
+    error_message = "Caller-provided environment variables must reach every termination-watcher Lambda."
+  }
+
+  assert {
+    condition = (
+      contains(keys(output.spot_termination_notification.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_WARNING")
+      && !contains(keys(output.spot_termination_notification.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_TERMINATION")
+      && contains(keys(output.spot_termination_handler.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_TERMINATION")
+      && !contains(keys(output.spot_termination_handler.lambda.function.environment[0].variables), "ENABLE_METRICS_SPOT_WARNING")
+      && !contains(keys(output.deregister_retry.lambda.environment[0].variables), "ENABLE_METRICS_SPOT_WARNING")
+      && !contains(keys(output.deregister_retry.lambda.environment[0].variables), "ENABLE_METRICS_SPOT_TERMINATION")
+    )
+    error_message = "Generated metric variables must remain scoped to their owning Lambda."
+  }
+
+  assert {
+    condition = alltrue([
+      output.spot_termination_notification.lambda.function.environment[0].variables["TAG_FILTERS"] == jsonencode(var.config.tag_filters),
+      output.spot_termination_handler.lambda.function.environment[0].variables["TAG_FILTERS"] == jsonencode(var.config.tag_filters),
+      output.deregister_retry.lambda.environment[0].variables["TAG_FILTERS"] == jsonencode(var.config.tag_filters),
+    ])
+    error_message = "Every termination-watcher Lambda must retain TAG_FILTERS."
+  }
+
+  assert {
+    condition     = local.config._ssm_kms_key_id == var.config.ssm_kms_key_id
+    error_message = "The configured Parameter Store KMS key must reach the canonical watcher configuration."
+  }
+
+  assert {
+    condition = anytrue([
+      for statement in jsondecode(aws_iam_role_policy.deregister_retry_ssm[0].policy).Statement :
+      contains(statement.Action, "kms:Decrypt")
+      && contains(statement.Resource, var.config.ssm_kms_key_id)
+    ])
+    error_message = "The deregistration-retry role must receive KMS decrypt access scoped to the configured key."
+  }
+}
+
+run "uses_inert_kms_arn_when_unset" {
+  command = plan
+
+  variables {
+    config = {
+      prefix    = "termination-watcher-no-kms"
+      s3_bucket = "lambda-artifacts"
+      s3_key    = "termination-watcher.zip"
+    }
+  }
+
+  assert {
+    condition     = local.config._ssm_kms_key_id == "arn:aws:kms:*:000000000000:key/00000000-0000-0000-0000-000000000000"
+    error_message = "An unset Parameter Store KMS key must retain a static, inert IAM resource shape."
+  }
+}